Skip to content

Commit 5052001

Browse files
kevinjqliukevinjqliu
committed
use cachetools
1 parent de47590 commit 5052001

File tree

4 files changed

+344
-4
lines changed

4 files changed

+344
-4
lines changed

‎poetry.lock‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyiceberg/table/snapshots.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@
1919
import time
2020
from collections import defaultdict
2121
from enum import Enum
22-
from functools import lru_cache
2322
from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional
2423

24+
from cachetools import cached
25+
from cachetools.keys import hashkey
2526
from pydantic import Field, PrivateAttr, model_serializer
2627

2728
from pyiceberg.io import FileIO
@@ -231,7 +232,7 @@ def __eq__(self, other: Any) -> bool:
231232
)
232233

233234

234-
@lru_cache
235+
@cached(cache={}, key=lambda io, manifest_list: hashkey(manifest_list))
235236
def _manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]:
236237
"""Return the manifests from the manifest list."""
237238
file = io.new_input(manifest_list)

‎pyproject.toml‎

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ numpy = [
8383
{ version = "1.26.0", python = ">=3.9,<3.13", optional = true },
8484
{ version = "1.24.4", python = ">=3.8,<3.9", optional = true }
8585
]
86+
cachetools = "^5.5.0"
8687

8788
[tool.poetry.group.dev.dependencies]
8889
pytest = "7.4.4"
@@ -571,6 +572,318 @@ ignore_missing_imports = true
571572
module = "tenacity.*"
572573
ignore_missing_imports = true
573574

575+
[[tool.mypy.overrides]]
576+
module = "pyarrow.*"
577+
ignore_missing_imports = true
578+
579+
[[tool.mypy.overrides]]
580+
module = "pandas.*"
581+
ignore_missing_imports = true
582+
583+
[[tool.mypy.overrides]]
584+
module = "snappy.*"
585+
ignore_missing_imports = true
586+
587+
[[tool.mypy.overrides]]
588+
module = "zstandard.*"
589+
ignore_missing_imports = true
590+
591+
[[tool.mypy.overrides]]
592+
module = "pydantic.*"
593+
ignore_missing_imports = true
594+
595+
[[tool.mypy.overrides]]
596+
module = "pydantic_core.*"
597+
ignore_missing_imports = true
598+
599+
[[tool.mypy.overrides]]
600+
module = "pytest.*"
601+
ignore_missing_imports = true
602+
603+
[[tool.mypy.overrides]]
604+
module = "fastavro.*"
605+
ignore_missing_imports = true
606+
607+
[[tool.mypy.overrides]]
608+
module = "mmh3.*"
609+
ignore_missing_imports = true
610+
611+
[[tool.mypy.overrides]]
612+
module = "hive_metastore.*"
613+
ignore_missing_imports = true
614+
615+
[[tool.mypy.overrides]]
616+
module = "thrift.*"
617+
ignore_missing_imports = true
618+
619+
[[tool.mypy.overrides]]
620+
module = "requests_mock.*"
621+
ignore_missing_imports = true
622+
623+
[[tool.mypy.overrides]]
624+
module = "click.*"
625+
ignore_missing_imports = true
626+
627+
[[tool.mypy.overrides]]
628+
module = "rich.*"
629+
ignore_missing_imports = true
630+
631+
[[tool.mypy.overrides]]
632+
module = "fsspec.*"
633+
ignore_missing_imports = true
634+
635+
[[tool.mypy.overrides]]
636+
module = "s3fs.*"
637+
ignore_missing_imports = true
638+
639+
[[tool.mypy.overrides]]
640+
module = "azure.*"
641+
ignore_missing_imports = true
642+
643+
[[tool.mypy.overrides]]
644+
module = "adlfs.*"
645+
ignore_missing_imports = true
646+
647+
[[tool.mypy.overrides]]
648+
module = "gcsfs.*"
649+
ignore_missing_imports = true
650+
651+
[[tool.mypy.overrides]]
652+
module = "packaging.*"
653+
ignore_missing_imports = true
654+
655+
[[tool.mypy.overrides]]
656+
module = "tests.*"
657+
ignore_missing_imports = true
658+
659+
[[tool.mypy.overrides]]
660+
module = "boto3"
661+
ignore_missing_imports = true
662+
663+
[[tool.mypy.overrides]]
664+
module = "botocore.*"
665+
ignore_missing_imports = true
666+
667+
[[tool.mypy.overrides]]
668+
module = "mypy_boto3_glue.*"
669+
ignore_missing_imports = true
670+
671+
[[tool.mypy.overrides]]
672+
module = "moto"
673+
ignore_missing_imports = true
674+
675+
[[tool.mypy.overrides]]
676+
module = "aiobotocore.*"
677+
ignore_missing_imports = true
678+
679+
[[tool.mypy.overrides]]
680+
module = "aiohttp.*"
681+
ignore_missing_imports = true
682+
683+
[[tool.mypy.overrides]]
684+
module = "duckdb.*"
685+
ignore_missing_imports = true
686+
687+
[[tool.mypy.overrides]]
688+
module = "ray.*"
689+
ignore_missing_imports = true
690+
691+
[[tool.mypy.overrides]]
692+
module = "daft.*"
693+
ignore_missing_imports = true
694+
695+
[[tool.mypy.overrides]]
696+
module = "pyparsing.*"
697+
ignore_missing_imports = true
698+
699+
[[tool.mypy.overrides]]
700+
module = "pyspark.*"
701+
ignore_missing_imports = true
702+
703+
[[tool.mypy.overrides]]
704+
module = "strictyaml.*"
705+
ignore_missing_imports = true
706+
707+
[[tool.mypy.overrides]]
708+
module = "sortedcontainers.*"
709+
ignore_missing_imports = true
710+
711+
[[tool.mypy.overrides]]
712+
module = "numpy.*"
713+
ignore_missing_imports = true
714+
715+
[[tool.mypy.overrides]]
716+
module = "sqlalchemy.*"
717+
ignore_missing_imports = true
718+
719+
[[tool.mypy.overrides]]
720+
module = "Cython.*"
721+
ignore_missing_imports = true
722+
723+
[[tool.mypy.overrides]]
724+
module = "setuptools.*"
725+
ignore_missing_imports = true
726+
727+
[[tool.mypy.overrides]]
728+
module = "tenacity.*"
729+
ignore_missing_imports = true
730+
731+
[[tool.mypy.overrides]]
732+
module = "pyarrow.*"
733+
ignore_missing_imports = true
734+
735+
[[tool.mypy.overrides]]
736+
module = "pandas.*"
737+
ignore_missing_imports = true
738+
739+
[[tool.mypy.overrides]]
740+
module = "snappy.*"
741+
ignore_missing_imports = true
742+
743+
[[tool.mypy.overrides]]
744+
module = "zstandard.*"
745+
ignore_missing_imports = true
746+
747+
[[tool.mypy.overrides]]
748+
module = "pydantic.*"
749+
ignore_missing_imports = true
750+
751+
[[tool.mypy.overrides]]
752+
module = "pydantic_core.*"
753+
ignore_missing_imports = true
754+
755+
[[tool.mypy.overrides]]
756+
module = "pytest.*"
757+
ignore_missing_imports = true
758+
759+
[[tool.mypy.overrides]]
760+
module = "fastavro.*"
761+
ignore_missing_imports = true
762+
763+
[[tool.mypy.overrides]]
764+
module = "mmh3.*"
765+
ignore_missing_imports = true
766+
767+
[[tool.mypy.overrides]]
768+
module = "hive_metastore.*"
769+
ignore_missing_imports = true
770+
771+
[[tool.mypy.overrides]]
772+
module = "thrift.*"
773+
ignore_missing_imports = true
774+
775+
[[tool.mypy.overrides]]
776+
module = "requests_mock.*"
777+
ignore_missing_imports = true
778+
779+
[[tool.mypy.overrides]]
780+
module = "click.*"
781+
ignore_missing_imports = true
782+
783+
[[tool.mypy.overrides]]
784+
module = "rich.*"
785+
ignore_missing_imports = true
786+
787+
[[tool.mypy.overrides]]
788+
module = "fsspec.*"
789+
ignore_missing_imports = true
790+
791+
[[tool.mypy.overrides]]
792+
module = "s3fs.*"
793+
ignore_missing_imports = true
794+
795+
[[tool.mypy.overrides]]
796+
module = "azure.*"
797+
ignore_missing_imports = true
798+
799+
[[tool.mypy.overrides]]
800+
module = "adlfs.*"
801+
ignore_missing_imports = true
802+
803+
[[tool.mypy.overrides]]
804+
module = "gcsfs.*"
805+
ignore_missing_imports = true
806+
807+
[[tool.mypy.overrides]]
808+
module = "packaging.*"
809+
ignore_missing_imports = true
810+
811+
[[tool.mypy.overrides]]
812+
module = "tests.*"
813+
ignore_missing_imports = true
814+
815+
[[tool.mypy.overrides]]
816+
module = "boto3"
817+
ignore_missing_imports = true
818+
819+
[[tool.mypy.overrides]]
820+
module = "botocore.*"
821+
ignore_missing_imports = true
822+
823+
[[tool.mypy.overrides]]
824+
module = "mypy_boto3_glue.*"
825+
ignore_missing_imports = true
826+
827+
[[tool.mypy.overrides]]
828+
module = "moto"
829+
ignore_missing_imports = true
830+
831+
[[tool.mypy.overrides]]
832+
module = "aiobotocore.*"
833+
ignore_missing_imports = true
834+
835+
[[tool.mypy.overrides]]
836+
module = "aiohttp.*"
837+
ignore_missing_imports = true
838+
839+
[[tool.mypy.overrides]]
840+
module = "duckdb.*"
841+
ignore_missing_imports = true
842+
843+
[[tool.mypy.overrides]]
844+
module = "ray.*"
845+
ignore_missing_imports = true
846+
847+
[[tool.mypy.overrides]]
848+
module = "daft.*"
849+
ignore_missing_imports = true
850+
851+
[[tool.mypy.overrides]]
852+
module = "pyparsing.*"
853+
ignore_missing_imports = true
854+
855+
[[tool.mypy.overrides]]
856+
module = "pyspark.*"
857+
ignore_missing_imports = true
858+
859+
[[tool.mypy.overrides]]
860+
module = "strictyaml.*"
861+
ignore_missing_imports = true
862+
863+
[[tool.mypy.overrides]]
864+
module = "sortedcontainers.*"
865+
ignore_missing_imports = true
866+
867+
[[tool.mypy.overrides]]
868+
module = "numpy.*"
869+
ignore_missing_imports = true
870+
871+
[[tool.mypy.overrides]]
872+
module = "sqlalchemy.*"
873+
ignore_missing_imports = true
874+
875+
[[tool.mypy.overrides]]
876+
module = "Cython.*"
877+
ignore_missing_imports = true
878+
879+
[[tool.mypy.overrides]]
880+
module = "setuptools.*"
881+
ignore_missing_imports = true
882+
883+
[[tool.mypy.overrides]]
884+
module = "tenacity.*"
885+
ignore_missing_imports = true
886+
574887
[tool.poetry.scripts]
575888
pyiceberg = "pyiceberg.cli.console:run"
576889

0 commit comments

Comments
 (0)