Skip to content

Commit f6f1e5f

Browse files
ndrluisndrluis
committed
Deprecate ADLFS prefix in favor of ADLS
1 parent 2acba74 commit f6f1e5f

4 files changed

Lines changed: 88 additions & 34 deletions

File tree

‎mkdocs/docs/configuration.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -105,15 +105,15 @@ For the FileIO there are several configuration options available:
105105

106106
<!-- markdown-link-check-disable -->
107107

108-
| Key | Example | Description |
109-
| ----------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
110-
| adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). |
111-
| adlfs.account-name | devstoreaccount1 | The account that you want to connect to |
112-
| adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. |
113-
| adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature |
114-
| adlfs.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id |
115-
| adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id |
116-
| adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret |
108+
| Key | Example | Description |
109+
| ---------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
110+
| adls.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). |
111+
| adls.account-name | devstoreaccount1 | The account that you want to connect to |
112+
| adls.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. |
113+
| adls.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature |
114+
| adls.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id |
115+
| adls.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id |
116+
| adls.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret |
117117

118118
<!-- markdown-link-check-enable-->
119119

‎pyiceberg/io/__init__.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@
4646

4747
logger = logging.getLogger(__name__)
4848

49+
DEPRECATED_ADLFS_CONNECTION_STRING = "adlfs.connection-string"
50+
DEPRECATED_ADLFS_ACCOUNT_NAME = "adlfs.account-name"
51+
DEPRECATED_ADLFS_ACCOUNT_KEY = "adlfs.account-key"
52+
DEPRECATED_ADLFS_SAS_TOKEN = "adlfs.sas-token"
53+
DEPRECATED_ADLFS_TENANT_ID = "adlfs.tenant-id"
54+
DEPRECATED_ADLFS_CLIENT_ID = "adlfs.client-id"
55+
DEPRECATED_ADLFS_ClIENT_SECRET = "adlfs.client-secret"
56+
DEPRECATED_ADLFS_PREFIX = "adlfs"
4957
AWS_REGION = "client.region"
5058
AWS_ACCESS_KEY_ID = "client.access-key-id"
5159
AWS_SECRET_ACCESS_KEY = "client.secret-access-key"
@@ -62,13 +70,13 @@
6270
HDFS_PORT = "hdfs.port"
6371
HDFS_USER = "hdfs.user"
6472
HDFS_KERB_TICKET = "hdfs.kerberos_ticket"
65-
ADLFS_CONNECTION_STRING = "adlfs.connection-string"
66-
ADLFS_ACCOUNT_NAME = "adlfs.account-name"
67-
ADLFS_ACCOUNT_KEY = "adlfs.account-key"
68-
ADLFS_SAS_TOKEN = "adlfs.sas-token"
69-
ADLFS_TENANT_ID = "adlfs.tenant-id"
70-
ADLFS_CLIENT_ID = "adlfs.client-id"
71-
ADLFS_ClIENT_SECRET = "adlfs.client-secret"
73+
ADLFS_CONNECTION_STRING = "adls.connection-string"
74+
ADLFS_ACCOUNT_NAME = "adls.account-name"
75+
ADLFS_ACCOUNT_KEY = "adls.account-key"
76+
ADLFS_SAS_TOKEN = "adls.sas-token"
77+
ADLFS_TENANT_ID = "adls.tenant-id"
78+
ADLFS_CLIENT_ID = "adls.client-id"
79+
ADLFS_ClIENT_SECRET = "adls.client-secret"
7280
GCS_TOKEN = "gcs.oauth2.token"
7381
GCS_TOKEN_EXPIRES_AT_MS = "gcs.oauth2.token-expires-at"
7482
GCS_PROJECT_ID = "gcs.project-id"

‎pyiceberg/io/fsspec.py‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@
5050
AWS_REGION,
5151
AWS_SECRET_ACCESS_KEY,
5252
AWS_SESSION_TOKEN,
53+
DEPRECATED_ADLFS_ACCOUNT_KEY,
54+
DEPRECATED_ADLFS_ACCOUNT_NAME,
55+
DEPRECATED_ADLFS_CLIENT_ID,
56+
DEPRECATED_ADLFS_CONNECTION_STRING,
57+
DEPRECATED_ADLFS_PREFIX,
58+
DEPRECATED_ADLFS_SAS_TOKEN,
59+
DEPRECATED_ADLFS_TENANT_ID,
5360
GCS_ACCESS,
5461
GCS_CACHE_TIMEOUT,
5562
GCS_CONSISTENCY,
@@ -69,13 +76,16 @@
6976
S3_SESSION_TOKEN,
7077
S3_SIGNER_URI,
7178
ADLFS_ClIENT_SECRET,
79+
DEPRECATED_ADLFS_ClIENT_SECRET,
7280
FileIO,
7381
InputFile,
7482
InputStream,
7583
OutputFile,
7684
OutputStream,
7785
)
86+
from pyiceberg.table import PropertyUtil
7887
from pyiceberg.typedef import Properties
88+
from pyiceberg.utils.deprecated import deprecated
7989

8090
logger = logging.getLogger(__name__)
8191

@@ -176,14 +186,50 @@ def _gs(properties: Properties) -> AbstractFileSystem:
176186
def _adlfs(properties: Properties) -> AbstractFileSystem:
177187
from adlfs import AzureBlobFileSystem
178188

189+
for property_name in properties:
190+
if property_name.startswith(DEPRECATED_ADLFS_PREFIX):
191+
deprecated(
192+
deprecated_in="0.7.0",
193+
removed_in="0.8.0",
194+
help_message=f"The property {property_name} is deprecated. Please use properties that start with adls.",
195+
)(lambda: None)()
196+
179197
return AzureBlobFileSystem(
180-
connection_string=properties.get(ADLFS_CONNECTION_STRING),
181-
account_name=properties.get(ADLFS_ACCOUNT_NAME),
182-
account_key=properties.get(ADLFS_ACCOUNT_KEY),
183-
sas_token=properties.get(ADLFS_SAS_TOKEN),
184-
tenant_id=properties.get(ADLFS_TENANT_ID),
185-
client_id=properties.get(ADLFS_CLIENT_ID),
186-
client_secret=properties.get(ADLFS_ClIENT_SECRET),
198+
connection_string=PropertyUtil.get_first_property_value(
199+
properties,
200+
ADLFS_CONNECTION_STRING,
201+
DEPRECATED_ADLFS_CONNECTION_STRING,
202+
),
203+
account_name=PropertyUtil.get_first_property_value(
204+
properties,
205+
ADLFS_ACCOUNT_NAME,
206+
DEPRECATED_ADLFS_ACCOUNT_NAME,
207+
),
208+
account_key=PropertyUtil.get_first_property_value(
209+
properties,
210+
ADLFS_ACCOUNT_KEY,
211+
DEPRECATED_ADLFS_ACCOUNT_KEY,
212+
),
213+
sas_token=PropertyUtil.get_first_property_value(
214+
properties,
215+
ADLFS_SAS_TOKEN,
216+
DEPRECATED_ADLFS_SAS_TOKEN,
217+
),
218+
tenant_id=PropertyUtil.get_first_property_value(
219+
properties,
220+
ADLFS_TENANT_ID,
221+
DEPRECATED_ADLFS_TENANT_ID,
222+
),
223+
client_id=PropertyUtil.get_first_property_value(
224+
properties,
225+
ADLFS_CLIENT_ID,
226+
DEPRECATED_ADLFS_CLIENT_ID,
227+
),
228+
client_secret=PropertyUtil.get_first_property_value(
229+
properties,
230+
ADLFS_ClIENT_SECRET,
231+
DEPRECATED_ADLFS_ClIENT_SECRET,
232+
),
187233
)
188234

189235

‎tests/conftest.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -111,23 +111,23 @@ def pytest_addoption(parser: pytest.Parser) -> None:
111111
parser.addoption(
112112
"--s3.secret-access-key", action="store", default="password", help="The AWS secret access key ID for tests marked as s3"
113113
)
114-
# ADLFS options
114+
# ADLS options
115115
# Azurite provides default account name and key. Those can be customized using env variables.
116116
# For more information, see README file at https://github.com/azure/azurite#default-storage-account
117117
parser.addoption(
118-
"--adlfs.endpoint",
118+
"--adls.endpoint",
119119
action="store",
120120
default="http://127.0.0.1:10000",
121-
help="The ADLS endpoint URL for tests marked as adlfs",
121+
help="The ADLS endpoint URL for tests marked as adls",
122122
)
123123
parser.addoption(
124-
"--adlfs.account-name", action="store", default="devstoreaccount1", help="The ADLS account key for tests marked as adlfs"
124+
"--adls.account-name", action="store", default="devstoreaccount1", help="The ADLS account key for tests marked as adls"
125125
)
126126
parser.addoption(
127-
"--adlfs.account-key",
127+
"--adls.account-key",
128128
action="store",
129129
default="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
130-
help="The ADLS secret account key for tests marked as adlfs",
130+
help="The ADLS secret account key for tests marked as adls",
131131
)
132132
parser.addoption(
133133
"--gcs.endpoint", action="store", default="http://0.0.0.0:4443", help="The GCS endpoint URL for tests marked gcs"
@@ -1958,13 +1958,13 @@ def fixture_dynamodb(_aws_credentials: None) -> Generator[boto3.client, None, No
19581958
def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileIO, None, None]:
19591959
from azure.storage.blob import BlobServiceClient
19601960

1961-
azurite_url = request.config.getoption("--adlfs.endpoint")
1962-
azurite_account_name = request.config.getoption("--adlfs.account-name")
1963-
azurite_account_key = request.config.getoption("--adlfs.account-key")
1961+
azurite_url = request.config.getoption("--adls.endpoint")
1962+
azurite_account_name = request.config.getoption("--adls.account-name")
1963+
azurite_account_key = request.config.getoption("--adls.account-key")
19641964
azurite_connection_string = f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};"
19651965
properties = {
1966-
"adlfs.connection-string": azurite_connection_string,
1967-
"adlfs.account-name": azurite_account_name,
1966+
"adls.connection-string": azurite_connection_string,
1967+
"adls.account-name": azurite_account_name,
19681968
}
19691969

19701970
bbs = BlobServiceClient.from_connection_string(conn_str=azurite_connection_string)

0 commit comments

Comments
 (0)