Skip to content

Commit 752f925

Browse files
ndrluisndrluis
committed
Deprecate ADLFS prefix in favor of ADLS
1 parent 7d25bad commit 752f925

File tree

4 files changed

+87
-34
lines changed

4 files changed

+87
-34
lines changed

‎mkdocs/docs/configuration.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -107,15 +107,15 @@ For the FileIO there are several configuration options available:
107107

108108
<!-- markdown-link-check-disable -->
109109

110-
| Key | Example | Description |
111-
| ----------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
112-
| adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). |
113-
| adlfs.account-name | devstoreaccount1 | The account that you want to connect to |
114-
| adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. |
115-
| adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature |
116-
| adlfs.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id |
117-
| adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id |
118-
| adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret |
110+
| Key | Example | Description |
111+
| ---------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
112+
| adls.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). |
113+
| adls.account-name | devstoreaccount1 | The account that you want to connect to |
114+
| adls.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. |
115+
| adls.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature |
116+
| adls.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id |
117+
| adls.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id |
118+
| adls.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret |
119119

120120
<!-- markdown-link-check-enable-->
121121

‎pyiceberg/io/__init__.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@
4646

4747
logger = logging.getLogger(__name__)
4848

49+
ADLFS_CONNECTION_STRING = "adlfs.connection-string"
50+
ADLFS_ACCOUNT_NAME = "adlfs.account-name"
51+
ADLFS_ACCOUNT_KEY = "adlfs.account-key"
52+
ADLFS_SAS_TOKEN = "adlfs.sas-token"
53+
ADLFS_TENANT_ID = "adlfs.tenant-id"
54+
ADLFS_CLIENT_ID = "adlfs.client-id"
55+
ADLFS_ClIENT_SECRET = "adlfs.client-secret"
56+
ADLFS_PREFIX = "adlfs"
4957
AWS_REGION = "client.region"
5058
AWS_ACCESS_KEY_ID = "client.access-key-id"
5159
AWS_SECRET_ACCESS_KEY = "client.secret-access-key"
@@ -62,13 +70,13 @@
6270
HDFS_PORT = "hdfs.port"
6371
HDFS_USER = "hdfs.user"
6472
HDFS_KERB_TICKET = "hdfs.kerberos_ticket"
65-
ADLFS_CONNECTION_STRING = "adlfs.connection-string"
66-
ADLFS_ACCOUNT_NAME = "adlfs.account-name"
67-
ADLFS_ACCOUNT_KEY = "adlfs.account-key"
68-
ADLFS_SAS_TOKEN = "adlfs.sas-token"
69-
ADLFS_TENANT_ID = "adlfs.tenant-id"
70-
ADLFS_CLIENT_ID = "adlfs.client-id"
71-
ADLFS_ClIENT_SECRET = "adlfs.client-secret"
73+
ADLS_CONNECTION_STRING = "adls.connection-string"
74+
ADLS_ACCOUNT_NAME = "adls.account-name"
75+
ADLS_ACCOUNT_KEY = "adls.account-key"
76+
ADLS_SAS_TOKEN = "adls.sas-token"
77+
ADLS_TENANT_ID = "adls.tenant-id"
78+
ADLS_CLIENT_ID = "adls.client-id"
79+
ADLS_ClIENT_SECRET = "adls.client-secret"
7280
GCS_TOKEN = "gcs.oauth2.token"
7381
GCS_TOKEN_EXPIRES_AT_MS = "gcs.oauth2.token-expires-at"
7482
GCS_PROJECT_ID = "gcs.project-id"

‎pyiceberg/io/fsspec.py‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,15 @@
4444
ADLFS_ACCOUNT_NAME,
4545
ADLFS_CLIENT_ID,
4646
ADLFS_CONNECTION_STRING,
47+
ADLFS_PREFIX,
4748
ADLFS_SAS_TOKEN,
4849
ADLFS_TENANT_ID,
50+
ADLS_ACCOUNT_KEY,
51+
ADLS_ACCOUNT_NAME,
52+
ADLS_CLIENT_ID,
53+
ADLS_CONNECTION_STRING,
54+
ADLS_SAS_TOKEN,
55+
ADLS_TENANT_ID,
4956
AWS_ACCESS_KEY_ID,
5057
AWS_REGION,
5158
AWS_SECRET_ACCESS_KEY,
@@ -69,13 +76,15 @@
6976
S3_SESSION_TOKEN,
7077
S3_SIGNER_URI,
7178
ADLFS_ClIENT_SECRET,
79+
ADLS_ClIENT_SECRET,
7280
FileIO,
7381
InputFile,
7482
InputStream,
7583
OutputFile,
7684
OutputStream,
7785
)
7886
from pyiceberg.typedef import Properties
87+
from pyiceberg.utils.deprecated import deprecation_message
7988
from pyiceberg.utils.properties import get_first_property_value, property_as_bool
8089

8190
logger = logging.getLogger(__name__)
@@ -175,14 +184,50 @@ def _gs(properties: Properties) -> AbstractFileSystem:
175184
def _adlfs(properties: Properties) -> AbstractFileSystem:
176185
from adlfs import AzureBlobFileSystem
177186

187+
for property_name in properties:
188+
if property_name.startswith(ADLFS_PREFIX):
189+
deprecation_message(
190+
deprecated_in="0.8.0",
191+
removed_in="0.9.0",
192+
help_message=f"The property {property_name} is deprecated. Please use properties that start with adls.",
193+
)
194+
178195
return AzureBlobFileSystem(
179-
connection_string=properties.get(ADLFS_CONNECTION_STRING),
180-
account_name=properties.get(ADLFS_ACCOUNT_NAME),
181-
account_key=properties.get(ADLFS_ACCOUNT_KEY),
182-
sas_token=properties.get(ADLFS_SAS_TOKEN),
183-
tenant_id=properties.get(ADLFS_TENANT_ID),
184-
client_id=properties.get(ADLFS_CLIENT_ID),
185-
client_secret=properties.get(ADLFS_ClIENT_SECRET),
196+
connection_string=get_first_property_value(
197+
properties,
198+
ADLS_CONNECTION_STRING,
199+
ADLFS_CONNECTION_STRING,
200+
),
201+
account_name=get_first_property_value(
202+
properties,
203+
ADLS_ACCOUNT_NAME,
204+
ADLFS_ACCOUNT_NAME,
205+
),
206+
account_key=get_first_property_value(
207+
properties,
208+
ADLS_ACCOUNT_KEY,
209+
ADLFS_ACCOUNT_KEY,
210+
),
211+
sas_token=get_first_property_value(
212+
properties,
213+
ADLS_SAS_TOKEN,
214+
ADLFS_SAS_TOKEN,
215+
),
216+
tenant_id=get_first_property_value(
217+
properties,
218+
ADLS_TENANT_ID,
219+
ADLFS_TENANT_ID,
220+
),
221+
client_id=get_first_property_value(
222+
properties,
223+
ADLS_CLIENT_ID,
224+
ADLFS_CLIENT_ID,
225+
),
226+
client_secret=get_first_property_value(
227+
properties,
228+
ADLS_ClIENT_SECRET,
229+
ADLFS_ClIENT_SECRET,
230+
),
186231
)
187232

188233

‎tests/conftest.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -111,23 +111,23 @@ def pytest_addoption(parser: pytest.Parser) -> None:
111111
parser.addoption(
112112
"--s3.secret-access-key", action="store", default="password", help="The AWS secret access key ID for tests marked as s3"
113113
)
114-
# ADLFS options
114+
# ADLS options
115115
# Azurite provides default account name and key. Those can be customized using env variables.
116116
# For more information, see README file at https://github.com/azure/azurite#default-storage-account
117117
parser.addoption(
118-
"--adlfs.endpoint",
118+
"--adls.endpoint",
119119
action="store",
120120
default="http://127.0.0.1:10000",
121-
help="The ADLS endpoint URL for tests marked as adlfs",
121+
help="The ADLS endpoint URL for tests marked as adls",
122122
)
123123
parser.addoption(
124-
"--adlfs.account-name", action="store", default="devstoreaccount1", help="The ADLS account key for tests marked as adlfs"
124+
"--adls.account-name", action="store", default="devstoreaccount1", help="The ADLS account key for tests marked as adls"
125125
)
126126
parser.addoption(
127-
"--adlfs.account-key",
127+
"--adls.account-key",
128128
action="store",
129129
default="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
130-
help="The ADLS secret account key for tests marked as adlfs",
130+
help="The ADLS secret account key for tests marked as adls",
131131
)
132132
parser.addoption(
133133
"--gcs.endpoint", action="store", default="http://0.0.0.0:4443", help="The GCS endpoint URL for tests marked gcs"
@@ -1958,13 +1958,13 @@ def fixture_dynamodb(_aws_credentials: None) -> Generator[boto3.client, None, No
19581958
def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileIO, None, None]:
19591959
from azure.storage.blob import BlobServiceClient
19601960

1961-
azurite_url = request.config.getoption("--adlfs.endpoint")
1962-
azurite_account_name = request.config.getoption("--adlfs.account-name")
1963-
azurite_account_key = request.config.getoption("--adlfs.account-key")
1961+
azurite_url = request.config.getoption("--adls.endpoint")
1962+
azurite_account_name = request.config.getoption("--adls.account-name")
1963+
azurite_account_key = request.config.getoption("--adls.account-key")
19641964
azurite_connection_string = f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};"
19651965
properties = {
1966-
"adlfs.connection-string": azurite_connection_string,
1967-
"adlfs.account-name": azurite_account_name,
1966+
"adls.connection-string": azurite_connection_string,
1967+
"adls.account-name": azurite_account_name,
19681968
}
19691969

19701970
bbs = BlobServiceClient.from_connection_string(conn_str=azurite_connection_string)

0 commit comments

Comments
 (0)