diff --git a/users/how-to/quota_pb_minio.ipynb b/users/how-to/quota_pb_minio.ipynb new file mode 100644 index 0000000..d70172b --- /dev/null +++ b/users/how-to/quota_pb_minio.ipynb @@ -0,0 +1,1221 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2ce331a6-818a-4b43-a139-942f2f9361a2", + "metadata": {}, + "source": [ + "# Can't create file of 100 MiB on minIO space??" + ] + }, + { + "cell_type": "markdown", + "id": "74798628-59e8-49e3-87fa-56c377588c8e", + "metadata": {}, + "source": [ + "## Clear my space" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "df739287-646b-4db2-97c1-8a0fa408ebc0", + "metadata": {}, + "outputs": [], + "source": [ + "import s3fs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c660357b-4cff-4f07-8f01-7bcd743bea6f", + "metadata": {}, + "outputs": [], + "source": [ + "access_key = !aws configure get aws_access_key_id\n", + "access_key = access_key[0]\n", + "secret_key = !aws configure get aws_secret_access_key\n", + "secret_key = secret_key[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3f938612-2a10-48f0-97ef-f6ce5d14f8a2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client_kwargs={'endpoint_url': 'https://pangeo-eosc-minioapi.vm.fedcloud.eu/'}\n", + "\n", + "#s3 = s3fs.S3FileSystem(anon=False, client_kwargs=client_kwargs) # Works only when using s3 in this Notebook, not with distributed.\n", + "s3 = s3fs.S3FileSystem(key=access_key, secret=secret_key, client_kwargs=client_kwargs)\n", + "s3.rm(\"s3://todaka-test/myfile\")\n", + "s3.rm(\"s3://todaka-test/myfile_pr_an_limit\")\n", + "\n", + "s3.ls(\"s3://todaka-test/\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3baa6fe0-b049-44b2-8537-80e1e7301e46", + "metadata": {}, + "source": [ + "## Create test dataset for writing\n", + "\n", + "Change the value \n", + "size_in_MiB \n", + "for tests" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "8957eb61-3479-488d-a8f3-0b9710008a9c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:   (x: 50, y: 128, z: 1024)\n",
+       "Coordinates:\n",
+       "  * x         (x) int64 0 1 2 3 4 5 6 7 8 9 10 ... 40 41 42 43 44 45 46 47 48 49\n",
+       "  * y         (y) int64 0 1 2 3 4 5 6 7 8 ... 120 121 122 123 124 125 126 127\n",
+       "  * z         (z) int64 0 1 2 3 4 5 6 7 ... 1017 1018 1019 1020 1021 1022 1023\n",
+       "Data variables:\n",
+       "    variable  (x, y, z) float64 dask.array<chunksize=(50, 128, 1024), meta=np.ndarray>
" + ], + "text/plain": [ + "\n", + "Dimensions: (x: 50, y: 128, z: 1024)\n", + "Coordinates:\n", + " * x (x) int64 0 1 2 3 4 5 6 7 8 9 10 ... 40 41 42 43 44 45 46 47 48 49\n", + " * y (y) int64 0 1 2 3 4 5 6 7 8 ... 120 121 122 123 124 125 126 127\n", + " * z (z) int64 0 1 2 3 4 5 6 7 ... 1017 1018 1019 1020 1021 1022 1023\n", + "Data variables:\n", + " variable (x, y, z) float64 dask.array" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import xarray as xr\n", + "import numpy as np\n", + "\n", + "# Calculate the number of elements needed for \"size_in_MiB\" MiB of 64-bit floating point data\n", + "size_in_MiB = 50\n", + "size_in_bytes = size_in_MiB * 1024 * 1024 # 30 MiB\n", + "bytes_per_element = 8 # 64-bit floating points\n", + "number_of_elements = 1024 // 8\n", + "\n", + "# Create the NumPy array\n", + "data = np.random.rand(number_of_elements* 1024 * size_in_MiB)\n", + "\n", + "# Choose dimensions that multiply together to equal the total number of elements\n", + "dim1, dim2, dim3 = size_in_MiB, number_of_elements, 1024 #// (120 * 120)\n", + "#if dim1 * dim2 * dim3 != number_of_elements:\n", + "# print \n", + "# raise ValueError(\"Chosen dimensions do not multiply to the total number of elements\")\n", + "\n", + "data = data.reshape((dim1, dim2, dim3))\n", + "\n", + "# Create an xarray dataset\n", + "ds = xr.Dataset(\n", + " {\n", + " \"variable\": ((\"x\", \"y\", \"z\"), data)\n", + " },\n", + " coords={\n", + " \"x\": np.arange(dim1),\n", + " \"y\": np.arange(dim2),\n", + " \"z\": np.arange(dim3)\n", + " }\n", + ")\n", + "\n", + "# Save the dataset to a NetCDF file\n", + "ds.chunk('auto')\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "203d767d-c46e-4238-a595-f4c9870c2c2b", + "metadata": {}, + "source": [ + "## Store the file in s3" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "dff49207-6526-4408-b1e5-0f9a84fea72d", + "metadata": {}, + "outputs": [], + "source": [ + "target_root=\"s3://todaka-test/\"\n", + "filepath=f\"{target_root}test.zarr\"\n", + "\n", + "storage_options = {\n", + " 'anon': False, \n", + " 'profile' : \"default\",\n", + " 'client_kwargs': {\n", + " \"endpoint_url\": 'https://pangeo-eosc-minioapi.vm.fedcloud.eu/',\n", + " }}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "52e94207-f0fc-464a-84df-ab6b5bbd9c9d", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.chunk('auto').to_zarr(\n", + "# 'test.zarr', mode='w',compute=True)\n", + " filepath, mode='w', \n", + " consolidated=True, \n", + " storage_options=storage_options)" + ] + }, + { + "cell_type": "markdown", + "id": "04baa4f0-3fb1-4e31-85d4-7e45abd9b2ce", + "metadata": {}, + "source": [ + "\n", + "## Results\n", + "\n", + "| size | works? |\n", + "|------|--------|\n", + "| 100 | KO |\n", + "| 70 | KO |\n", + "| 50 | OK |" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6aeefb3a-5e81-4a8d-9304-bdcf69e235db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:   (x: 50, y: 128, z: 1024)\n",
+       "Coordinates:\n",
+       "  * x         (x) int64 0 1 2 3 4 5 6 7 8 9 10 ... 40 41 42 43 44 45 46 47 48 49\n",
+       "  * y         (y) int64 0 1 2 3 4 5 6 7 8 ... 120 121 122 123 124 125 126 127\n",
+       "  * z         (z) int64 0 1 2 3 4 5 6 7 ... 1017 1018 1019 1020 1021 1022 1023\n",
+       "Data variables:\n",
+       "    variable  (x, y, z) float64 dask.array<chunksize=(50, 128, 1024), meta=np.ndarray>
" + ], + "text/plain": [ + "\n", + "Dimensions: (x: 50, y: 128, z: 1024)\n", + "Coordinates:\n", + " * x (x) int64 0 1 2 3 4 5 6 7 8 9 10 ... 40 41 42 43 44 45 46 47 48 49\n", + " * y (y) int64 0 1 2 3 4 5 6 7 8 ... 120 121 122 123 124 125 126 127\n", + " * z (z) int64 0 1 2 3 4 5 6 7 ... 1017 1018 1019 1020 1021 1022 1023\n", + "Data variables:\n", + " variable (x, y, z) float64 dask.array" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "storage_options = {\n", + " 'anon': False, \n", + " 'profile' : \"default\",\n", + " 'client_kwargs': {\n", + " \"endpoint_url\": 'https://pangeo-eosc-minioapi.vm.fedcloud.eu/',\n", + " }}\n", + "\n", + "xr.open_zarr(\n", + " filepath,\n", + " storage_options=storage_options)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d45bb00c-4b3e-4ee8-8792-b670b081ef08", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}