[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Author: pre-commit-ci[bot]

src/vdf_io/notebooks/01_Introducing_txtai.ipynb

@@ -33,7 +33,7 @@
    },
    "outputs": [],
    "source": [
-    "from rich import print as rprint\n"
+    "from rich import print as rprint"
    ]
   },
   {
@@ -181,7 +181,10 @@
     "auth_provider = PlainTextAuthProvider(\n",
     "    os.environ.get(\"CASSANDRA_USER\"), os.environ.get(\"CASSANDRA_PASSWORD\")\n",
     ")\n",
-    "ssl_options = {\"ca_certs\": \"/Users/dhruvanand/Code/vector-io/aiven.pem\", \"cert_reqs\": ssl.CERT_REQUIRED}\n",
+    "ssl_options = {\n",
+    "    \"ca_certs\": \"/Users/dhruvanand/Code/vector-io/aiven.pem\",\n",
+    "    \"cert_reqs\": ssl.CERT_REQUIRED,\n",
+    "}\n",
     "CASSANDRA_URI = os.environ.get(\"CASSANDRA_URI\")\n",
     "CASSANDRA_URI, CASSANDRA_PORT = CASSANDRA_URI.split(\":\")\n",
     "with Cluster(\n",

src/vdf_io/notebooks/aiven-qs.ipynb

@@ -162,7 +162,7 @@
    },
    "outputs": [],
    "source": [
-    "table = coll.find()['data']['documents']"
+    "table = coll.find()[\"data\"][\"documents\"]"
    ]
   },
   {
@@ -325,6 +325,7 @@
    "source": [
     "# convert list of dicts to pd.DataFrame\n",
     "import pandas as pd\n",
+    "\n",
     "df = pd.DataFrame(table)\n",
     "df.head()"
    ]
@@ -359,7 +360,7 @@
     }
    ],
    "source": [
-    "len(resp['data']['documents'])"
+    "len(resp[\"data\"][\"documents\"])"
    ]
   },
   {
@@ -379,7 +380,7 @@
     }
    ],
    "source": [
-    "resp['data']['documents'][0].keys()"
+    "resp[\"data\"][\"documents\"][0].keys()"
    ]
   },
   {
@@ -497,14 +498,12 @@
    "source": [
     "# write to a new parquet file\n",
     "import pyarrow as pa\n",
-    "import pyarrow.parquet as pq\n",
     "\n",
     "table = pa.Table.from_pandas(coll.find().to_pandas())\n",
     "\n",
     "for r in coll.paginated_find():\n",
     "    print(type(r), r.keys())\n",
-    "    # append data into a parquet file\n",
-    "    "
+    "    # append data into a parquet file"
    ]
   },
   {
@@ -665,9 +664,9 @@
     }
    ],
    "source": [
-    "i=0\n",
+    "i = 0\n",
     "for r in tqdm(collection2.paginated_find()):\n",
-    "    i+=1\n",
+    "    i += 1\n",
     "print(i)"
    ]
   },
@@ -690,7 +689,7 @@
     "        break\n",
     "    next_page_state = a[\"data\"][\"nextPageState\"]\n",
     "    i += 1\n",
-    "    print(i,len(id_set), tot_docs)\n",
+    "    print(i, len(id_set), tot_docs)\n",
     "# len(a[\"data\"][\"documents\"])\n",
     "# len(id_set)"
    ]
@@ -728,7 +727,7 @@
     }
    ],
    "source": [
-    "a['data']['documents'][0]"
+    "a[\"data\"][\"documents\"][0]"
    ]
   },
   {
@@ -748,7 +747,7 @@
     }
    ],
    "source": [
-    "len(collection2.find_one()['data']['document'][\"vector\"])"
+    "len(collection2.find_one()[\"data\"][\"document\"][\"vector\"])"
    ]
   },
   {
@@ -768,8 +767,8 @@
     }
    ],
    "source": [
-    "mr=db.collection(\"movie_reviews\")\n",
-    "mr.find_one()['data']['document'].keys()"
+    "mr = db.collection(\"movie_reviews\")\n",
+    "mr.find_one()[\"data\"][\"document\"].keys()"
    ]
   },
   {

src/vdf_io/notebooks/astra_usage.ipynb

@@ -191,7 +191,7 @@
    },
    "outputs": [],
    "source": [
-    "import chromadb\n"
+    "import chromadb"
    ]
   },
   {
@@ -200,9 +200,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\n",
     "# setup Chroma in-memory, for easy prototyping. Can add persistence easily!\n",
-    "client = chromadb.PersistentClient()\n"
+    "client = chromadb.PersistentClient()"
    ]
   },
   {
@@ -247,21 +246,23 @@
     }
    ],
    "source": [
-    "\n",
     "# Create collection. get_collection, get_or_create_collection, delete_collection also available!\n",
     "collection2 = client.get_or_create_collection(\"test\")\n",
     "\n",
     "# Add docs to the collection. Can also update and delete. Row-based API coming soon!\n",
     "collection2.add(\n",
-    "    documents=[\"This is document1\", \"This is document2\"], # we handle tokenization, embedding, and indexing automatically. You can skip that and add your own embeddings as well\n",
-    "    metadatas=[{\"source\": \"notion\"}, {\"source\": \"google-docs\"}], # filter on these!\n",
-    "    ids=[\"doc1\", \"doc2\"], # unique for each doc\n",
-    "    embeddings=[[1,2,3], [4,5,6]] # optional, we can also embed for you\n",
+    "    documents=[\n",
+    "        \"This is document1\",\n",
+    "        \"This is document2\",\n",
+    "    ],  # we handle tokenization, embedding, and indexing automatically. You can skip that and add your own embeddings as well\n",
+    "    metadatas=[{\"source\": \"notion\"}, {\"source\": \"google-docs\"}],  # filter on these!\n",
+    "    ids=[\"doc1\", \"doc2\"],  # unique for each doc\n",
+    "    embeddings=[[1, 2, 3], [4, 5, 6]],  # optional, we can also embed for you\n",
     ")\n",
     "\n",
     "# Query/search 2 most similar results. You can also .get by id\n",
     "results = collection2.query(\n",
-    "    query_embeddings=[[1,2,3]],\n",
+    "    query_embeddings=[[1, 2, 3]],\n",
     "    n_results=2,\n",
     "    # where={\"metadata_field\": \"is_equal_to_this\"}, # optional filter\n",
     "    # where_document={\"$contains\":\"search_string\"}  # optional filter\n",
@@ -304,7 +305,7 @@
    },
    "outputs": [],
    "source": [
-    "coll=client.get_collection(\"test\")"
+    "coll = client.get_collection(\"test\")"
    ]
   },
   {
@@ -335,7 +336,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "client.delete_collection(\"test\") # delete collection"
+    "client.delete_collection(\"test\")  # delete collection"
    ]
   },
   {
@@ -357,7 +358,7 @@
     }
    ],
    "source": [
-    "client.list_collections() # list collections"
+    "client.list_collections()  # list collections"
    ]
   },
   {
@@ -399,7 +400,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pclient = chromadb.PersistentClient(\"~/.chroma4\") # persistent client\n",
+    "pclient = chromadb.PersistentClient(\"~/.chroma4\")  # persistent client\n",
     "\n",
     "pcol = pclient.create_collection(\"test3\")"
    ]
@@ -414,8 +415,8 @@
     "    documents=[\"This is document1\", \"This is document2\"],\n",
     "    metadatas=[{\"source\": \"notion\"}, {\"source\": \"google-docs\"}],\n",
     "    ids=[\"doc1\", \"doc2\"],\n",
-    "    embeddings=[[1,2,3], [4,5,6]]\n",
-    ")\n"
+    "    embeddings=[[1, 2, 3], [4, 5, 6]],\n",
+    ")"
    ]
   },
   {
@@ -479,7 +480,9 @@
    },
    "outputs": [],
    "source": [
-    "client2= chromadb.PersistentClient(\"/Users/dhruvanand/Code/vector-io/src/vdf_io/notebooks/chroma\")\n"
+    "client2 = chromadb.PersistentClient(\n",
+    "    \"/Users/dhruvanand/Code/vector-io/src/vdf_io/notebooks/chroma\"\n",
+    ")"
    ]
   },
   {
@@ -559,7 +562,7 @@
    },
    "outputs": [],
    "source": [
-    "coll2=client2.get_collection(\"vdf_2024_9-11\")"
+    "coll2 = client2.get_collection(\"vdf_2024_9-11\")"
    ]
   },
   {
@@ -653,7 +656,7 @@
     "    documents=[\"This is document1\", \"This is document2\"],\n",
     "    metadatas=[{\"source\": \"notion\"}, {\"source\": \"google-docs\"}],\n",
     "    ids=[\"doc5\", \"doc6\"],\n",
-    "    embeddings=[[1,2,3], None]\n",
+    "    embeddings=[[1, 2, 3], None],\n",
     ")"
    ]
   },

src/vdf_io/notebooks/chroma-qs.ipynb

@@ -156,7 +156,7 @@
    "source": [
     "import deeplake\n",
     "\n",
-    "ds = deeplake.load('hub://activeloop/coco-train')"
+    "ds = deeplake.load(\"hub://activeloop/coco-train\")"
    ]
   },
   {

src/vdf_io/notebooks/deeplake.ipynb

@@ -17,34 +17,26 @@
    "source": [
     "import json\n",
     "import pandas as pd\n",
-    "js=[\n",
+    "\n",
+    "js = [\n",
     "    {\n",
     "        \"id\": 1,\n",
     "        \"name\": \"John Doe\",\n",
     "        \"age\": 30,\n",
-    "        \"attribute\": {\n",
-    "            \"height\": 176,\n",
-    "            \"weight\": 80\n",
-    "        }\n",
+    "        \"attribute\": {\"height\": 176, \"weight\": 80},\n",
     "    },\n",
     "    {\n",
     "        \"id\": 2,\n",
     "        \"name\": \"Alice Smith\",\n",
     "        \"age\": 28,\n",
-    "        \"attribute\": {\n",
-    "            \"height\": 167,\n",
-    "            \"weight\": 55\n",
-    "        }\n",
+    "        \"attribute\": {\"height\": 167, \"weight\": 55},\n",
     "    },\n",
     "    {\n",
     "        \"id\": 3,\n",
     "        \"name\": \"Bob Johnson\",\n",
     "        \"age\": 35,\n",
-    "        \"attribute\": {\n",
-    "            \"height\": 192,\n",
-    "            \"weight\": 85\n",
-    "        }\n",
-    "    }\n",
+    "        \"attribute\": {\"height\": 192, \"weight\": 85},\n",
+    "    },\n",
     "]\n",
     "\n",
     "df = pd.read_json(json.dumps(js))"

src/vdf_io/notebooks/json_pandas.ipynb

@@ -15,9 +15,7 @@
    "outputs": [],
    "source": [
     "# Importing required libraries\n",
-    "import pandas as pd\n",
-    "import pyarrow as pa\n",
-    "import pyarrow.parquet as pq"
+    "import pandas as pd"
    ]
   },
   {
@@ -35,7 +33,7 @@
    "outputs": [],
    "source": [
     "# Load JSONL File\n",
-    "jsonl_file = '/Users/dhruvanand/Code/datasets-dumps/shard-00000.jsonl 2'\n"
+    "jsonl_file = \"/Users/dhruvanand/Code/datasets-dumps/shard-00000.jsonl 2\""
    ]
   },
   {
@@ -53,7 +51,9 @@
    "outputs": [],
    "source": [
     "# Convert JSONL to DataFrame\n",
-    "df = pd.read_json(jsonl_file, lines=True)  # Convert the loaded jsonl data into a pandas DataFrame"
+    "df = pd.read_json(\n",
+    "    jsonl_file, lines=True\n",
+    ")  # Convert the loaded jsonl data into a pandas DataFrame"
    ]
   },
   {
@@ -172,7 +172,7 @@
     }
    ],
    "source": [
-    "df['metadata'].iloc[3]"
+    "df[\"metadata\"].iloc[3]"
    ]
   },
   {
@@ -213,9 +213,9 @@
    "outputs": [],
    "source": [
     "# Save DataFrame as Parquet\n",
-    "for comp in ['snappy', 'gzip', 'brotli', 'zstd']:\n",
-    "    parquet_file = f'path_to_output_file-{comp}.parquet'  # replace with your desired output parquet file path\n",
-    "    df.to_parquet(parquet_file, engine='pyarrow', compression=comp)"
+    "for comp in [\"snappy\", \"gzip\", \"brotli\", \"zstd\"]:\n",
+    "    parquet_file = f\"path_to_output_file-{comp}.parquet\"  # replace with your desired output parquet file path\n",
+    "    df.to_parquet(parquet_file, engine=\"pyarrow\", compression=comp)"
    ]
   },
   {

src/vdf_io/notebooks/jsonl_to_parquet.ipynb

@@ -17,9 +17,7 @@
    "outputs": [],
    "source": [
     "# Importing required libraries\n",
-    "import pandas as pd\n",
-    "import pyarrow as pa\n",
-    "import pyarrow.parquet as pq"
+    "import pandas as pd"
    ]
   },
   {
@@ -71,7 +69,7 @@
     "with open(jsonl_file) as f:\n",
     "    for i, l in enumerate(f):\n",
     "        pass\n",
-    "    print(i)\n"
+    "    print(i)"
    ]
   },
   {
@@ -120,7 +118,7 @@
     }
    ],
    "source": [
-    "df.iloc[0]['url']"
+    "df.iloc[0][\"url\"]"
    ]
   },
   {