Set an upper limit for max_workers on electricity extract and load.

It was observed that on a machine with 16 cpus, max_workers=20
and some files would fail when extracting content. It is assumed
that reading too many files from sharepoint causes issues.

Author: martyngigg

warehouses/accelerator/extract_load/electricity_sharepoint/.dlt/config.toml

@@ -1,3 +1,4 @@
 [electricity_sharepoint.sources.m365]
 files_per_page = 200
 skip_rows = 7
+max_workers = 10

warehouses/accelerator/extract_load/electricity_sharepoint/extract_and_load.py

@@ -77,7 +77,7 @@ def read_power_consumption_excel(
 def extract_content_and_read(
     items: Iterator[M365DriveItem],
     skip_rows: int = dlt.config.value,
-    max_threads: Optional[int] = dlt.config.value,
+    max_workers: Optional[int] = dlt.config.value,
 ) -> Iterator[TDataItems]:
     """Extracts the file content and reads it assuming it is a .csv or a .xlsx file
 
@@ -86,7 +86,7 @@ def extract_content_and_read(
 
     :param items: An iterator of dicts describing the file content
     :param skip_rows: Number of rows in the csv/xlsx files to skip
-    :param max_threads (optional): How many threads to use to process the files. Defaults to concurrent.futures.ThreadPoolExecutor default value.
+    :param max_workers (optional): How many threads to use to process the files. Defaults to concurrent.futures.ThreadPoolExecutor default value.
     """
 
     # The files are all independent. Process them in parallel and combine for a single yield
@@ -111,7 +111,7 @@ def read_as_dataframe(file_obj: M365DriveItem) -> pd.DataFrame | None:
         return df
 
     df_batch = None
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         future_to_file_item = {
             executor.submit(read_as_dataframe, file_obj): file_obj for file_obj in items
         }