Merge pull request #2075 from hackforla/ssiegal.fallback-to-prior-year

Fix unit tests and parameterize dataset year

Author: rayneng

backend/DbProvider.jsx

@@ -11,25 +11,6 @@ import DbContext from '@db/DbContext';
 import moment from 'moment';
 
 const hf_account = import.meta.env.VITE_ENV === 'DEV' ? '311-Data-Dev' : '311-data';
-
-// List of remote dataset locations used by db.registerFileURL
-const datasets = {
-  parquet: {
-    // huggingface
-    hfYtd2025:
-      `https://huggingface.co/datasets/${hf_account}/2025/resolve/main/2025.parquet`, // 2025 year-to-date
-    hfYtd2024:
-      `https://huggingface.co/datasets/${hf_account}/2024/resolve/main/2024.parquet`, // 2024 entire year
-    hfYtd2023:
-      `https://huggingface.co/datasets/${hf_account}/2023/resolve/main/2023.parquet`, // 2023 entire year
-    hfYtd2022:
-      `https://huggingface.co/datasets/${hf_account}/2022/resolve/main/2022.parquet`, // 2022 entire year
-    hfYtd2021:
-      `https://huggingface.co/datasets/${hf_account}/2021/resolve/main/2021.parquet`, // 2021 entire year
-    hfYtd2020:
-      `https://huggingface.co/datasets/${hf_account}/2020/resolve/main/2020.parquet`, // 2020 entire year
-  },
-};
 function DbProvider({ children, startDate }) {
   const [db, setDb] = useState(null);
   const [conn, setConn] = useState(null);
@@ -65,43 +46,20 @@ function DbProvider({ children, startDate }) {
           DUCKDB_CONFIG.pthreadWorker,
         );
 
-        // register parquet
-        await newDb.registerFileURL(
-          'requests2025.parquet',
-          //* Quick fix - change hfYtd2024 to hfYtd2025 when 2025 data available
-          datasets.parquet.hfYtd2025,
-          4, // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol
-        );
-
-        await newDb.registerFileURL(
-          'requests2024.parquet',
-          datasets.parquet.hfYtd2024,
-          4, // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol
-        );
-
-        await newDb.registerFileURL(
-          'requests2023.parquet',
-          datasets.parquet.hfYtd2023,
-          4,
-        );
-
-        await newDb.registerFileURL(
-          'requests2022.parquet',
-          datasets.parquet.hfYtd2022,
-          4,
-        );
-
-        await newDb.registerFileURL(
-          'requests2021.parquet',
-          datasets.parquet.hfYtd2021,
-          4,
-        );
-
-        await newDb.registerFileURL(
-          'requests2020.parquet',
-          datasets.parquet.hfYtd2020,
-          4,
-        );
+        // Register parquet files for years 2020 to current year
+        const currentYear = new Date().getFullYear();
+        for (let year = 2020; year <= currentYear; year++) {
+          const datasetUrl = `https://huggingface.co/datasets/${hf_account}/${year}/resolve/main/${year}.parquet`;
+          try {
+            await newDb.registerFileURL(
+              `requests${year}.parquet`,
+              datasetUrl,
+              4, // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol
+            );
+          } catch (err) {
+            console.warn(`Failed to register dataset for year ${year}:`, err);
+          }
+        }
 
         // Create db connection
         const newConn = await newDb.connect();

package.json

@@ -17,7 +17,7 @@
     "lint": "eslint \"./**/*.js*\"",
     "lint:fix": "eslint --fix \"./**/*.js*\"",
     "test": "vitest",
-    "check-env": "node ./utils/checkEnv"
+    "check-env": "node ./src/utils/checkEnv"
   },
   "husky": {
     "hooks": {

src/features/Map/index.jsx

@@ -70,10 +70,11 @@ class MapContainer extends React.Component {
 
   createRequestsTable = async () => {
     this.setState({ isTableLoading: true });
-    const { tableNameByYear, setDbStartTime } = this.context;
-    const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store
-    const year = moment(startDate).year(); // extract the year
-    const datasetFileName = `requests${year}.parquet`;
+    const { setDbStartTime } = this.context;
+    const { startDate } = this.props;
+    let year = moment(startDate).year(); // extract the year
+    const tableNameByYear = `requests_${year}`;
+    let datasetFileName = `requests${year}.parquet`;
 
     // Create the year data table if not exist already
     const createSQL = `CREATE TABLE IF NOT EXISTS ${tableNameByYear} AS SELECT * FROM '${datasetFileName}'`; // query from parquet
@@ -90,13 +91,11 @@ class MapContainer extends React.Component {
         )} ms.`
       );
     } catch (error) {
-      console.error("Error in creating table or registering dataset:", error);
-    } finally {
-      this.setState({ isTableLoading: false });
+      console.error(`Failed to load dataset for year ${year}:`, error);
     }
   };
 
-  async componentDidMount(props) {
+  async componentDidMount() {
     this.isSubscribed = true;
     this.processSearchParams();
     if (DATA_SOURCE !== "SOCRATA") await this.createRequestsTable();

src/scripts/updateHfDataset.py

@@ -5,8 +5,18 @@
 from tqdm import tqdm
 from huggingface_hub import HfApi, login
 from dotenv import load_dotenv
+from datetime import datetime
 load_dotenv()
 
+# Lookup table for data URLs by year
+DATA_URLS = {
+    '2025': 'https://data.lacity.org/api/views/h73f-gn57/rows.csv?accessType=DOWNLOAD',
+    '2026': 'https://data.lacity.org/api/v3/views/2cy6-i7zn/query.csv',
+}
+
+def get_current_year():
+    return str(datetime.now().year)
+
 # set environment as 'dev' or 'prod'
 ENV = os.getenv('VITE_ENV')
 
@@ -21,26 +31,37 @@
 
 def dlData():
     '''
-    Download the current year's dataset from data.lacity.org
+    Download the current year's dataset from data.lacity.org.
+    Returns the year, so it can be passed to subsequent steps.
     '''
-    url = "https://data.lacity.org/api/views/h73f-gn57/rows.csv?accessType=DOWNLOAD"
-    outfile = "2025.csv"
+    year = get_current_year()
+    if year not in DATA_URLS:
+        raise ValueError(f"No data URL configured for year {year}")
+    url = DATA_URLS[year]
+    outfile = f"{year}.csv"
 
     response = requests.get(url, stream=True)
 
+    # If we get a 4xx or 5xx HTTP status, raise an exception and stop processing altogether
+    response.raise_for_status()
+
     # Save downloaded file
     with open(outfile, "wb") as file:
         for data in tqdm(response.iter_content()):
             file.write(data)
 
+    return year
+
 
-def hfClean():
+def hfClean(year=None):
     '''
     Clean the dataset by removing problematic string combinations and update timestamp to ISO format
     '''
-    infile = "2025.csv"
-    fixed_filename = "2025-fixed.csv"
-    clean_filename = "2025-clean.parquet"
+    if year is None:
+        year = get_current_year()
+    infile = f"{year}.csv"
+    fixed_filename = f"{year}-fixed.csv"
+    clean_filename = f"{year}-clean.parquet"
 
     # List of problmenatic strings to be replaced with ""
     replace_strings = ["VE, 0"]
@@ -65,13 +86,15 @@ def hfClean():
         print(f"File {infile} not found.")
 
 
-def hfUpload():
+def hfUpload(year=None):
     '''
     Upload the clean dataset to huggingface.co
     '''
-    local_filename = '2025-clean.parquet'
-    dest_filename = '2025.parquet'
-    repo_name = '2025'
+    if year is None:
+        year = get_current_year()
+    local_filename = f'{year}-clean.parquet'
+    dest_filename = f'{year}.parquet'
+    repo_name = f'{year}'
     repo_type = 'dataset'
 
     repo_id = f"{HF_USERNAME}/{repo_name}"
@@ -95,9 +118,9 @@ def cleanUp():
 
 
 def main():
-    dlData()
-    hfClean()
-    hfUpload()
+    year = dlData()
+    hfClean(year)
+    hfUpload(year)
     cleanUp()
 
 

src/utils/checkEnv.js

@@ -13,8 +13,8 @@ const red = '\x1b[31m%s\x1b[0m';
 const green = '\x1b[32m%s\x1b[0m';
 const vitePrefix = 'VITE_';
 
-const envPath = path.resolve(__dirname, '../.env');
-const exampleEnvPath = path.resolve(__dirname, '../.example.env');
+const envPath = path.resolve(__dirname, '../../.env');
+const exampleEnvPath = path.resolve(__dirname, '../../.example.env');
 
 function getEnv(fileName) {
   return dotenv.parse(fs.readFileSync(fileName));

src/utils/test-utils.jsx

@@ -1,7 +1,9 @@
 import React from 'react'
 import { render } from '@testing-library/react'
 import { Provider } from 'react-redux'
+import { ThemeProvider } from '@mui/material/styles'
 import { setupStore } from '../redux/store'
+import theme from '@theme/theme'
 
 // https://redux.js.org/usage/writing-tests#setting-up-a-reusable-test-render-function
 export function renderWithProviders(ui, extendedRenderOptions = {}) {
@@ -13,7 +15,9 @@ export function renderWithProviders(ui, extendedRenderOptions = {}) {
   } = extendedRenderOptions
 
   const Wrapper = ({ children }) => (
-    <Provider store={store}>{children}</Provider>
+    <Provider store={store}>
+      <ThemeProvider theme={theme}>{children}</ThemeProvider>
+    </Provider>
   )
 
   // Return an object with the store and all of RTL's query functions

vite.config.js

@@ -31,7 +31,7 @@ export default defineConfig(() => {
     },
     test: {
       environment: 'jsdom',
-      setupFiles: 'utils/test-setup.js'
+      setupFiles: 'src/utils/test-setup.js'
     }
   };
 });