main.py

import logging
import time

from config import *
from data_scraping import web_scraper, xml_processing
from database import *

from datetime import datetime, timezone, timedelta

logging.basicConfig(level=logging.DEBUG)


def initialize():
    check_database(db_name)
    if download_past_data:
        logging.info(f"Scraping past data from {download_start_date} to {download_end_date}")
        web_scraper.download_period(download_start_date, download_end_date)
        logging.info("Download of past data completed")
        xml_processing.process_input_data()


def run_loop(max_iterations=None):
    # Current date in utc timezone
    date_utc = datetime.now(timezone.utc).date()

    counter = 0

    while True:
        date_new = datetime.now(timezone.utc).date()

        if date_new > date_utc:
            logging.info("New day, clearing temp buffer")

            print(len(xml_processing.inserted_files))
            xml_processing.inserted_files.clear()
            print(len(xml_processing.inserted_files))
            date_utc = date_new

        web_scraper.download_date(date_utc)
        xml_processing.process_input_data()

        counter += 1
        if max_iterations and counter >= max_iterations:
            break

        time.sleep(600)


if __name__ == "__main__":
    initialize()

    run_loop()

    # date_utc = datetime.now(timezone.utc).date() - timedelta(days=1)

    # counter = 0

    # counter += 1
    #
    # if counter > 5:
    #     # get the new date
    #     date_new = datetime.now(timezone.utc).date()
    # else:
    #     date_new = date_utc

    # if the date switched, clear the list of inserted files and update the date