from llmware.configs import LLMWareConfig
from llmware.models import ModelCatalog
from llmware.library import Library

# Set engine to save library data
LLMWareConfig().set_active_db('sqlite')

lib = Library().create_new_library(
    account_name = 'dhika',
    library_name = 'test_lib'
)

urls = [
    'https://en.tempo.co/read/1915490/prabowo-indifferent-to-gibrans-alleged-fufufafa-account-says-dasco',
    # More URLs after that
]

print(len(urls))

for url in urls:
    lib.add_website(
        url = url,
        # Don't search site recursively
        get_links = False
    )