Merge pull request #132 from patrickfleith/bug/125-invalid-json-qa-entries

Bug/125 invalid json qa entries

Author: patrickfleith

README.md

@@ -46,7 +46,7 @@ pip install datafast
 
 ### 1. Environment Setup
 
-Make sure you have created a `secrets.env` file with your API keys.
+Make sure you have created a `.env` file with your API keys.
 HF token is needed if you want to push the dataset to your HF hub.
 Other keys depends on which LLM providers you use.
 ```
@@ -64,7 +64,7 @@ from datafast.llms import OpenAIProvider, AnthropicProvider, GeminiProvider
 from dotenv import load_dotenv
 
 # Load environment variables
-load_dotenv("secrets.env") # <--- your API keys
+load_dotenv() # <--- your API keys
 ```
 
 ### 3. Configure Dataset

datafast/__init__.py

@@ -1,6 +1,7 @@
 """Datafast - A Python package for synthetic text dataset generation"""
 
 import importlib.metadata
+from datafast.logger_config import configure_logger
 
 try:
     __version__ = importlib.metadata.version("datafast")
@@ -11,3 +12,5 @@
 def get_version():
     """Return the current version of the datafast package."""
     return __version__
+
+__all__ = ["configure_logger", "get_version"]

datafast/datasets.py

@@ -1,10 +1,14 @@
 from datafast.datasets import ClassificationDataset
 from datafast.schema.config import ClassificationDatasetConfig, PromptExpansionConfig
 from datafast.llms import OpenAIProvider, AnthropicProvider
+from datafast.logger_config import configure_logger
 from dotenv import load_dotenv
 
 # Load API keys
-load_dotenv("secrets.env")
+load_dotenv()
+
+# Configure logger
+configure_logger()
 
 # Configure dataset
 config = ClassificationDatasetConfig(

datafast/examples/classification_trail_conditions_example.py

@@ -7,6 +7,8 @@
 from datafast.schema.config import GenericPipelineDatasetConfig
 from datafast.datasets import GenericPipelineDataset
 from datafast.llms import OpenAIProvider, GeminiProvider, OllamaProvider
+from datafast.logger_config import configure_logger
+from dotenv import load_dotenv
 
 
 PROMPT_TEMPLATE = """I will give you a persona.
@@ -80,7 +82,6 @@ def main():
 
 
 if __name__ == "__main__":
-    from dotenv import load_dotenv
-
-    load_dotenv("secrets.env")
+    load_dotenv()
+    configure_logger()
     main()

datafast/examples/generic_pipeline_example.py

@@ -2,6 +2,10 @@
 
 from datafast.schema.config import GenericPipelineDatasetConfig
 from datafast.utils import create_response_model
+from datafast.logger_config import configure_logger
+
+# Configure logger
+configure_logger()
 
 # Test with multiple columns and num_samples_per_prompt = 3
 config = GenericPipelineDatasetConfig(

datafast/examples/generic_pipeline_response_format_example.py

@@ -2,6 +2,10 @@
 
 from datafast.schema.config import GenericPipelineDatasetConfig
 from datafast.utils import create_generic_pipeline_row_model
+from datafast.logger_config import configure_logger
+
+# Configure logger
+configure_logger()
 
 # Test with multiple input, forward, and output columns
 config = GenericPipelineDatasetConfig(

datafast/examples/generic_pipeline_row_model_example.py

@@ -5,15 +5,19 @@
     python -m datafast.examples.inspect_dataset_example
 
 Requires:
-    - OpenAI API key in secrets.env or environment
+    - OpenAI API key in .env or environment
     - gradio package (pip install gradio)
 """
 from datafast.datasets import ClassificationDataset
 from datafast.schema.config import ClassificationDatasetConfig, PromptExpansionConfig
+from datafast.logger_config import configure_logger
 from dotenv import load_dotenv
 
-# Load API keys from environment or secrets.env
-load_dotenv("secrets.env")
+# Load API keys from environment or .env
+load_dotenv()
+
+# Configure logger
+configure_logger()
 
 # Configure the dataset generation
 config = ClassificationDatasetConfig(

datafast/examples/inspect_dataset_example.py

@@ -7,6 +7,12 @@
 from datafast.schema.config import GenericPipelineDatasetConfig
 from datafast.datasets import GenericPipelineDataset
 from datafast.llms import OpenAIProvider, GeminiProvider, OllamaProvider, OpenRouterProvider
+from datafast.logger_config import configure_logger
+from dotenv import load_dotenv
+
+# Load environment variables and configure logger
+load_dotenv()
+configure_logger()
 
 PROMPT_TEMPLATE = """I will give you a tweet.
 Generate a comma separated list of 3 keywords for the tweet. Avoid multi-word keywords.
@@ -63,7 +69,4 @@ def main():
 
 
 if __name__ == "__main__":
-    from dotenv import load_dotenv
-
-    load_dotenv("secrets.env")
     main()

datafast/examples/keywords_extraction_example.py

@@ -10,11 +10,15 @@
 import pandas as pd
 from dotenv import load_dotenv
 
-load_dotenv("secrets.env")
+load_dotenv()
 
 from datafast.schema.config import MCQDatasetConfig
 from datafast.datasets import MCQDataset
 from datafast.llms import OpenAIProvider
+from datafast.logger_config import configure_logger
+
+# Configure logger
+configure_logger()
 
 def main():
     # 1. Create a temporary filtered version of the dataset