Fix/research module (#694)

Author: wangxingjun778

.gitignore

@@ -94,8 +94,9 @@ celerybeat-schedule
 *.sage.py
 
 # Environments
-.env
+projects/deep_research/.env
 .venv
+.env
 env/
 venv/
 ENV/

ms_agent/tools/search_engine.py

@@ -1,6 +1,7 @@
 import os
 from typing import Any, Dict
 
+from ms_agent.config.env import Env
 from ms_agent.tools.exa import ExaSearch
 from ms_agent.tools.search.arxiv import ArxivSearch
 from ms_agent.tools.search.search_base import SearchEngineType
@@ -68,6 +69,8 @@ def replace_env_vars(value: str) -> str:
     if not isinstance(value, str):
         return value
 
+    Env.load_env()
+
     if value.startswith('$'):
         env_var = value[1:]
         return os.getenv(env_var, None)

ms_agent/workflow/research_workflow.py

@@ -3,7 +3,6 @@
 import copy
 import os
 import re
-from datetime import datetime
 from typing import Any, Dict, List, Optional, Union
 
 import json
@@ -236,6 +235,8 @@ def filter_search_res(single_res: SearchResult):
         search_results = [
             filter_search_res(single_res) for single_res in search_results
         ]
+
+        # TODO: Implement a more robust way to handle multiple search results
         dump_batch_search_results(results=search_results, file_path=self.workdir_structure['search'])
 
         return self.workdir_structure['search']
@@ -244,25 +245,35 @@ def filter_search_res(single_res: SearchResult):
     def parse_json_from_content(text_content: str) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
         """
         Parses the given text content to extract JSON format data.
+        It can parse JSON embedded within triple backticks (```json...```)
+        or stand-alone JSON text.
 
         Args:
             text_content (str): The text content containing JSON format data.
 
         Returns:
-            list: A dict or list of dictionaries representing the target data.
+            list: A dict or list of dictionaries representing the parsed JSON data.
         """
+        # Try to find JSON embedded in ```json...```
         pattern = r'```json(.*?)```'
-
         matches = re.findall(pattern, text_content, re.DOTALL)
 
-        if len(matches) == 0:
+        json_string = ''
+        if matches:
+            # If matches are found, use the first one
+            json_string = matches[0].strip()
+        else:
+            # If no ```json...``` block is found, assume the entire content is JSON
+            json_string = text_content.strip()
+
+        if not json_string:
             return []
 
         try:
-            items = json.loads(matches[0].strip())
+            items = json.loads(json_string)
             return items
         except json.JSONDecodeError as e:
-            raise ValueError(f'parse_py_todo: JSON decoding error: {e}') from e
+            raise ValueError(f'Failed to parse JSON content. Error: {e}')
 
     def _dump_todo_file(self):
 

projects/deep_research/.env.example

@@ -0,0 +1,2 @@
+EXA_API_KEY=xxx
+SERPAPI_API_KEY=xxx

projects/deep_research/README.md

@@ -58,11 +58,15 @@ pip install -r requirements/research.txt
 
 #### Environment Setting
 
-1. If you’re using Exa or SerpApi, make sure your .env file includes the following configuration settings:
+1. If you're using Exa or SerpApi, make sure your .env file includes the following configuration settings:
 ```bash
-EXA_API_KEY="xxx"
-SERPAPI_API_KEY="xxx"
+cp .env.example ../../.env
+
+# Then, edit the `.env` file to include your API keys:
+EXA_API_KEY=xxx
+SERPAPI_API_KEY=xxx
 ```
+
 2. Configure the search engine in conf.yaml, using free arxiv search by default:
 ```yaml
 SEARCH_ENGINE:

projects/deep_research/conf.yaml

@@ -1,4 +1,5 @@
-#Search Engine, Supported values: EXA, SERPAPI, ARXIV
+## Search Engine, Supported values: EXA, SERPAPI, ARXIV ##
+
 #SEARCH_ENGINE:
 #  engine: exa
 #  exa_api_key: $EXA_API_KEY

projects/deep_research/run.py

@@ -24,7 +24,7 @@ def run_workflow(user_prompt: str, task_dir: str, reuse: bool,
 if __name__ == '__main__':
 
     query: str = 'Survey of the AI Agent within the recent 3 month, including the latest research papers, open-source projects, and industry applications.'  # noqa
-    task_workdir: str = '/path/to/your_task_dir'
+    task_workdir: str = '/path/to/your_workdir'  # Specify your task work directory here
     reuse: bool = False
 
     # Get chat client OpenAI compatible api