Skip to content

Commit bfcc39d

Browse files
authored
Merge pull request #3 from FlowLLM-AI/shuchang_dev
add local THS data reading operator with configuration
2 parents e166595 + 4faab2b commit bfcc39d

16 files changed

Lines changed: 1193 additions & 6 deletions

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,5 @@ build/*
2222
**/cache/**
2323
*.pdf
2424
*.csv
25-
site/*
25+
site/*
26+
tool_cache/*

extract_ths_tool_cache/compress.sh

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
# 定义目标文件夹和输出文件名前缀
4+
TARGET_DIR="tool_cache"
5+
OUTPUT_PREFIX="extract_ths_tool_cache/tool_cache.tar.gz.part_"
6+
SIZE_LIMIT="50m"
7+
8+
echo "正在压缩并切分 $TARGET_DIR ..."
9+
10+
# tar -c: 创建
11+
# -z: gzip压缩
12+
# -f -: 将结果输出到标准输出流
13+
# split -b: 按大小切分
14+
# -: 从标准输入流读取
15+
tar -czf - "$TARGET_DIR" --exclude='.DS_Store' | split -b $SIZE_LIMIT - "$OUTPUT_PREFIX"
16+
17+
echo "压缩完成!生成的切分文件如下:"
18+
ls -lh ${OUTPUT_PREFIX}*

extract_ths_tool_cache/extract.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
# 定义切分文件的前缀
4+
TARGET_DIR="tool_cache"
5+
INPUT_PREFIX="extract_ths_tool_cache/tool_cache.tar.gz.part_"
6+
7+
# 检查是否有分卷文件
8+
if ! ls ${INPUT_PREFIX}* 1> /dev/null 2>&1; then
9+
echo "错误:未发现分卷文件 ${INPUT_PREFIX}*"
10+
exit 1
11+
fi
12+
13+
echo "正在合并并解压文件..."
14+
15+
# cat: 合并所有分卷
16+
# tar -x: 解压
17+
cat ${INPUT_PREFIX}* | tar -xzf -
18+
19+
echo "解压完成!文件夹 '$TARGET_DIR' 已还原。"

extract_ths_tool_cache/merge.py

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
import os
2+
import json
3+
import shutil
4+
5+
# 配置路径(Python 字符串不需要反斜杠转义空格)
6+
SRC_ROOT_BASE = "/Users/tsc/研究工作/金融ASIO/代码/finance-mcp_3"
7+
DST_ROOT_BASE = "/Users/tsc/研究工作/金融ASIO/代码/finance-mcp"
8+
9+
# 需要合并的根文件夹
10+
TARGET_FOLDERS = ["tool_cache", "cache"]
11+
12+
def load_json_data(file_path):
13+
"""加载 JSON 数据,支持列表或单个对象"""
14+
try:
15+
with open(file_path, 'r', encoding='utf-8') as f:
16+
data = json.load(f)
17+
return data if isinstance(data, list) else [data]
18+
except Exception as e:
19+
print(f" [跳过] 无法解析 JSON: {file_path}. 错误: {e}")
20+
return None
21+
22+
def save_json_data(file_path, data):
23+
"""保存 JSON 数据"""
24+
with open(file_path, 'w', encoding='utf-8') as f:
25+
json.dump(data, f, ensure_ascii=False, indent=2)
26+
27+
def get_code(sample):
28+
"""根据你提供的结构提取 code"""
29+
try:
30+
# 路径: sample -> tool_args -> code
31+
return sample.get("tool_args", {}).get("code")
32+
except:
33+
return None
34+
35+
def merge_json_files(src_file, dst_file):
36+
"""核心逻辑:合并两个文件中的 samples,按 code 去重"""
37+
src_data = load_json_data(src_file)
38+
dst_data = load_json_data(dst_file)
39+
40+
if src_data is None or dst_data is None:
41+
return False # 读取出错,不执行合并
42+
43+
# 获取目标文件中已有的所有 code
44+
existing_codes = {get_code(s) for s in dst_data if get_code(s) is not None}
45+
46+
initial_count = len(dst_data)
47+
for sample in src_data:
48+
code = get_code(sample)
49+
if code and code not in existing_codes:
50+
dst_data.append(sample)
51+
existing_codes.add(code)
52+
53+
new_added = len(dst_data) - initial_count
54+
if new_added > 0:
55+
save_json_data(dst_file, dst_data)
56+
print(f" [合并完成] {os.path.basename(dst_file)}: 新增了 {new_added} 条 code 样本")
57+
else:
58+
print(f" [无需合并] {os.path.basename(dst_file)}: 未发现新 code")
59+
return True
60+
61+
def start_recursive_merge():
62+
for target in TARGET_FOLDERS:
63+
src_folder = os.path.join(SRC_ROOT_BASE, target)
64+
dst_folder = os.path.join(DST_ROOT_BASE, target)
65+
66+
if not os.path.exists(src_folder):
67+
print(f"源文件夹不存在,跳过: {src_folder}")
68+
continue
69+
70+
print(f"\n>>> 正在扫描目录: {target}")
71+
72+
# 使用 os.walk 递归遍历所有子文件夹
73+
for root, dirs, files in os.walk(src_folder):
74+
# 计算当前子目录相对于源根目录的路径
75+
rel_path = os.path.relpath(root, src_folder)
76+
# 对应的目标子目录路径
77+
target_dst_dir = os.path.join(dst_folder, rel_path)
78+
79+
# 1. 如果目标子目录不存在,直接创建
80+
if not os.path.exists(target_dst_dir):
81+
os.makedirs(target_dst_dir)
82+
print(f"创建新目录: {target_dst_dir}")
83+
84+
# 2. 处理当前目录下的所有文件
85+
for filename in files:
86+
# 隐藏文件跳过 (如 .DS_Store)
87+
if filename.startswith('.'): continue
88+
89+
src_file_path = os.path.join(root, filename)
90+
dst_file_path = os.path.join(target_dst_dir, filename)
91+
92+
if not os.path.exists(dst_file_path):
93+
# 如果目标位置没有这个文件,直接整体复制
94+
shutil.copy2(src_file_path, dst_file_path)
95+
print(f" [新文件] 已复制: {rel_path}/{filename}")
96+
else:
97+
# 如果目标位置有重名文件,执行深度合并
98+
merge_json_files(src_file_path, dst_file_path)
99+
100+
if __name__ == "__main__":
101+
print("开始深度合并任务...")
102+
start_recursive_merge()
103+
print("\n所有任务已结束。")
50 MB
Binary file not shown.
50 MB
Binary file not shown.
6.82 MB
Binary file not shown.

finance_mcp/config/ths_local.yaml

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
flow:
2+
crawl_ths_company:
3+
flow_content: |
4+
ReadLocalThsOp(tag="company")
5+
enable_cache: false
6+
cache_expire_hours: 1
7+
description: "通过A股股票代码获取公司资料信息,例如:详细情况,高管介绍,发行相关,参控股公司,最后返回和query相关的信息。"
8+
input_schema:
9+
code:
10+
type: string
11+
description: "stock code"
12+
required: true
13+
14+
crawl_ths_holder:
15+
flow_content: |
16+
ReadLocalThsOp(tag="holder")
17+
enable_cache: false
18+
cache_expire_hours: 1
19+
description: "通过A股股票代码获取股东研究信息,例如:股东人数、十大流通股东、十大股东、十大债券持有人、控股层级关系,最后返回和query相关的信息。"
20+
input_schema:
21+
code:
22+
type: string
23+
description: "stock code"
24+
required: true
25+
26+
crawl_ths_operate:
27+
flow_content: |
28+
ReadLocalThsOp(tag="operate")
29+
enable_cache: false
30+
cache_expire_hours: 1
31+
description: "通过A股股票代码获取经营分析信息,例如:主营介绍、运营业务数据、主营构成分析、主要客户及供应商、董事会经营评述、产品价格,最后返回和query相关的信息。"
32+
input_schema:
33+
code:
34+
type: string
35+
description: "stock code"
36+
required: true
37+
38+
crawl_ths_equity:
39+
flow_content: |
40+
ReadLocalThsOp(tag="equity")
41+
enable_cache: false
42+
cache_expire_hours: 1
43+
description: "通过A股股票代码获取股本结构信息,例如:解禁时间表、总股本构成、A股结构图、历次股本变动,最后返回和query相关的信息。"
44+
input_schema:
45+
code:
46+
type: string
47+
description: "stock code"
48+
required: true
49+
50+
crawl_ths_capital:
51+
flow_content: |
52+
ReadLocalThsOp(tag="capital")
53+
enable_cache: false
54+
cache_expire_hours: 1
55+
description: "通过A股股票代码获取资本运作信息,例如:募集资金来源、项目投资、收购兼并、股权投资、参股IPO、股权转让、关联交易、质押解冻,最后返回和query相关的信息。"
56+
input_schema:
57+
code:
58+
type: string
59+
description: "stock code"
60+
required: true
61+
62+
crawl_ths_worth:
63+
flow_content: |
64+
ReadLocalThsOp(tag="worth")
65+
enable_cache: false
66+
cache_expire_hours: 1
67+
description: "通过A股股票代码获取盈利预测信息,例如:业绩预测、业绩预测详表、研报评级,最后返回和query相关的信息。"
68+
input_schema:
69+
code:
70+
type: string
71+
description: "stock code"
72+
required: true
73+
74+
crawl_ths_news:
75+
flow_content: |
76+
ReadLocalThsOp(tag="news")
77+
enable_cache: false
78+
cache_expire_hours: 1
79+
description: "通过A股股票代码获取新闻公告信息,例如:新闻与股价联动、公告列表、热点新闻列表、研报列表,最后返回和query相关的信息。"
80+
input_schema:
81+
code:
82+
type: string
83+
description: "stock code"
84+
required: true
85+
86+
crawl_ths_concept:
87+
flow_content: |
88+
ReadLocalThsOp(tag="concept")
89+
enable_cache: false
90+
cache_expire_hours: 1
91+
description: "通过A股股票代码获取概念题材信息,例如:常规概念、其他概念、题材要点、概念对比,最后返回和query相关的信息。"
92+
input_schema:
93+
code:
94+
type: string
95+
description: "stock code"
96+
required: true
97+
98+
crawl_ths_position:
99+
flow_content: |
100+
ReadLocalThsOp(tag="position")
101+
enable_cache: false
102+
cache_expire_hours: 1
103+
description: "通过A股股票代码获取主力持仓信息,例如:机构持股汇总、机构持股明细、被举牌情况、IPO获配机构,最后返回和query相关的信息。"
104+
input_schema:
105+
code:
106+
type: string
107+
description: "stock code"
108+
required: true
109+
110+
crawl_ths_finance:
111+
flow_content: |
112+
ReadLocalThsOp(tag="finance")
113+
enable_cache: false
114+
cache_expire_hours: 1
115+
description: "通过A股股票代码获取财务分析信息,例如:财务诊断、财务指标、指标变动说明、资产负债构成、财务报告、杜邦分析,最后返回和query相关的信息。"
116+
input_schema:
117+
code:
118+
type: string
119+
description: "stock code"
120+
required: true
121+
122+
crawl_ths_bonus:
123+
flow_content: |
124+
ReadLocalThsOp(tag="bonus")
125+
enable_cache: false
126+
cache_expire_hours: 1
127+
description: "通过A股股票代码获取分红融资信息,例如:分红诊断、分红情况、增发机构获配明细、增发概况、配股概况,最后返回和query相关的信息。"
128+
input_schema:
129+
code:
130+
type: string
131+
description: "stock code"
132+
required: true
133+
134+
crawl_ths_event:
135+
flow_content: |
136+
ReadLocalThsOp(tag="event")
137+
enable_cache: false
138+
cache_expire_hours: 1
139+
description: "通过A股股票代码获取公司大事信息,例如:高管持股变动、股东持股变动、担保明细、违规处理、机构调研、投资者互动,最后返回和query相关的信息。"
140+
input_schema:
141+
code:
142+
type: string
143+
description: "stock code"
144+
required: true
145+
146+
crawl_ths_field:
147+
flow_content: |
148+
ReadLocalThsOp(tag="field")
149+
enable_cache: false
150+
cache_expire_hours: 1
151+
description: "通过A股股票代码获取行业对比信息,例如:行业地位、行业新闻,最后返回和query相关的信息。"
152+
input_schema:
153+
code:
154+
type: string
155+
description: "stock code"
156+
required: true

finance_mcp/core/agent/conduct_research_op.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def build_tool_call(self) -> ToolCall:
7474
)
7575

7676
async def async_execute(self):
77-
"""Run the multi-step research loop and produce a final answer.
77+
"""Run the multistep research loop and produce a final answer.
7878
7979
The method performs the following high-level steps:
8080

finance_mcp/core/crawl/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@
1313
"""
1414

1515
from .crawl4ai_op import Crawl4aiOp, Crawl4aiLongTextOp
16+
from .read_local_ths_op import ReadLocalThsOp
1617
from .ths_url_op import ThsUrlOp
1718

1819
__all__ = [
1920
"Crawl4aiOp",
2021
"Crawl4aiLongTextOp",
2122
"ThsUrlOp",
23+
"ReadLocalThsOp",
2224
]

0 commit comments

Comments
 (0)