|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Script to process .md and .ailly.md file pairs and generate JSONL output. |
| 4 | +
|
| 5 | +This script: |
| 6 | +1. Collects file data from .md and .ailly.md pairs |
| 7 | +2. Extracts content without front matter |
| 8 | +3. Generates JSONL output with prompt and model responses |
| 9 | +""" |
| 10 | + |
| 11 | +import os |
| 12 | +import json |
| 13 | +import re |
| 14 | +import argparse |
| 15 | +from pathlib import Path |
| 16 | +from typing import Dict, List, Tuple, Optional |
| 17 | + |
| 18 | + |
| 19 | +def collect_file_pairs( |
| 20 | + directory: str, limit: Optional[int] = None |
| 21 | +) -> List[Tuple[str, str]]: |
| 22 | + """ |
| 23 | + Collect pairs of .md and .ailly.md files from the specified directory. |
| 24 | +
|
| 25 | + Args: |
| 26 | + directory: Path to the directory containing the files |
| 27 | + limit: Optional limit on the number of pairs to process |
| 28 | +
|
| 29 | + Returns: |
| 30 | + List of tuples containing (md_file_path, ailly_md_file_path) |
| 31 | + """ |
| 32 | + md_files = {} |
| 33 | + ailly_md_files = {} |
| 34 | + |
| 35 | + # Walk through the directory and collect all .md and .ailly.md files |
| 36 | + for root, _, files in os.walk(directory): |
| 37 | + for file in files: |
| 38 | + file_path = os.path.join(root, file) |
| 39 | + if file.endswith(".md") and not file.endswith(".ailly.md"): |
| 40 | + md_files[file] = file_path |
| 41 | + elif file.endswith(".ailly.md"): |
| 42 | + base_name = file[:-9] # Remove '.ailly.md' |
| 43 | + ailly_md_files[base_name] = file_path |
| 44 | + |
| 45 | + # Match the pairs |
| 46 | + pairs = [] |
| 47 | + for base_name, md_path in md_files.items(): |
| 48 | + if base_name in ailly_md_files: |
| 49 | + pairs.append((md_path, ailly_md_files[base_name])) |
| 50 | + |
| 51 | + # Apply limit if specified |
| 52 | + if limit is not None and limit > 0: |
| 53 | + pairs = pairs[:limit] |
| 54 | + |
| 55 | + return pairs |
| 56 | + |
| 57 | + |
| 58 | +def extract_content(file_path: str) -> str: |
| 59 | + """ |
| 60 | + Extract content from a file, removing any front matter. |
| 61 | +
|
| 62 | + Args: |
| 63 | + file_path: Path to the file |
| 64 | +
|
| 65 | + Returns: |
| 66 | + Content of the file without front matter |
| 67 | + """ |
| 68 | + with open(file_path, "r", encoding="utf-8") as f: |
| 69 | + content = f.read() |
| 70 | + |
| 71 | + # Remove front matter if it exists (content between --- markers) |
| 72 | + front_matter_pattern = r"^---\n.*?\n---\n" |
| 73 | + content = re.sub(front_matter_pattern, "", content, flags=re.DOTALL) |
| 74 | + |
| 75 | + return content.strip() |
| 76 | + |
| 77 | + |
| 78 | +def get_aillyrc_content(directory: str) -> str: |
| 79 | + """ |
| 80 | + Get the content of the .aillyrc file without the front matter. |
| 81 | +
|
| 82 | + Args: |
| 83 | + directory: Directory containing the .aillyrc file |
| 84 | +
|
| 85 | + Returns: |
| 86 | + Content of the .aillyrc file without front matter |
| 87 | + """ |
| 88 | + # Find the .aillyrc file by going up directories if needed |
| 89 | + current_dir = directory |
| 90 | + aillyrc_path = None |
| 91 | + |
| 92 | + while current_dir and current_dir != "/": |
| 93 | + potential_path = os.path.join(current_dir, ".aillyrc") |
| 94 | + if os.path.exists(potential_path): |
| 95 | + aillyrc_path = potential_path |
| 96 | + break |
| 97 | + current_dir = os.path.dirname(current_dir) |
| 98 | + |
| 99 | + if not aillyrc_path: |
| 100 | + raise FileNotFoundError("Could not find .aillyrc file") |
| 101 | + |
| 102 | + return extract_content(aillyrc_path) |
| 103 | + |
| 104 | + |
| 105 | +def extract_model_identifier(ailly_file_path: str) -> Dict: |
| 106 | + """ |
| 107 | + Extract model identifier from the .ailly.md file's front matter. |
| 108 | +
|
| 109 | + Args: |
| 110 | + ailly_file_path: Path to the .ailly.md file |
| 111 | +
|
| 112 | + Returns: |
| 113 | + Dictionary containing model identifier information |
| 114 | + """ |
| 115 | + with open(ailly_file_path, "r", encoding="utf-8") as f: |
| 116 | + content = f.read() |
| 117 | + |
| 118 | + # Extract front matter |
| 119 | + front_matter_match = re.search(r"^---\n(.*?)\n---\n", content, re.DOTALL) |
| 120 | + if not front_matter_match: |
| 121 | + return {} |
| 122 | + |
| 123 | + front_matter = front_matter_match.group(1) |
| 124 | + |
| 125 | + # Extract debug information |
| 126 | + debug_match = re.search(r"debug:\s*\n(.*?)(\n\w|$)", front_matter, re.DOTALL) |
| 127 | + if not debug_match: |
| 128 | + return {} |
| 129 | + |
| 130 | + debug_content = debug_match.group(1) |
| 131 | + |
| 132 | + # Extract model information |
| 133 | + model_match = re.search(r"model:\s*(.*?)$", debug_content, re.MULTILINE) |
| 134 | + region_match = re.search(r"region:\s*(.*?)$", debug_content, re.MULTILINE) |
| 135 | + |
| 136 | + model_identifier = {} |
| 137 | + if model_match: |
| 138 | + model_identifier["model"] = model_match.group(1).strip() |
| 139 | + if region_match: |
| 140 | + model_identifier["region"] = region_match.group(1).strip() |
| 141 | + |
| 142 | + return model_identifier |
| 143 | + |
| 144 | + |
| 145 | +def convert_to_jsonl_format( |
| 146 | + file_pairs: List[Tuple[str, str]], aillyrc_content: str |
| 147 | +) -> List[Dict]: |
| 148 | + """ |
| 149 | + Convert file pairs to JSONL format. |
| 150 | +
|
| 151 | + Args: |
| 152 | + file_pairs: List of (md_file_path, ailly_md_file_path) tuples |
| 153 | + aillyrc_content: Content of the .aillyrc file |
| 154 | +
|
| 155 | + Returns: |
| 156 | + List of dictionaries in the required format |
| 157 | + """ |
| 158 | + jsonl_entries = [] |
| 159 | + |
| 160 | + for md_path, ailly_md_path in file_pairs: |
| 161 | + # Extract content from files |
| 162 | + md_content = extract_content(md_path) |
| 163 | + ailly_md_content = extract_content(ailly_md_path) |
| 164 | + |
| 165 | + # Extract model identifier |
| 166 | + model_identifier = extract_model_identifier(ailly_md_path) |
| 167 | + |
| 168 | + # Create JSONL entry |
| 169 | + entry = { |
| 170 | + "prompt": aillyrc_content + "\n\n" + md_content, |
| 171 | + "modelResponses": [ |
| 172 | + {"response": ailly_md_content, "modelIdentifier": model_identifier} |
| 173 | + ], |
| 174 | + } |
| 175 | + |
| 176 | + jsonl_entries.append(entry) |
| 177 | + |
| 178 | + return jsonl_entries |
| 179 | + |
| 180 | + |
| 181 | +def write_jsonl_file(entries: List[Dict], output_path: str) -> None: |
| 182 | + """ |
| 183 | + Write entries to a JSONL file. |
| 184 | +
|
| 185 | + Args: |
| 186 | + entries: List of dictionaries to write |
| 187 | + output_path: Path to the output file |
| 188 | + """ |
| 189 | + with open(output_path, "w", encoding="utf-8") as f: |
| 190 | + for entry in entries: |
| 191 | + f.write(json.dumps(entry) + "\n") |
| 192 | + |
| 193 | + |
| 194 | +def main(): |
| 195 | + """Main function to process files and generate JSONL output.""" |
| 196 | + parser = argparse.ArgumentParser( |
| 197 | + description="Process .md and .ailly.md file pairs and generate JSONL output." |
| 198 | + ) |
| 199 | + parser.add_argument( |
| 200 | + "--directory", |
| 201 | + "-d", |
| 202 | + type=str, |
| 203 | + default=".ailly_iam_policy/batch_01", |
| 204 | + help="Directory containing the file pairs", |
| 205 | + ) |
| 206 | + parser.add_argument( |
| 207 | + "--output", |
| 208 | + "-o", |
| 209 | + type=str, |
| 210 | + default="output.jsonl", |
| 211 | + help="Path to the output JSONL file", |
| 212 | + ) |
| 213 | + parser.add_argument( |
| 214 | + "--limit", |
| 215 | + "-n", |
| 216 | + type=int, |
| 217 | + default=None, |
| 218 | + help="Limit the number of file pairs to process", |
| 219 | + ) |
| 220 | + |
| 221 | + args = parser.parse_args() |
| 222 | + |
| 223 | + # Resolve paths |
| 224 | + base_dir = os.path.dirname(os.path.abspath(__file__)) |
| 225 | + directory = os.path.join(base_dir, args.directory) |
| 226 | + output_path = os.path.join(base_dir, args.output) |
| 227 | + |
| 228 | + # Step 1: Collect file pairs |
| 229 | + print(f"Collecting file pairs from {directory}...") |
| 230 | + file_pairs = collect_file_pairs(directory, args.limit) |
| 231 | + print(f"Found {len(file_pairs)} file pairs.") |
| 232 | + |
| 233 | + # Step 2: Get .aillyrc content |
| 234 | + print("Reading .aillyrc content...") |
| 235 | + aillyrc_content = get_aillyrc_content(directory) |
| 236 | + |
| 237 | + # Step 3: Convert to JSONL format |
| 238 | + print("Converting to JSONL format...") |
| 239 | + jsonl_entries = convert_to_jsonl_format(file_pairs, aillyrc_content) |
| 240 | + |
| 241 | + # Step 4: Write to output file |
| 242 | + print(f"Writing {len(jsonl_entries)} entries to {output_path}...") |
| 243 | + write_jsonl_file(jsonl_entries, output_path) |
| 244 | + |
| 245 | + print("Done!") |
| 246 | + |
| 247 | + |
| 248 | +if __name__ == "__main__": |
| 249 | + main() |
0 commit comments