|
7 | 7 |
|
8 | 8 | import aiofiles |
9 | 9 | import aiofiles.os |
| 10 | +from pydantic import ValidationError |
10 | 11 |
|
11 | 12 | from crowdgit.database.crud import ( |
12 | 13 | fetch_member_organizations, |
@@ -459,11 +460,25 @@ def normalize_parsed_affiliations( |
459 | 460 |
|
460 | 461 | async def parse_affiliations(self, content: str) -> tuple[list[AffiliationInfoItem], float]: |
461 | 462 | """Extract affiliations with AI, splitting large files into chunks when needed.""" |
| 463 | + |
| 464 | + async def invoke_parse(file_content: str): |
| 465 | + for attempt in range(2): |
| 466 | + try: |
| 467 | + return await invoke_bedrock( |
| 468 | + self.get_extraction_prompt(file_content), |
| 469 | + pydantic_model=AffiliationParseOutput, |
| 470 | + ) |
| 471 | + except ValidationError: |
| 472 | + if attempt == 0: |
| 473 | + self.logger.info("Malformed affiliation parse response, retrying once") |
| 474 | + continue |
| 475 | + raise AffiliationAnalysisError( |
| 476 | + retain_file_hash=True, |
| 477 | + error_message="Affiliation file could not be parsed cleanly after retry", |
| 478 | + ) from None |
| 479 | + |
462 | 480 | if len(content) <= self.MAX_CHUNK_SIZE: |
463 | | - parse_result = await invoke_bedrock( |
464 | | - self.get_extraction_prompt(content), |
465 | | - pydantic_model=AffiliationParseOutput, |
466 | | - ) |
| 481 | + parse_result = await invoke_parse(content) |
467 | 482 |
|
468 | 483 | affiliations = parse_result.output.affiliations |
469 | 484 | if affiliations is not None: |
@@ -504,10 +519,7 @@ async def parse_affiliations(self, content: str) -> tuple[list[AffiliationInfoIt |
504 | 519 |
|
505 | 520 | async def process_chunk(chunk: str): |
506 | 521 | async with semaphore: |
507 | | - return await invoke_bedrock( |
508 | | - self.get_extraction_prompt(chunk), |
509 | | - pydantic_model=AffiliationParseOutput, |
510 | | - ) |
| 522 | + return await invoke_parse(chunk) |
511 | 523 |
|
512 | 524 | chunk_results = await asyncio.gather(*[process_chunk(chunk) for chunk in chunks]) |
513 | 525 |
|
@@ -807,7 +819,7 @@ async def process_affiliations( |
807 | 819 | ) |
808 | 820 | ) |
809 | 821 |
|
810 | | - self.logger.info("Starting affiliations") |
| 822 | + self.logger.info(f"Starting affiliations processing for repo: {batch_info.remote}") |
811 | 823 |
|
812 | 824 | saved_file_path = registry.file_path if registry else None |
813 | 825 | latest_file_path, discovery_cost = await self.resolve_affiliation_file( |
|
0 commit comments