fix(jianyu): block inaccessible detail links and verification pages (#918)

leozejia · jackwener · web-flow · commit 4d25b2b99e6b · 2026-04-20T23:00:10.000+08:00
* fix(jianyu): filter blocked detail links

* fix(jianyu): keep recency filter opt-in

---------

Co-authored-by: jackwener &lt;jakevingoo@gmail.com&gt;
diff --git a/clis/jianyu/search.js b/clis/jianyu/search.js
@@ -35,6 +35,10 @@ const NAVIGATION_PATH_PREFIXES = [
     '/exhibition/',
     '/swordfish/page_big_pc/search/',
 ];
+const BLOCKED_DETAIL_PATH_PREFIXES = [
+    '/nologin/content/',
+    '/article/bdprivate/',
+];
 const JIANYU_API_TYPES = ['fType', 'eType', 'vType', 'mType'];
 export function buildSearchUrl(query) {
     const url = new URL(SEARCH_ENTRY);
@@ -74,6 +78,92 @@ function isLikelyNavigationUrl(rawUrl) {
         return true;
     }
 }
+function classifyDetailStatus(rawUrl) {
+    const urlText = cleanText(rawUrl);
+    if (!urlText) {
+        return {
+            detail_status: 'blocked',
+            detail_reason: 'missing_url',
+        };
+    }
+    try {
+        const parsed = new URL(urlText);
+        const path = cleanText(parsed.pathname).toLowerCase().replace(/\/+$/, '/') || '/';
+        if (path.includes('/jybx/')) {
+            return {
+                detail_status: 'ok',
+                detail_reason: 'jybx_detail',
+            };
+        }
+        if (BLOCKED_DETAIL_PATH_PREFIXES.some((prefix) => path.includes(prefix))) {
+            return {
+                detail_status: 'blocked',
+                detail_reason: 'verification_or_paid_wall',
+            };
+        }
+        if (isLikelyNavigationUrl(urlText)) {
+            return {
+                detail_status: 'entry_only',
+                detail_reason: 'navigation_or_profile_entry',
+            };
+        }
+        return {
+            detail_status: 'entry_only',
+            detail_reason: 'non_jybx_entry',
+        };
+    }
+    catch {
+        return {
+            detail_status: 'blocked',
+            detail_reason: 'invalid_url',
+        };
+    }
+}
+function extractNoticeId(rawUrl) {
+    const value = cleanText(rawUrl);
+    if (!value)
+        return '';
+    try {
+        const parsed = new URL(value);
+        const path = cleanText(parsed.pathname);
+        const jybxMatched = path.match(/\/jybx\/([^/?#]+)\.html$/i);
+        if (jybxMatched?.[1])
+            return cleanText(jybxMatched[1]);
+        const segments = path.split('/').filter(Boolean);
+        const tail = cleanText(segments[segments.length - 1] || '');
+        return cleanText(tail.replace(/\.html?$/i, ''));
+    }
+    catch {
+        return '';
+    }
+}
+function isWithinSinceDays(dateText, sinceDays, now = new Date()) {
+    const normalized = normalizeDate(dateText);
+    if (!normalized)
+        return false;
+    const timestamp = Date.parse(`${normalized}T00:00:00Z`);
+    if (!Number.isFinite(timestamp))
+        return false;
+    const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
+    const deltaDays = Math.floor((today - timestamp) / (24 * 3600 * 1000));
+    return deltaDays >= 0 && deltaDays <= sinceDays;
+}
+function dedupeByNoticeKey(items) {
+    const deduped = [];
+    const seen = new Set();
+    for (const item of items) {
+        const source = cleanText(item.source_id || '');
+        const notice = cleanText(item.notice_id || '');
+        const key = source && notice
+            ? `${source}\t${notice}`
+            : `${cleanText(item.title)}\t${cleanText(item.url)}`;
+        if (!key || seen.has(key))
+            continue;
+        seen.add(key);
+        deduped.push(item);
+    }
+    return deduped;
+}
 function filterNavigationRows(query, items) {
     const queryTokens = cleanText(query).split(/\s+/).filter(Boolean).map((token) => token.toLowerCase());
     return items
@@ -86,6 +176,9 @@ function filterNavigationRows(query, items) {
         .filter((item) => {
         if (!item.title || !item.url)
             return false;
+        const detailSignal = classifyDetailStatus(item.url);
+        if (detailSignal.detail_status !== 'ok')
+            return false;
         const haystack = `${item.title} ${item.contextText}`.toLowerCase();
         const hasQuery = queryTokens.length === 0 || queryTokens.some((token) => haystack.includes(token));
         const hasProcurementHint = PROCUREMENT_TITLE_HINT.test(`${item.title} ${item.contextText}`);
@@ -446,11 +539,16 @@ cli({
     args: [
         { name: 'query', required: true, positional: true, help: 'Search keyword, e.g. "procurement"' },
         { name: 'limit', type: 'int', default: 20, help: 'Number of results (max 50)' },
+        { name: 'since_days', type: 'int', help: 'Only keep rows published within N days' },
     ],
-    columns: ['rank', 'content_type', 'title', 'publish_time', 'project_code', 'budget_or_limit', 'url'],
+    columns: ['rank', 'content_type', 'title', 'published_at', 'detail_status', 'project_code', 'budget_or_limit', 'url'],
     func: async (page, kwargs) => {
         const query = cleanText(kwargs.query);
         const limit = Math.max(1, Math.min(Number(kwargs.limit) || 20, 50));
+        const rawSinceDays = Number(kwargs.since_days);
+        const sinceDays = Number.isFinite(rawSinceDays) && rawSinceDays > 0
+            ? Math.max(1, Math.min(rawSinceDays, 3650))
+            : null;
         const apiResult = await fetchJianyuApiRows(page, query, limit);
         const mergedRows = dedupeCandidates(filterNavigationRows(query, apiResult.rows));
         const extractedRows = await searchRowsFromEntries(page, {
@@ -465,21 +563,61 @@ cli({
             const indexedRows = await fetchDuckDuckGoIndexRows(query, limit);
             const filteredIndexedRows = dedupeCandidates(filterNavigationRows(query, indexedRows));
             if (filteredIndexedRows.length > 0) {
-                return toProcurementSearchRecords(filteredIndexedRows, {
+                const records = toProcurementSearchRecords(filteredIndexedRows, {
                     site: SITE,
                     query,
                     limit,
                 });
+                const enriched = dedupeByNoticeKey(records.map((row) => {
+                    const detailSignal = classifyDetailStatus(row.url);
+                    const publishedAt = normalizeDate(row.publish_time || row.date);
+                    return {
+                        ...row,
+                        source_id: SITE,
+                        notice_id: extractNoticeId(row.url),
+                        published_at: publishedAt,
+                        detail_status: detailSignal.detail_status,
+                        detail_reason: detailSignal.detail_reason,
+                    };
+                }))
+                    .filter((row) => row.detail_status === 'ok')
+                    .filter((row) => sinceDays == null || isWithinSinceDays(row.published_at, sinceDays))
+                    .slice(0, limit)
+                    .map((row, index) => ({
+                    ...row,
+                    rank: index + 1,
+                }));
+                return enriched;
             }
             if (apiResult.challenge || await isAuthRequired(page)) {
                 throw new AuthRequiredError(DOMAIN, '[taxonomy=selector_drift] site=jianyu command=search blocked by human verification / access challenge');
             }
         }
-        return toProcurementSearchRecords(rows, {
+        const records = toProcurementSearchRecords(rows, {
             site: SITE,
             query,
             limit,
         });
+        const enriched = dedupeByNoticeKey(records.map((row) => {
+            const detailSignal = classifyDetailStatus(row.url);
+            const publishedAt = normalizeDate(row.publish_time || row.date);
+            return {
+                ...row,
+                source_id: SITE,
+                notice_id: extractNoticeId(row.url),
+                published_at: publishedAt,
+                detail_status: detailSignal.detail_status,
+                detail_reason: detailSignal.detail_reason,
+            };
+        }))
+            .filter((row) => row.detail_status === 'ok')
+            .filter((row) => sinceDays == null || isWithinSinceDays(row.published_at, sinceDays))
+            .slice(0, limit)
+            .map((row, index) => ({
+            ...row,
+            rank: index + 1,
+        }));
+        return enriched;
     },
 });
 export const __test__ = {
@@ -494,4 +632,8 @@ export const __test__ = {
     normalizeApiRow,
     fetchJianyuApiRows,
     collectApiRowsFromResponses,
+    classifyDetailStatus,
+    extractNoticeId,
+    isWithinSinceDays,
+    dedupeByNoticeKey,
 };
diff --git a/clis/jianyu/search.test.js b/clis/jianyu/search.test.js
@@ -31,7 +31,7 @@ describe('jianyu search helpers', () => {
         const filtered = __test__.filterNavigationRows('电梯', [
             { title: '招标公告', url: 'https://www.jianyu360.cn/list/stype/ZBGG.html', date: '' },
             { title: '帮助中心', url: 'https://www.jianyu360.cn/helpCenter/index', date: '' },
-            { title: '某项目电梯采购公告', url: 'https://www.jianyu360.cn/notice/detail/123', date: '2026-04-07' },
+            { title: '某项目电梯采购公告', url: 'https://shandong.jianyu360.cn/jybx/20260407_123.html', date: '2026-04-07' },
         ]);
         expect(filtered).toHaveLength(1);
         expect(filtered[0].title).toContain('电梯采购公告');
@@ -125,4 +125,20 @@ describe('jianyu search helpers', () => {
         expect(result.rows[0].title).toContain('电梯采购公告');
         expect(result.rows[1].title).toContain('另一条电梯采购公告');
     });
+    it('classifies nologin links as blocked detail targets', () => {
+        const signal = __test__.classifyDetailStatus('https://www.jianyu360.cn/nologin/content/ABC.html');
+        expect(signal.detail_status).toBe('blocked');
+    });
+    it('extracts stable notice id from jybx urls', () => {
+        const id = __test__.extractNoticeId('https://shandong.jianyu360.cn/jybx/20260310_26030938267551.html');
+        expect(id).toBe('20260310_26030938267551');
+    });
+    it('keeps only rows inside recency window', () => {
+        const within = __test__.isWithinSinceDays('2026-03-20', 30, new Date('2026-04-09T00:00:00Z'));
+        const stale = __test__.isWithinSinceDays('2026-02-01', 30, new Date('2026-04-09T00:00:00Z'));
+        const missing = __test__.isWithinSinceDays('', 30, new Date('2026-04-09T00:00:00Z'));
+        expect(within).toBe(true);
+        expect(stale).toBe(false);
+        expect(missing).toBe(false);
+    });
 });
diff --git a/clis/jianyu/shared/procurement-detail.js b/clis/jianyu/shared/procurement-detail.js
@@ -7,6 +7,13 @@ const RETRYABLE_DETAIL_ERROR_PATTERNS = [
     /cannot find context with specified id/i,
     /\[taxonomy=empty_result\]/i,
 ];
+const DETAIL_AUTH_CHALLENGE_PATTERNS = [
+    /请在下图依次点击/i,
+    /验证码/i,
+    /请完成验证/i,
+    /验证登录/i,
+    /登录即可获得更多浏览权限/i,
+];
 function isRetryableDetailError(error) {
     const message = error instanceof Error
         ? cleanText(error.message)
@@ -61,6 +68,14 @@ export async function runProcurementDetail(page, { url, site, query = '', }) {
             const title = cleanText(row.title);
             const detailText = cleanText(row.detailText);
             const publishTime = cleanText(row.publishTime);
+            const authGateText = cleanText(`${title} ${detailText}`);
+            if (DETAIL_AUTH_CHALLENGE_PATTERNS.some((pattern) => pattern.test(authGateText))) {
+                throw taxonomyError('selector_drift', {
+                    site,
+                    command: 'detail',
+                    detail: `detail page blocked by verification challenge: ${targetUrl}`,
+                });
+            }
             if (!title && !detailText) {
                 throw taxonomyError('empty_result', {
                     site,
diff --git a/clis/jianyu/shared/procurement-detail.test.js b/clis/jianyu/shared/procurement-detail.test.js
@@ -69,4 +69,16 @@ describe('procurement detail runner', () => {
         })).rejects.toThrow('[taxonomy=extraction_drift]');
         expect(attempts).toBe(1);
     });
+    it('rejects captcha/verification pages as selector_drift', async () => {
+        const page = createPage(async () => ({
+            title: '验证码',
+            detailText: '请在下图依次点击：槨畽黛',
+            publishTime: '',
+        }));
+        await expect(runProcurementDetail(page, {
+            url: 'https://www.jianyu360.cn/nologin/content/ABC.html',
+            site: 'jianyu',
+            query: '电梯',
+        })).rejects.toThrow('[taxonomy=selector_drift]');
+    });
 });
diff --git a/docs/adapters/browser/jianyu.md b/docs/adapters/browser/jianyu.md
@@ -6,7 +6,7 @@
 
 | Command | Description |
 |---------|-------------|
-| `opencli jianyu search "<query>" --limit <n>` | Search Jianyu bid notices (V2 structured contract) |
+| `opencli jianyu search "<query>" --limit <n> [--since_days <n>]` | Search Jianyu bid notices and keep only accessible detail links |
 | `opencli jianyu detail "<url>"` | Extract detail-page evidence blocks from a search URL |
 
 ## Usage Examples
@@ -15,8 +15,8 @@
 # Search by keyword
 opencli jianyu search "procurement" --limit 20 -f json
 
-# Search another keyword with a smaller window
-opencli jianyu search "substation" --limit 10 -f json
+# Search another keyword with an explicit recency window
+opencli jianyu search "substation" --limit 10 --since_days 30 -f json
 
 # Extract structured detail evidence
 opencli jianyu detail "https://www.jianyu360.cn/nologin/content/....html" -f json
@@ -29,10 +29,11 @@ opencli jianyu detail "https://www.jianyu360.cn/nologin/content/....html" -f jso
 
 ## Notes
 
-- `search` now returns V2 fields: `publish_time`, `source_site`, `content_type`, `is_detail_page`, `snippet`, `quality_flags`, plus compatible `date/summary`.
+- `search` returns accessible procurement rows with `published_at`, `detail_status`, `project_code`, `budget_or_limit`, `url`, plus compatible `publish_time/date`.
+- `search` keeps all reachable rows by default. `--since_days` enables an explicit recency filter.
 - `detail` returns the same structured fields and adds `detail_text` + `evidence_blocks`.
 - Date fields are normalized to `YYYY-MM-DD` when date text is detectable.
-- Results are deduplicated by `title + url`.
+- Results are deduplicated by stable notice id when it is available.
 - `--limit` defaults to `20` and is capped at `50`.
 
 ## Troubleshooting
