Return 451 for legally blocked wiki repos

Author: nelsonjchen

cfworkers/ghwsee-indexable-redirect/src/handler.ts

@@ -4,6 +4,31 @@ interface OriginalInfo {
   moved_to?: string
 }
 
+const LEGAL_BLOCK_REPOS = new Set(['mms75/sfz'])
+
+function repoSlugFromPathComponents(
+  pathComponents: string[],
+): string | undefined {
+  if (pathComponents.length < 3) {
+    return undefined
+  }
+
+  return `${pathComponents[1]}/${pathComponents[2]}`.toLowerCase()
+}
+
+function legalBlockResponse(repoSlug: string): Response {
+  return new Response(
+    `451 Unavailable For Legal Reasons - ${repoSlug} is unavailable on this service.`,
+    {
+      status: 451,
+      statusText: 'Unavailable For Legal Reasons',
+      headers: {
+        'content-type': 'text/plain; charset=utf-8',
+      },
+    },
+  )
+}
+
 class ModifiedDateAppender implements HTMLRewriterElementContentHandlers {
   date: Date
 
@@ -23,15 +48,19 @@ export async function handleRequest(request: Request): Promise<Response> {
     request.url.replace('github-wiki-see.page/m', 'github.com'),
   )
 
+  const pathComponents = githubUrl.pathname.split('/')
+  const repoSlug = repoSlugFromPathComponents(pathComponents)
+  if (repoSlug && LEGAL_BLOCK_REPOS.has(repoSlug)) {
+    return legalBlockResponse(repoSlug)
+  }
+
   const ghwseeResponse = fetch(request, {
     cf: {
       cacheEverything: true,
       cacheTtl: 7200,
     },
   })
 
-  const pathComponents = githubUrl.pathname.split('/')
-
   // Don't redirect wiki_index path. Index that, even for indexable wikis.
   if (pathComponents.length > 3 && pathComponents[2] === 'wiki_index') {
     return await ghwseeResponse

cfworkers/ghwsee-indexable-redirect/test/handler.test.ts

@@ -20,6 +20,16 @@ describe('handle', () => {
     expect(result.headers.has('Last-Modified')).toBeFalsy()
   })
 
+  test('returns 451 for a legally blocked repo', async () => {
+    const request_url = `https://github-wiki-see.page/m/mms75/sfz/wiki`
+    const result = await handleRequest(
+      new Request(request_url, { method: 'GET' }),
+    )
+
+    expect(result.status).toEqual(451)
+    expect(await result.text()).toContain('mms75/sfz')
+  })
+
   test('can determine if a URL is not indexable', async () => {
     const url = new URL('https://github.com/commaai/openpilot/wiki')
     expect(await (await originalInfo(url)).indexable).toBeFalsy()
@@ -110,6 +120,5 @@ describe('handle', () => {
     expect(result.headers.get('location')).toEqual(
       'https://github-wiki-see.page/m/node-config/node-config/wiki',
     )
-    
   })
 })

src/legal_block.rs

@@ -0,0 +1,15 @@
+use phf::phf_set;
+
+pub static LEGAL_BLOCK_LIST: phf::Set<&'static str> = phf_set! {
+    // Cloudflare Trust & Safety report 23d89ee107ed0942.
+    "mms75/sfz",
+};
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_contain_phf() {
+        let repo = "mms75/sfz".to_string();
+        assert!(super::LEGAL_BLOCK_LIST.contains(repo.as_str()));
+    }
+}

src/main.rs

@@ -16,6 +16,7 @@ use crate::scraper::process_markdown;
 
 mod decommission;
 mod gh_extensions;
+mod legal_block;
 mod retrieval;
 mod scraper;
 
@@ -152,7 +153,16 @@ async fn wiki_debug_sitemaps(
 ) -> Result<content::RawXml<String>, status::Custom<String>> {
     let content = retrieve_wiki_sitemap_index(account, repository, client)
         .await
-        .map_err(|error| status::Custom(Status::InternalServerError, format!("Error: {error}")))?;
+        .map_err(|error| {
+            let status = match error {
+                retrieval::ContentError::UnavailableForLegalReasons => {
+                    Status::UnavailableForLegalReasons
+                }
+                _ => Status::InternalServerError,
+            };
+
+            status::Custom(status, format!("Error: {error}"))
+        })?;
 
     Ok(content::RawXml(content))
 }
@@ -171,6 +181,7 @@ struct MirrorTemplate {
 enum MirrorError {
     // DocumentNotFound(NotFound<MirrorTemplate>),
     InternalError(status::Custom<content::RawHtml<String>>),
+    UnavailableForLegalReasons(status::Custom<content::RawHtml<String>>),
     GiveUpSendToGitHub(Redirect),
 }
 
@@ -186,6 +197,15 @@ fn mirror_internal_error(template: MirrorTemplate) -> MirrorError {
     MirrorError::InternalError(status::Custom(Status::InternalServerError, rendered))
 }
 
+fn mirror_unavailable_for_legal_reasons(account: &str, repository: &str) -> MirrorError {
+    MirrorError::UnavailableForLegalReasons(status::Custom(
+        Status::UnavailableForLegalReasons,
+        content::RawHtml(format!(
+            "451 Unavailable For Legal Reasons - {account}/{repository} is unavailable on this service."
+        )),
+    ))
+}
+
 #[get("/<account>/<repository>/wiki")]
 async fn mirror_home(
     account: &str,
@@ -290,6 +310,9 @@ async fn mirror_page(
             ContentError::Decommissioned => {
                 GiveUpSendToGitHub(Redirect::permanent(original_url_encoded.clone()))
             }
+            ContentError::UnavailableForLegalReasons => {
+                mirror_unavailable_for_legal_reasons(account, repository)
+            }
             ContentError::OtherError(e) => mirror_internal_error(MirrorTemplate {
                 original_title: page_title.clone(),
                 original_url: original_url.clone(),
@@ -361,6 +384,9 @@ async fn mirror_page_index(
             ContentError::Decommissioned => {
                 GiveUpSendToGitHub(Redirect::permanent(original_url.clone()))
             }
+            ContentError::UnavailableForLegalReasons => {
+                mirror_unavailable_for_legal_reasons(account, repository)
+            }
             ContentError::OtherError(e) => mirror_internal_error(MirrorTemplate {
                 original_title: page_title.clone(),
                 original_url: original_url.clone(),

src/retrieval.rs

@@ -6,6 +6,7 @@ use std::sync::LazyLock;
 use thiserror::Error;
 
 use crate::decommission::DECOMMISSION_LIST;
+use crate::legal_block::LEGAL_BLOCK_LIST;
 use crate::scraper::process_html_index;
 
 #[allow(dead_code)]
@@ -31,6 +32,8 @@ pub enum ContentError {
     TooMayRequests,
     #[error("wiki has been decommissioned")]
     Decommissioned,
+    #[error("unavailable for legal reasons")]
+    UnavailableForLegalReasons,
     #[error("{0}")]
     OtherError(String),
 }
@@ -45,6 +48,19 @@ fn repo_slug(account: &str, repository: &str) -> String {
     format!("{account}/{repository}")
 }
 
+fn reject_blocked_repo(account: &str, repository: &str) -> Result<(), ContentError> {
+    let repo = repo_slug(account, repository);
+    if LEGAL_BLOCK_LIST.contains(repo.as_str()) {
+        return Err(ContentError::UnavailableForLegalReasons);
+    }
+
+    if DECOMMISSION_LIST.contains(repo.as_str()) {
+        return Err(ContentError::Decommissioned);
+    }
+
+    Ok(())
+}
+
 fn raw_wiki_source_url(account: &str, repository: &str, page: &str, extension: &str) -> String {
     let page_encoded =
         percent_encoding::utf8_percent_encode(page, percent_encoding::NON_ALPHANUMERIC);
@@ -93,10 +109,7 @@ pub async fn retrieve_source_file(
     page: &str,
     client: &Client,
 ) -> Result<Content, ContentError> {
-    // Skip decommissioned wikis
-    if DECOMMISSION_LIST.contains(repo_slug(account, repository).as_str()) {
-        return Err(ContentError::Decommissioned);
-    }
+    reject_blocked_repo(account, repository)?;
 
     match retrieve_source_file_extension(account, repository, page, client, Content::Markdown, "md")
         .await
@@ -183,6 +196,8 @@ pub async fn retrieve_wiki_index(
     repository: &str,
     client: &Client,
 ) -> Result<Content, ContentError> {
+    reject_blocked_repo(account, repository)?;
+
     let html = with_rate_limit_fallback(|domain| async move {
         retrieve_github_com_html(account, repository, "", client, domain).await
     })
@@ -208,6 +223,8 @@ pub async fn retrieve_wiki_sitemap_index(
     repository: &str,
     client: &Client,
 ) -> Result<String, ContentError> {
+    reject_blocked_repo(account, repository)?;
+
     let html = with_rate_limit_fallback(|domain| async move {
         retrieve_github_com_html(account, repository, "", client, domain).await
     })
@@ -368,6 +385,28 @@ mod tests {
         assert!(content.is_ok());
     }
 
+    #[tokio::test]
+    async fn legal_block_source_file() {
+        let client = Client::new();
+        let content = retrieve_source_file("mms75", "sfz", "Home", &client).await;
+
+        assert!(matches!(
+            content,
+            Err(ContentError::UnavailableForLegalReasons)
+        ));
+    }
+
+    #[tokio::test]
+    async fn legal_block_wiki_index() {
+        let client = Client::new();
+        let content = retrieve_wiki_index("mms75", "sfz", &client).await;
+
+        assert!(matches!(
+            content,
+            Err(ContentError::UnavailableForLegalReasons)
+        ));
+    }
+
     #[tokio::test]
     async fn wiki_sitemap_index() {
         let client = Client::new();