feat: move away from Wikipedia to UNESCO for a crawling exercise, JavaScript

Author: honzajavorek

sources/academy/webscraping/scraping_basics_javascript/10_crawling.md

@@ -8,7 +8,7 @@ slug: /scraping-basics-javascript/crawling
 import CodeBlock from '@theme/CodeBlock';
 import LegacyJsCourseAdmonition from '@site/src/components/LegacyJsCourseAdmonition';
 import Exercises from '../scraping_basics/_exercises.mdx';
-import WikipediaCallingCodesExercise from '!!raw-loader!roa-loader!./exercises/wikipedia_calling_codes.mjs';
+import UnescoWhsCountsExercise from '!!raw-loader!roa-loader!./exercises/unesco_whs_counts.mjs';
 import GuardianF1AuthorsExercise from '!!raw-loader!roa-loader!./exercises/guardian_f1_authors.mjs';
 
 <LegacyJsCourseAdmonition />
@@ -210,24 +210,21 @@ In the next lesson, we'll scrape the product detail pages so that each product v
 
 <Exercises />
 
-### Scrape calling codes of African countries
+### Scrape UNESCO World Heritage Sites
 
-Scrape links to Wikipedia pages for all African states and territories. Follow each link and extract the _calling code_ from the info table. Print the URL and the calling code for each country. Start with this URL:
+Scrape links to detail pages of all UNESCO members. Follow each link and extract the count of the World Heritage Sites. Print the URL and the number for each country. Start with this URL:
 
 ```text
-https://en.wikipedia.org/wiki/List_of_sovereign_states_and_dependent_territories_in_Africa
+https://www.unesco.org/en/countries
 ```
 
 Your program should print the following:
 
 ```text
-https://en.wikipedia.org/wiki/Algeria +213
-https://en.wikipedia.org/wiki/Angola +244
-https://en.wikipedia.org/wiki/Benin +229
-https://en.wikipedia.org/wiki/Botswana +267
-https://en.wikipedia.org/wiki/Burkina_Faso +226
-https://en.wikipedia.org/wiki/Burundi null
-https://en.wikipedia.org/wiki/Cameroon +237
+https://www.unesco.org/en/countries/af 2
+https://www.unesco.org/en/countries/al 4
+https://www.unesco.org/en/countries/dz 7
+https://www.unesco.org/en/countries/ad 1
 ...
 ```
 
@@ -239,7 +236,7 @@ Locating cells in tables is sometimes easier if you know how to [filter](https:/
 
 <details>
   <summary>Solution</summary>
-  <CodeBlock language="js">{WikipediaCallingCodesExercise.code}</CodeBlock>
+  <CodeBlock language="js">{UnescoWhsCountsExercise.code}</CodeBlock>
 </details>
 
 ### Scrape authors of F1 news articles

sources/academy/webscraping/scraping_basics_javascript/exercises/test.bats

@@ -90,7 +90,7 @@ teardown_file() {
   run node unesco_links.mjs
 
   [[ "$output" == *$'https://www.unesco.org/en/countries/af\nhttps://www.unesco.org/en/countries/al\n'* ]]
-  [[ $(echo "$output" | wc -l) -gt 50 ]]
+  [[ $(echo "$output" | wc -l) -gt 5 ]]
 }
 
 @test "lists Guardian F1 article links" {
@@ -100,11 +100,11 @@ teardown_file() {
   [[ $(echo "$output" | wc -l) -gt 5 ]]
 }
 
-@test "prints Wikipedia calling codes" {
-  run node wikipedia_calling_codes.mjs
+@test "prints counts of UNESCO WHS" {
+  run node unesco_whs_counts.mjs
 
-  [[ "$output" == *$'https://en.wikipedia.org/wiki/Comoros +269\n'* ]]
-  [[ "$output" == *$'https://en.wikipedia.org/wiki/Sahrawi_Arab_Democratic_Republic null\n'* ]]
+  [[ "$output" == *$'https://www.unesco.org/en/countries/af 2\n'* ]]
+  [[ "$output" == *$'https://www.unesco.org/en/countries/bs 0\n'* ]]
   [[ $(echo "$output" | wc -l) -gt 5 ]]
 }
 

sources/academy/webscraping/scraping_basics_javascript/exercises/unesco_whs_counts.mjs

@@ -0,0 +1,41 @@
+import * as cheerio from 'cheerio';
+
+async function download(url) {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}`);
+  }
+  const html = await response.text();
+  return cheerio.load(html);
+}
+
+function parseWhsCount($) {
+  for (const element of $('.card-body').toArray()) {
+    const $card = $(element);
+    const title = $card.find('.card-title').text();
+
+    if (title.includes('World Heritage Sites')) {
+      const number = $card.find('.card-number').text().trim();
+      return Number.parseInt(number, 10);
+    }
+  }
+  return 0;
+}
+
+const listingUrl = 'https://www.unesco.org/en/countries';
+const $listing = await download(listingUrl);
+
+for (const element of $listing('.node--type-country').toArray()) {
+  const $countryCard = $listing(element);
+  const $link = $countryCard.find('a').first();
+  const href = $link.attr('href');
+
+  if (!href) {
+    continue;
+  }
+
+  const countryUrl = new URL(href, listingUrl).href;
+  const $country = await download(countryUrl);
+  const whsCount = parseWhsCount($country);
+  console.log(`${countryUrl} ${whsCount}`);
+}

sources/academy/webscraping/scraping_basics_python/exercises/test.bats

@@ -83,7 +83,7 @@ teardown() {
   run uv run --with=httpx --with=beautifulsoup4 python unesco_links.py
 
   [[ "$output" == *$'https://www.unesco.org/en/countries/af\nhttps://www.unesco.org/en/countries/al\n'* ]]
-  [[ $(echo "$output" | wc -l) -gt 50 ]]
+  [[ $(echo "$output" | wc -l) -gt 5 ]]
 }
 
 @test "lists Guardian F1 article links" {
@@ -98,7 +98,7 @@ teardown() {
 
   [[ "$output" == *$'https://www.unesco.org/en/countries/af 2\n'* ]]
   [[ "$output" == *$'https://www.unesco.org/en/countries/bs 0\n'* ]]
-  [[ $(echo "$output" | wc -l) -gt 50 ]]
+  [[ $(echo "$output" | wc -l) -gt 5 ]]
 }
 
 @test "lists Guardian F1 authors" {