fix(sitemap-scraper): remove skipNavigation and parse sitemap from body

Author: nikitachapovskii-dev

packages/actor-scraper/sitemap-scraper/src/internals/crawler_setup.ts

@@ -200,8 +200,6 @@ export class CrawlerSetup {
                 url: sitemapUrl,
                 useExtendedUniqueKey: true,
                 keepUrlFragment: this.input.keepUrlFragments,
-                // sitemaps are fetched inside the handler
-                skipNavigation: true,
             }),
         );
 
@@ -249,6 +247,7 @@ export class CrawlerSetup {
 
         const options: HttpCrawlerOptions = {
             proxyConfiguration: this.proxyConfiguration,
+            httpClient: this.sitemapHttpClient,
             requestHandler: this._createRequestHandler(),
             preNavigationHooks: [],
             postNavigationHooks: [],
@@ -323,15 +322,17 @@ export class CrawlerSetup {
     protected async _handleSitemapRequest(
         crawlingContext: HttpCrawlingContext,
     ) {
-        const { request } = crawlingContext;
+        const { request, body } = crawlingContext;
 
         // Make sure that an object containing internal metadata
         // is present on every request.
         tools.ensureMetaData(request as any);
 
         log.info('Processing sitemap', { url: request.url });
+        const sitemapContent =
+            typeof body === 'string' ? body : body.toString('utf8');
         const parsed = parseSitemap(
-            [{ type: 'url', url: request.url }],
+            [{ type: 'raw', content: sitemapContent }],
             await this.proxyConfiguration?.newUrl(),
             {
                 emitNestedSitemaps: true,