fix: extract only body from iframe elements (#2986)

barjin · web-flow · commit c36166e24887 · 2025-05-27T15:53:00.000+02:00
After the `iframe` element extraction, `parseWithCheerio` ends up with invalid HTML (e.g. `meta` or `title` elements in the parent `body` element). The proposed changes replace the `iframe` element with a `div` containing the inside of the `iframe`'s `body`. Closes #2979
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -623,7 +623,15 @@ export async function parseWithCheerio(
                     const iframe = await frame.contentFrame();
 
                     if (iframe) {
-                        const contents = await iframe.content();
+                        const getIframeHTML = async (): Promise<string> => {
+                            try {
+                                return iframe.locator('body').first().innerHTML();
+                            } catch {
+                                return iframe.content();
+                            }
+                        };
+
+                        const contents = await getIframeHTML();
 
                         await frame.evaluate((f, c) => {
                             const replacementNode = document.createElement('div');
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -204,7 +204,15 @@ export async function parseWithCheerio(
                 try {
                     const iframe = await frame.contentFrame();
                     if (iframe) {
-                        const contents = await iframe.content();
+                        const getIframeHTML = async (): Promise<string> => {
+                            try {
+                                return iframe.$eval('body', (el) => el.innerHTML);
+                            } catch {
+                                return iframe.content();
+                            }
+                        };
+
+                        const contents = await getIframeHTML();
 
                         await frame.evaluate((f, c) => {
                             const replacementNode = document.createElement('div');
diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
@@ -170,8 +170,14 @@ describe('playwrightUtils', () => {
             const $ = await playwrightUtils.parseWithCheerio(page);
 
             const headings = $('h1')
-                .map((i, el) => $(el).text())
+                .map((_, el) => $(el).text())
                 .get();
+
+            const titles = $('title')
+                .map((_, el) => $(el).text())
+                .get();
+
+            expect(titles).toEqual(['Outside iframe title']);
             expect(headings).toEqual(['Outside iframe', 'In iframe']);
         } finally {
             await browser.close();
diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts
@@ -171,8 +171,14 @@ describe('puppeteerUtils', () => {
                 const $ = await puppeteerUtils.parseWithCheerio(page);
 
                 const headings = $('h1')
-                    .map((i, el) => $(el).text())
+                    .map((_, el) => $(el).text())
                     .get();
+
+                const titles = $('title')
+                    .map((_, el) => $(el).text())
+                    .get();
+
+                expect(titles).toEqual(['Outside iframe title']);
                 expect(headings).toEqual(['Outside iframe', 'In iframe']);
             } finally {
                 await browser.close();
diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
@@ -176,7 +176,7 @@ console.log('Hello world!');
     <!DOCTYPE html>
     <html>
         <head>
-            <title>Outside iframe</title>
+            <title>Outside iframe title</title>
         </head>
         <body>
             <h1>Outside iframe</h1>
