Merge pull request #221 from yujiosaka/fix-csp-bug

fix(hccrawler): fix a bug of CSP breaks the crawler
yujiosaka · Apr 20, 2018 · a3498f7 · a3498f7
2 parents 44a44b3 + 465d9dc
commit a3498f7
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 0 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -76,6 +76,7 @@ class Crawler {
       this._authenticate(),
       this._emulate(),
       this._setViewport(),
+      this._setBypassCSP(),
       this._setCacheEnabled(),
       this._setUserAgent(),
       this._setExtraHeaders(),
@@ -134,6 +135,16 @@ class Crawler {
     await this._page.setCacheEnabled(false);
   }
 
+  /**
+   * @return {!Promise}
+   * @private
+   */
+  async _setBypassCSP() {
+    if (!this._options.jQuery) return;
+    // @ts-ignore
+    await this._page.setBypassCSP(true);
+  }
+
   /**
    * @return {!Promise}
    * @private

diff --git a/test/hccrawler/index.test.js b/test/hccrawler/index.test.js
@@ -505,6 +505,35 @@ describe('HCCrawler', function () {
           assert.ok(includes(onError.firstCall.args[0].message, 'Evaluation failed:'));
         });
 
+        context('when the page is protected by CSP meta tag', async function () {
+          beforeEach(function () {
+            server.setContent('/csp.html', `
+            <meta http-equiv="Content-Security-Policy" content="default-src 'self'">
+            <h1>Welcome to ${PREFIX}/csp.html</h1>
+            `);
+          });
+
+          it('succeeds evaluating page', async function () {
+            await crawler.queue({ url: `${PREFIX}/csp.html`, evaluatePage });
+            await crawler.onIdle();
+            assert.equal(onSuccess.callCount, 1);
+            assert.ok(includes(onSuccess.firstCall.args[0].result, 'Welcome to'));
+          });
+        });
+
+        context('when the page is protected by CSP header', async function () {
+          beforeEach(function () {
+            server.setCSP('/empty.html', 'default-src "self"');
+          });
+
+          it('succeeds evaluating page', async function () {
+            await crawler.queue({ url: `${INDEX_PAGE}`, evaluatePage });
+            await crawler.onIdle();
+            assert.equal(onSuccess.callCount, 1);
+            assert.equal(onSuccess.firstCall.args[0].result, '/');
+          });
+        });
+
         context('when the page response is delayed', async function () {
           beforeEach(function () {
             server.setResponseDelay('/', 200);

diff --git a/test/server/index.js b/test/server/index.js
@@ -22,13 +22,15 @@ class Server {
     this._server.listen(port);
     this._delays = new Map();
     this._auths = new Map();
+    this._csps = new Map();
     this._contents = new Map();
   }
 
   reset() {
     this._delays.clear();
     this._auths.clear();
     this._contents.clear();
+    this._csps.clear();
   }
 
   /**
@@ -47,6 +49,14 @@ class Server {
     this._auths.set(path, { username, password });
   }
 
+  /**
+   * @param {string} path
+   * @param {string} csp
+   */
+  setCSP(path, csp) {
+    this._csps.set(path, csp);
+  }
+
   /**
    * @param {!string} path
    * @param {!string} content