diff --git a/lib/crawler.js b/lib/crawler.js index b74b85f6..37d58f73 100644 --- a/lib/crawler.js +++ b/lib/crawler.js @@ -76,6 +76,7 @@ class Crawler { this._authenticate(), this._emulate(), this._setViewport(), + this._setBypassCSP(), this._setCacheEnabled(), this._setUserAgent(), this._setExtraHeaders(), @@ -134,6 +135,16 @@ class Crawler { await this._page.setCacheEnabled(false); } + /** + * @return {!Promise} + * @private + */ + async _setBypassCSP() { + if (!this._options.jQuery) return; + // @ts-ignore + await this._page.setBypassCSP(true); + } + /** * @return {!Promise} * @private diff --git a/test/hccrawler/index.test.js b/test/hccrawler/index.test.js index 5b1baf42..8b819fb7 100644 --- a/test/hccrawler/index.test.js +++ b/test/hccrawler/index.test.js @@ -505,6 +505,35 @@ describe('HCCrawler', function () { assert.ok(includes(onError.firstCall.args[0].message, 'Evaluation failed:')); }); + context('when the page is protected by CSP meta tag', async function () { + beforeEach(function () { + server.setContent('/csp.html', ` + +

Welcome to ${PREFIX}/csp.html

+ `); + }); + + it('succeeds evaluating page', async function () { + await crawler.queue({ url: `${PREFIX}/csp.html`, evaluatePage }); + await crawler.onIdle(); + assert.equal(onSuccess.callCount, 1); + assert.ok(includes(onSuccess.firstCall.args[0].result, 'Welcome to')); + }); + }); + + context('when the page is protected by CSP header', async function () { + beforeEach(function () { + server.setCSP('/empty.html', 'default-src "self"'); + }); + + it('succeeds evaluating page', async function () { + await crawler.queue({ url: `${INDEX_PAGE}`, evaluatePage }); + await crawler.onIdle(); + assert.equal(onSuccess.callCount, 1); + assert.equal(onSuccess.firstCall.args[0].result, '/'); + }); + }); + context('when the page response is delayed', async function () { beforeEach(function () { server.setResponseDelay('/', 200); diff --git a/test/server/index.js b/test/server/index.js index d242a3b3..088f6657 100644 --- a/test/server/index.js +++ b/test/server/index.js @@ -22,6 +22,7 @@ class Server { this._server.listen(port); this._delays = new Map(); this._auths = new Map(); + this._csps = new Map(); this._contents = new Map(); } @@ -29,6 +30,7 @@ class Server { this._delays.clear(); this._auths.clear(); this._contents.clear(); + this._csps.clear(); } /** @@ -47,6 +49,14 @@ class Server { this._auths.set(path, { username, password }); } + /** + * @param {string} path + * @param {string} csp + */ + setCSP(path, csp) { + this._csps.set(path, csp); + } + /** * @param {!string} path * @param {!string} content