fix: suppress exceptions while maxrequestreached reached

Added checks for opened browser connection. yujiosaka#349
popstas · Mar 11, 2021 · 43a7c14 · 43a7c14
1 parent c5a2957
commit 43a7c14
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 4 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -77,7 +77,11 @@ class Crawler {
    * @return {!Promise}
    */
   async close() {
-    await this._page.close();
+    try {
+      await this._page.close();
+    } catch(e) {
+      // console.log('Error while close page');
+    }
   }
 
   /**

diff --git a/lib/hccrawler.js b/lib/hccrawler.js
@@ -202,6 +202,7 @@ class HCCrawler extends EventEmitter {
    * @return {!Promise<!string>}
    */
   userAgent() {
+    if (this._browser._connection._closed) return; // catch error after scan
     return this._browser.userAgent();
   }
 
@@ -342,11 +343,12 @@ class HCCrawler extends EventEmitter {
       await this._markRequested(options);
       await this._markRequestedRedirects(options, res.redirectChain, res.response);
       if (requested) return [];
+      if (this._browser._connection._closed) return; // catch error after scan
       this._exportLine(res);
       await this._success(res);
       return res.links;
     } catch (error) {
-      await crawler.close();
+      if (crawler) await crawler.close();
       extend(error, { options, depth, previousUrl });
       if (retryCount >= options.retryCount) {
         this.emit(HCCrawler.Events.RequestFailed, error);
@@ -551,8 +553,13 @@ class HCCrawler extends EventEmitter {
    * @private
    */
   async _newCrawler(options, depth, previousUrl) {
-    const page = await this._browser.newPage();
-    return new Crawler(page, options, depth, previousUrl);
+    try {
+      const page = await this._browser.newPage();
+      return new Crawler(page, options, depth, previousUrl);
+    } catch(e) {
+      if (this._browser._connection._closed) return; // catch error after scan
+      console.log('Error while _newCrawler');
+    }
   }
 
   /**