Merge pull request #45 from yujiosaka/fix_too_many_tabs

Fix a bug of opening too many tabs when error occured
yujiosaka · Dec 17, 2017 · 780b4d7 · 780b4d7
2 parents a322634 + 1e31217
commit 780b4d7
Showing 1 changed file with 18 additions and 16 deletions.
diff --git a/lib/hccrawler.js b/lib/hccrawler.js
@@ -265,25 +265,27 @@ class HCCrawler {
                   links: res.links,
                   options,
                 }))
+                  .then(() => void debugRequest(`End requesting ${this._options.url}`))
                   .then(() => crawler.close())
+                  .then(() => void this._checkRequestCount())
+                  .then(() => delay(options.delay))
                   .then(() => this._followLinks(res.links, options, depth))
               ))
-          ))
-          .then(() => void debugRequest(`End requesting ${this._options.url}`))
-          .then(() => delay(options.delay))
-          .then(() => void this._checkRequestCount());
-      })
-      .catch(error => {
-        if (retryCount >= options.retryCount) throw new Error(`Retry give-up for requesting ${options.url}!`, error);
-        debugRequest(`Retry requesting ${options.url} ${retryCount + 1} times`);
-        return delay(options.retryDelay)
-          .then(() => this._removeExists(options))
-          .then(() => this._request(options, depth, retryCount + 1));
-      })
-      .catch(error => {
-        debugRequest(`Retry give-up for requesting ${options.url} after ${retryCount} tries`);
-        const onError = options.onError || noop;
-        return Promise.resolve(onError(error));
+              .catch(error => {
+                if (retryCount >= options.retryCount) throw new Error(`Retry give-up for requesting ${options.url}!`, error);
+                debugRequest(`Retry requesting ${options.url} ${retryCount + 1} times`);
+                return crawler.close()
+                  .then(() => delay(options.retryDelay))
+                  .then(() => this._removeExists(options))
+                  .then(() => this._request(options, depth, retryCount + 1));
+              })
+              .catch(error => {
+                debugRequest(`Retry give-up for requesting ${options.url} after ${retryCount} tries`);
+                const onError = options.onError || noop;
+                return crawler.close()
+                  .then(() => Promise.resolve(onError(error)));
+              })
+          ));
       });
   }