Adding waitAfterLoadedFor setting, and basic link following

Author: jjgonecrypto

README.md

@@ -18,9 +18,11 @@ Currently
 * ~~`0.3.0` Stream support (instead of promises)~~
     * ~~`0.3.1` Migrated to [nightmare](https://github.com/segmentio/nightmare) - 3x faster than Webdriver/Phantom2 and option to open up devTools~~
     * ~~`0.3.2` Stream usage cleanup~~
-* ~~`0.4.0` Support to keep the session alive~~
-* *[pending]* Handle console errors that occur after `DOMContentLoaded` (with configurable timeout)
-* *[pending]* Follow and track all internal links
+* ~~`0.4.0` Support to keep the session alive via `keepAlive`~~
+* ~~`0.5.0` Handles console errors that occur after `DOMContentLoaded` (with configurable timeout `waitAfterLoadedFor (ms)`). Follows links now.~~
+* *[pending]* Prevent visiting the same link twice. 
+* *[pending]* Allow max number of links to follow or timeout.
+* *[pending]* Filter function to use for following links (defaults to implicit domain name, within same protocol://host:port)
 * *[pending]* Allow for `setup`/`teardown` actions in nightmare (such as login) (or perhaps just use cookies) 
 
 ##Usage
@@ -56,4 +58,9 @@ const hunt = hounds.release({
 }).on('error', console.error)
 ```
 
-> All [nightmare 2.7.0 options](https://github.com/segmentio/nightmare/tree/2.7.0#nightmareoptions) are supported
+##Options
+
+* `url` base URL to start from
+* `keepAlive` don't end the stream or the nightmare session when complete (when combined with `nightmare.show`, allows you to interact with the browser when done).
+* `waitAfterLoadedFor` The number of milliseconds to wait after each page is loaded before following the next link in the queue
+* `nightmare` All [nightmare 2.7.0 options](https://github.com/segmentio/nightmare/tree/2.7.0#nightmareoptions) are supported

example/index.js

@@ -13,12 +13,13 @@ app.listen(4441)
 
 const hunt = hounds.release({
     url: 'http://localhost:4441',
-    keepAlive: true,
-    waitAfterLoadedFor: 1000,
+    // keepAlive: true,
+    waitAfterLoadedFor: 600,
     nightmare: {
-        show: true, openDevTools: true
+        // show: true, openDevTools: true
     }
 }).on('error', console.error)
+.on('end', process.exit)
 
 const quarry = new Writable({
     objectMode: true,

index.js

@@ -12,11 +12,15 @@ exports.release = (options) => {
     const queue = [ options.url ]
     const nightmare = new Nightmare(options.nightmare)
 
+    // Note: this is potentially buggy (see pending test)
+    // We'd prefer that the page event below emit the URL at error time
+    let url
+
     let session = nightmare
         .on('page', function(type, message, stack) {
             const stackTrace = stack.split('\n').map(l => l.trim())
             quarry.push({
-                url: options.url,
+                url,
                 message,
                 stackTrace
             })
@@ -30,11 +34,18 @@ exports.release = (options) => {
 
             return
         }
-        const url = queue.pop()
+        url = queue.shift()
         session
             .goto(url)
             .wait(options.waitAfterLoadedFor)
-            .then(() => {
+            .evaluate(() => {
+                /* eslint-env browser */
+                const anchors = document.querySelectorAll('a[href]')
+                return [].slice.call(anchors).map(a => a.href)
+            })
+            .then(anchors => {
+                anchors = Array.isArray(anchors) ? anchors : [ anchors ]
+                anchors.forEach(href => queue.push(href))
                 processNextInQueue()
             })
             .catch(e => quarry.emit('error', e))

test/fixture/empty.html

@@ -23,7 +23,7 @@ describe('hounds', function() {
     })
 
     beforeEach(() => {
-        this.options = { url: 'http://localhost:4441' }
+        this.options = { url: 'http://localhost:4441/' }
 
         this.assertErrorReceived = () => {}
 
@@ -43,6 +43,10 @@ describe('hounds', function() {
             this.hunt = hounds.release(this.options)
         })
 
+        afterEach(() => {
+            this.hunt.unpipe(this.quarry)
+        })
+
         it('detects the single console error', done => {
             this.assertErrorReceived = (chunk, callCount) => {
                 if (callCount !== 1) return
@@ -68,13 +72,101 @@ describe('hounds', function() {
         })
     })
 
-    describe('returns error async after load if kept alive', () => {
+
+    describe('when setup on an empty page', () => {
+        beforeEach(() => {
+            this.options.url += 'empty.html'
+        })
+
+        describe('when released', () => {
+            beforeEach(() => {
+                this.hunt = hounds.release(this.options)
+            })
+
+            afterEach(() => {
+                this.hunt.unpipe(this.quarry)
+            })
+
+            it('then it ends the stream when complete', done => {
+                this.hunt.on('error', done).on('end', done).pipe(this.quarry)
+            })
+        })
+
+        describe('when keepAlive is provided and relesed', () => {
+            beforeEach(() => {
+                this.options.keepAlive = true
+                this.hunt = hounds.release(this.options)
+            })
+
+            afterEach(() => {
+                this.hunt.unpipe(this.quarry)
+            })
+
+            it('then it does NOT end the stream when complete', done => {
+                this.hunt.on('error', done).on('end', () => { throw Error('The hunt was ended early!') }).pipe(this.quarry)
+                setTimeout(done, 2000)
+            })
+        })
+
+    })
+
+    describe('returns error from followed page', () => {
         beforeEach(() => {
-            this.options.keepAlive = true
             this.hunt = hounds.release(this.options)
         })
 
-        it('detects the single console error', done => {
+        it('detects the console error in a followed page', done => {
+            this.assertErrorReceived = (chunk, callCount) => {
+                if (callCount !== 3) return
+                assert.equal(`${this.options.url}second.html`, chunk.url, 'URL is passed through')
+                assert.equal('Uncaught Error: This is supposed to happen', chunk.message, 'Page error on second.html caught')
+                assert.equal(3, chunk.stackTrace.length, 'Page error stacktrace is captured')
+                done()
+            }
+
+            this.hunt.on('error', done).pipe(this.quarry)
+        })
+    })
+
+    describe('when waiting on each page for 600ms after load before moving on', () => {
+        beforeEach(() => {
+            this.options.waitAfterLoadedFor = 600
+            this.hunt = hounds.release(this.options)
+        })
+
+        afterEach(() => {
+            this.hunt.unpipe(this.quarry)
+        })
+
+        it('then it detects a timed out error of 500ms', done => {
+            this.assertErrorReceived = (chunk, callCount) => {
+                if (callCount !== 3) return
+                assert.equal(this.options.url, chunk.url, 'URL is passed through')
+                assert.equal('Uncaught Error: Error after 500ms', chunk.message, 'Page error 500ms after load is caught')
+                assert.equal(3, chunk.stackTrace.length, 'Page error stacktrace is captured')
+                done()
+            }
+
+            this.hunt.on('error', done).pipe(this.quarry)
+        })
+    })
+
+    // Known bug:
+    // We aren't guaranteed that the URL is still valid here,
+    // as it may have changed between when this error was thrown and now.
+    // A better solution is for nightmare to also emit `win.webContents.getURL()`
+    // as another argument in https://github.com/segmentio/nightmare/blob/master/lib/runner.js#L115
+    xdescribe('when waiting on each page for 400ms after load before moving on', () => {
+        beforeEach(() => {
+            this.options.waitAfterLoadedFor = 400
+            this.hunt = hounds.release(this.options)
+        })
+
+        afterEach(() => {
+            this.hunt.unpipe(this.quarry)
+        })
+
+        it('then it still detects the timed out error of 500ms, with the correct, previous URL', done => {
             this.assertErrorReceived = (chunk, callCount) => {
                 if (callCount !== 3) return
                 assert.equal(this.options.url, chunk.url, 'URL is passed through')