Skip to content

Commit

Permalink
Handle redirects when blocking for ajax rquests. Enhance detection of…
Browse files Browse the repository at this point in the history 
… idle state for ajax.
  • Loading branch information
@hollingsworthd
hollingsworthd committed Mar 22, 2015
1 parent 4b25337 commit 0d7539c
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 28 deletions.
10 changes: 5 additions & 5 deletions browser/src/com/machinepublishers/jbrowserdriver/BrowserProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@ private static LinkedHashMap<String, String> defaultNavigator() {
return navigator;
}

//TODO handle shift key event in case of headless browser
private static String defaultSupplementaryJS() {
StringBuilder builder = new StringBuilder();
builder.append("Object.defineProperty(window, 'external', ");
Expand All @@ -176,10 +175,11 @@ private static String defaultSupplementaryJS() {
builder.append("{value:function(){return 'function IsSearchProviderInstalled() { [native code] }';}});");
builder.append("Object.defineProperty(window.external.addSearchEngine, 'toString', ");
builder.append("{value:function(){return 'function addSearchEngine() { [native code] }';}});");
builder.append("(function(){");
builder.append("var windowOpen = window.open;Object.defineProperty(window,'open',");
builder.append("{value:function(url, target, props){if(event && event.shiftKey){windowOpen(url);}else{windowOpen(url,'_self');}}}");
builder.append(")})();");
//TODO FIXME handle shift key event in case of headless browser
// builder.append("(function(){");
// builder.append("var windowOpen = window.open;Object.defineProperty(window,'open',");
// builder.append("{value:function(url, target, props){if(event && event.shiftKey){windowOpen(url);}else{windowOpen(url,'_self');}}}");
// builder.append(")})();");
return builder.toString();
}

Expand Down
43 changes: 32 additions & 11 deletions browser/src/com/machinepublishers/jbrowserdriver/DynamicAjaxListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,40 +22,43 @@
package com.machinepublishers.jbrowserdriver;

import java.lang.reflect.Method;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import com.sun.webkit.LoadListenerClient;

class DynamicAjaxListener implements Runnable {
private static final long WAIT_INTERVAL = 1000;
private static final long WAIT_INTERVAL = 100;
private static final int WAIT_COUNT = 5;
private static final long MAX_WAIT_DEFAULT = 15000;
private final int state;
private final AtomicInteger waitCount = new AtomicInteger();
private final Integer newStatusCode;
private final AtomicInteger statusCode;
private final Object statusMonitor;
private final Method clearStatusMonitor;
private final AtomicInteger resourceCount;
private final Set<String> resources;
private final AtomicBoolean superseded;
private final long timeoutMS;

DynamicAjaxListener(final int state, final int newStatusCode,
final AtomicInteger statusCode, final Object statusMonitor,
final Method clearStatusMonitor, final AtomicInteger resourceCount, final long timeoutMS) {
final Method clearStatusMonitor, final Set<String> resources, final long timeoutMS) {
this.state = state;
this.newStatusCode = newStatusCode;
this.statusCode = statusCode;
this.statusMonitor = statusMonitor;
this.clearStatusMonitor = clearStatusMonitor;
this.resourceCount = resourceCount;
this.resources = resources;
this.timeoutMS = timeoutMS <= 0 ? MAX_WAIT_DEFAULT : timeoutMS;
this.superseded = new AtomicBoolean();
}

DynamicAjaxListener(final AtomicInteger statusCode, final AtomicInteger resourceCount,
DynamicAjaxListener(final AtomicInteger statusCode, final Set<String> resources,
final AtomicBoolean superseded, final long timeoutMS) {
this.statusCode = statusCode;
this.resourceCount = resourceCount;
this.resources = resources;
this.superseded = superseded;
this.timeoutMS = timeoutMS <= 0 ? MAX_WAIT_DEFAULT : timeoutMS;
this.state = -1;
Expand All @@ -66,32 +69,50 @@ class DynamicAjaxListener implements Runnable {

@Override
public void run() {
if (Thread.interrupted()) {
if (superseded.get() || Thread.interrupted()) {
return;
}
if (newStatusCode == null || state == LoadListenerClient.PAGE_FINISHED) {
int totalWait = 0;
int size = 0;
boolean idle = false;
waitCount.set(0);
do {
try {
Thread.sleep(WAIT_INTERVAL);
} catch (InterruptedException e) {}
if (Thread.interrupted()) {
if (superseded.get() || Thread.interrupted()) {
return;
}
totalWait += WAIT_INTERVAL;
} while (resourceCount.get() > 0 && totalWait < timeoutMS);
synchronized (resources) {
size = resources.size();
}
if (size > 0) {
idle = false;
} else if (waitCount.get() == WAIT_COUNT) {
idle = true;
} else {
idle = false;
waitCount.incrementAndGet();
}
} while (!idle && totalWait < timeoutMS);
}
synchronized (statusCode) {
if (newStatusCode == null) {
synchronized (superseded) {
if (!superseded.get() && !Thread.interrupted()) {
resourceCount.set(0);
synchronized (resources) {
resources.clear();
}
statusCode.set(200);
statusCode.notifyAll();
}
}
} else {
resourceCount.set(0);
synchronized (resources) {
resources.clear();
}
statusCode.set(newStatusCode);
try {
clearStatusMonitor.invoke(statusMonitor);
Expand Down
51 changes: 39 additions & 12 deletions browser/src/com/machinepublishers/jbrowserdriver/DynamicHttpListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@

import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
Expand All @@ -33,7 +35,7 @@
class DynamicHttpListener implements LoadListenerClient {
private final List<Thread> threadsFromReset = new ArrayList<Thread>();
private final AtomicBoolean superseded = new AtomicBoolean();
private final AtomicInteger resourceCount = new AtomicInteger();
private final Set<String> resources = new HashSet<String>();
private final AtomicInteger statusCode;
private final long settingsId;
private final AtomicLong frame = new AtomicLong();
Expand All @@ -44,11 +46,13 @@ class DynamicHttpListener implements LoadListenerClient {
private static final Method startStatusMonitor;
private static final Method stopStatusMonitor;
private static final Method clearStatusMonitor;
private static final Method originalFromRedirect;
static {
Method getStatusMonitorTmp = null;
Method startStatusMonitorTmp = null;
Method stopStatusMonitorTmp = null;
Method clearStatusMonitorTmp = null;
Method originalFromRedirectTmp = null;
try {
Class statusMonitorClass = DynamicHttpListener.class.getClassLoader().
loadClass("com.machinepublishers.jbrowserdriver.StatusMonitor");
Expand All @@ -60,13 +64,16 @@ class DynamicHttpListener implements LoadListenerClient {
stopStatusMonitorTmp.setAccessible(true);
clearStatusMonitorTmp = statusMonitorClass.getDeclaredMethod("clearStatusMonitor");
clearStatusMonitorTmp.setAccessible(true);
originalFromRedirectTmp = statusMonitorClass.getDeclaredMethod("originalFromRedirect", String.class);
originalFromRedirectTmp.setAccessible(true);
} catch (Throwable t) {
t.printStackTrace();
}
getStatusMonitor = getStatusMonitorTmp;
startStatusMonitor = startStatusMonitorTmp;
stopStatusMonitor = stopStatusMonitorTmp;
clearStatusMonitor = clearStatusMonitorTmp;
originalFromRedirect = originalFromRedirectTmp;
}

DynamicHttpListener(AtomicInteger statusCode, AtomicLong timeoutMS, long settingsId) {
Expand Down Expand Up @@ -100,10 +107,23 @@ public void dispatchResourceLoadEvent(long frame, int state, String url,
String contentType, double progress, int errorCode) {
if (url.startsWith("http://") || url.startsWith("https://")) {
if (state == LoadListenerClient.RESOURCE_STARTED) {
resourceCount.incrementAndGet();
synchronized (resources) {
resources.add(frame + url);
}
} else if (state == LoadListenerClient.RESOURCE_FINISHED
|| state == LoadListenerClient.RESOURCE_FAILED) {
resourceCount.decrementAndGet();
String original = null;
try {
original = (String) originalFromRedirect.invoke(statusMonitor, url);
} catch (Throwable t) {
t.printStackTrace();
}
synchronized (resources) {
resources.remove(frame + url);
if (original != null) {
resources.remove(frame + original);
}
}
}
}
if (TRACE) {
Expand All @@ -120,10 +140,12 @@ public void resetStatusCode() {
synchronized (superseded) {
superseded.set(false);
statusCode.set(0);
resourceCount.set(0);
synchronized (resources) {
resources.clear();
}
}
Thread thread = new Thread(new DynamicAjaxListener(
statusCode, resourceCount, superseded, timeoutMS.get()));
statusCode, resources, superseded, timeoutMS.get()));
threadsFromReset.add(thread);
thread.start();
}
Expand All @@ -135,23 +157,28 @@ public void dispatchLoadEvent(long frame, final int state, String url,
try {
this.frame.compareAndSet(0l, frame);
if (this.frame.get() == frame
&& statusCode.get() == 0
&& (url.startsWith("http://") || url.startsWith("https://"))) {
if (state == LoadListenerClient.PAGE_STARTED
|| state == LoadListenerClient.PAGE_REDIRECTED) {
synchronized (superseded) {
statusCode.set(0);
superseded.set(true);
resourceCount.set(1);
synchronized (resources) {
resources.clear();
resources.add(frame + url);
}
}
startStatusMonitor.invoke(statusMonitor, url);
} else if (state == LoadListenerClient.PAGE_FINISHED
|| state == LoadListenerClient.LOAD_FAILED
|| state == LoadListenerClient.LOAD_STOPPED) {
} else if (statusCode.get() == 0
&& (state == LoadListenerClient.PAGE_FINISHED
|| state == LoadListenerClient.LOAD_FAILED)) {
final int code = (Integer) stopStatusMonitor.invoke(statusMonitor, url);
final int newStatusCode = state == LoadListenerClient.PAGE_FINISHED ? code : 499;
resourceCount.decrementAndGet();
synchronized (resources) {
resources.remove(frame + url);
}
new Thread(new DynamicAjaxListener(state, newStatusCode, statusCode,
statusMonitor, clearStatusMonitor, resourceCount, timeoutMS.get())).start();
statusMonitor, clearStatusMonitor, resources, timeoutMS.get())).start();
}
}
} catch (Throwable t) {
Expand Down
16 changes: 16 additions & 0 deletions browser/src/com/machinepublishers/jbrowserdriver/StatusMonitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class StatusMonitor {
private final Map<String, StreamConnection> connections = new HashMap<String, StreamConnection>();
private final Set<String> primaryDocuments = new HashSet<String>();
private final Set<String> discarded = new HashSet<String>();
private final Map<String, String> redirects = new HashMap<String, String>();
private boolean monitoring = false;

private StatusMonitor() {}
Expand Down Expand Up @@ -60,6 +61,20 @@ boolean isDiscarded(String url) {
}
}

void addRedirect(String original, String redirected) {
if (!original.equals(redirected)) {
synchronized (lock) {
redirects.put(redirected, original);
}
}
}

String originalFromRedirect(String redirected) {
synchronized (lock) {
return redirects.get(redirected);
}
}

void startStatusMonitor(String url) {
synchronized (lock) {
monitoring = true;
Expand Down Expand Up @@ -102,6 +117,7 @@ void clearStatusMonitor() {
connections.clear();
primaryDocuments.clear();
discarded.clear();
redirects.clear();
}
}
}
4 changes: 4 additions & 0 deletions browser/src/com/machinepublishers/jbrowserdriver/StreamConnection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ class StreamConnection extends HttpURLConnection {
private static final String pemFile = System.getProperty("jbd.pemfile");
private final HttpURLConnection conn;
private final boolean isSsl;
private final String originalUrl;
private final AtomicBoolean skip = new AtomicBoolean();
private static final Pattern downloadHeader = Pattern.compile(
"^\\s*attachment\\s*(?:;\\s*filename\\s*=\\s*[\"']?\\s*(.*?)\\s*[\"']?\\s*)?", Pattern.CASE_INSENSITIVE);
Expand Down Expand Up @@ -202,6 +203,7 @@ private static boolean isBlocked(String host) {
super(dummy);
this.conn = conn;
this.isSsl = true;
this.originalUrl = conn.getURL().toExternalForm();
SSLSocketFactory socketFactory = updatedSocketFactory();
if (socketFactory != null) {
conn.setSSLSocketFactory(socketFactory);
Expand All @@ -223,6 +225,7 @@ private static boolean isBlocked(String host) {
super(dummy);
this.conn = conn;
this.isSsl = false;
this.originalUrl = conn.getURL().toExternalForm();
connObjDelegate = conn;
}

Expand Down Expand Up @@ -269,6 +272,7 @@ public InputStream getInputStream() throws IOException {

@Override
public int getResponseCode() throws IOException {
StatusMonitor.get(settingsId.get()).addRedirect(originalUrl, conn.getURL().toExternalForm());
return skip.get() ? 204 : conn.getResponseCode();
}

Expand Down

0 comments on commit 0d7539c

Please sign in to comment.