Skip to content

Commit b54517a

Browse files
authored
Merge pull request #116 from the-markup/remove-unused-code
Remove unused code
2 parents 3e24bc2 + 5ce62a2 commit b54517a

22 files changed

+279
-360
lines changed

__tests__/canvas-fingerprinting.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import { join } from "path";
22
import {
33
sortCanvasCalls,
4-
getCanvasFp,
5-
getCanvasFontFp,
4+
getCanvasFingerprinters,
5+
getCanvasFontFingerprinters,
66
} from "../src/canvas-fingerprinting";
7-
import { loadEventData } from "../src/utils";
7+
import { loadEventData } from "../src/helpers/utils";
88
import { BlacklightEvent } from "../src/types";
99

1010
const TEST_DATA_DIR = join(__dirname, "test-data", "canvas-fingerprinting");
@@ -33,7 +33,7 @@ it("can sort a list of calls from the ndjson generated by the collector", async
3333
});
3434

3535
it("can identify canvas fp based on CITP criteria", async () => {
36-
const { fingerprinters } = getCanvasFp(jsCalls);
36+
const { fingerprinters } = getCanvasFingerprinters(jsCalls);
3737
expect(fingerprinters).toContain(
3838
"http://localhost:8125/canvas-fingerprinting.html"
3939
);
@@ -46,7 +46,7 @@ it("can identify font fingerprinting", async () => {
4646
(m) => m.message.type && m.message.type.indexOf("JsInstrument") > -1
4747
)
4848
.map((m) => m.message);
49-
const { canvas_font, text_measure } = getCanvasFontFp(jsCalls);
49+
const { canvas_font, text_measure } = getCanvasFontFingerprinters(jsCalls);
5050
expect(
5151
canvas_font["https://fc.kohls.com/2.2/w/w-756138/sync/js/"]
5252
).toHaveLength(69);

__tests__/cookies.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@ import puppeteer, { Page } from "puppeteer";
33
import { defaultPuppeteerBrowserOptions } from "../src/pptr-utils/default";
44
import { reportCookieEvents } from "../src/parser";
55
import { setupBlacklightInspector } from "../src/inspectors/inspector";
6-
import { loadEventData } from "../src/utils";
6+
import { loadEventData } from "../src/helpers/utils";
77
import { join } from "path";
88
import {
99
captureBrowserCookies,
1010
setupHttpCookieCapture,
1111
getJsCookies,
1212
matchCookiesToEvents,
13-
} from "../src/cookie-collector";
13+
} from "../src/inspectors/cookies";
1414
import { existsSync } from "fs";
15-
import { getLogger } from "../src/logger";
15+
import { getLogger } from "../src/helpers/logger";
1616
jest.setTimeout(20000);
1717

1818
const PP_TEST_RESULT = [

__tests__/fingerprinting.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import puppeteer, { Browser } from "puppeteer";
22
import { Global, JsInstrumentEvent } from "../src/types";
33
import { defaultPuppeteerBrowserOptions } from "../src/pptr-utils/default";
44
import { setupBlacklightInspector } from "../src/inspectors/inspector";
5-
import { getScriptUrl } from "../src/utils";
5+
import { getScriptUrl } from "../src/helpers/utils";
66
declare var global: Global;
77
let browser = {} as Browser;
88

__tests__/parser.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { join } from "path";
22
import { generateReport } from "../src/parser";
3-
import { loadEventData } from "../src/utils";
3+
import { loadEventData } from "../src/helpers/utils";
44
import puppeteer from "puppeteer";
55
import { defaultPuppeteerBrowserOptions } from "../src/pptr-utils/default";
66
import { Global } from "../src/types";

__tests__/pptr-utils.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { dedupLinks, getSocialLinks } from '../src/pptr-utils/get-links';
2-
import { SOCIAL_URLS } from '../src/pptr-utils/default';
2+
import { SOCIAL_URLS } from '../src/helpers/statics';
33
import { LinkObject } from '../src/types';
44

55
describe('get-links', () => {

__tests__/stack.ts

+1-37
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { defaultPuppeteerBrowserOptions } from "../src/pptr-utils/default";
44
import { Global, BlacklightEvent } from "../src/types";
55
import { setupBlacklightInspector } from "../src/inspectors/inspector";
66
import { getDomain } from "tldts";
7-
import { getStackType, getScriptUrl } from "../src/utils";
7+
import { getScriptUrl } from "../src/helpers/utils";
88
declare var global: Global;
99

1010
const JS_STACK_TEST_SCRIPT_URL = `${global.__DEV_SERVER__}/stack.js`;
@@ -45,39 +45,3 @@ it("can correctly parse the stack trace", async () => {
4545
});
4646
expect(output.sort()).toEqual(JS_STACK_CALLS.sort());
4747
});
48-
// it("can go through the stacktrace and find and see if there were third-party calls", async () => {
49-
// const browser = await launch(defaultPuppeteerBrowserOptions);
50-
// const page = (await browser.pages())[0];
51-
// const rows = [];
52-
// await setupBlacklightInspector(page, e => rows.push(e));
53-
// // await page.goto(`${global.__DEV_SERVER__}/simple.html`);
54-
// const url =JS_STACK_TEST_URL;
55-
// const domain = getDomain(url);
56-
// await page.goto(url);
57-
// await browser.close();
58-
// console.time();
59-
// rows.forEach(element => {
60-
// // if (!element.data.symbol.includes("addEvent")) return;
61-
// const stackType = getStackType(element.stack, domain);
62-
// console.log(stackType);
63-
// console.log(element.stack);
64-
// // let hasFirstParty = false;
65-
// // let hasThirdParty = false;
66-
// // element.stack.forEach(s => {
67-
// // if (s.hasOwnProperty("fileName")) {
68-
// // const scriptDomain = getDomain(s.fileName);
69-
// // if (scriptDomain === domain) {
70-
// // hasFirstParty = true;
71-
// // } else {
72-
// // hasThirdParty = true;
73-
// // }
74-
// // }
75-
// // });
76-
// // if (hasFirstParty && hasThirdParty) {
77-
// // console.log(element.stack);
78-
// // }
79-
// // });
80-
// console.timeEnd();
81-
// // console.log([].concat(rows.map(m => m.stack)));
82-
// // flatMap(rows.map(m => m.stach));
83-
// });

src/canvas-fingerprinting.ts

+35-43
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,24 @@
1-
import { BlacklightEvent, JsInstrumentEvent } from './types';
2-
31
/**
42
* @fileOverview Utility functions for canvas finerprinting analysis.
53
* Implemented following the Princeton study's methodology.
64
*/
7-
8-
import { getScriptUrl, serializeCanvasCallMap } from './utils';
5+
import { BlacklightEvent, JsInstrumentEvent, CanvasCallMap } from './types';
6+
import { getScriptUrl, serializeCanvasCallMap } from './helpers/utils';
97

108
const MIN_CANVAS_IMAGE_WIDTH = 16;
119
const MIN_CANVAS_IMAGE_HEIGHT = 16;
1210
const MIN_FONT_LIST_SIZE = 50;
1311
const MIN_TEXT_MEASURE_COUNT = 50;
1412
const MIN_TEXT_LENGTH = 10;
13+
14+
const CANVAS_READ_FUNCS = ['HTMLCanvasElement.toDataURL', 'CanvasRenderingContext2D.getImageData'];
15+
const CANVAS_WRITE_FUNCS = ['CanvasRenderingContext2D.fillText', 'CanvasRenderingContext2D.strokeText'];
16+
const CANVAS_FP_DO_NOT_CALL_LIST = ['CanvasRenderingContext2D.save', 'CanvasRenderingContext2D.restore', 'HTMLCanvasElement.addEventListener'];
17+
const CANVAS_FONT = ['CanvasRenderingContext2D.measureText', 'CanvasRenderingContext2D.font'];
18+
1519
/**
1620
* Return the string that is written onto canvas from function arguments
17-
* @param arguments
21+
* @param args
1822
*/
1923
const getCanvasText = (args: string[]) => {
2024
if (!args || !args[0]) {
@@ -31,40 +35,34 @@ const getTextLength = (text: string) => {
3135
/**
3236
* Check if the retrieved pixel data is larger than min. dimensions.
3337
* {@link https://developer.mozilla.org/en-US/docsz1/Web/API/CanvasRenderingContext2D/getImageData#Parameters | Web API Image Data Parameters}
34-
* @param arguments
38+
* @param args
3539
*/
36-
const isGetImageDataDimsTooSmall = (args: string[]) => {
37-
const sw = parseInt(args[2], 10);
38-
const sh = parseInt(args[3], 10);
39-
return sw < MIN_CANVAS_IMAGE_WIDTH || sh < MIN_CANVAS_IMAGE_HEIGHT;
40+
const isImageTooSmall = (args: string[]) => {
41+
const width = parseInt(args[2], 10);
42+
const height = parseInt(args[3], 10);
43+
return width < MIN_CANVAS_IMAGE_WIDTH || height < MIN_CANVAS_IMAGE_HEIGHT;
4044
};
41-
type ScriptUrl = string;
42-
type CanvasCallValue = string;
43-
type CanvasCallMap = Map<ScriptUrl, Set<CanvasCallValue>>;
45+
4446
/**
45-
* This function takes a list of Javascript calls to HTML Canvas properties from a browsers window object.
47+
* This function takes a list of Javascript calls to HTML Canvas properties from a browser's window object.
4648
* It sorts the functions in order to evaluate which script are fingerprinting a browser using the criteria
4749
* described by Englehardt & Narayanan, 2016
4850
* We Filter for 4 Criteria
49-
* Criteria 1: The canvas element’s height and width properties must not be set below 16
50-
* Criteria 2: Text must be written to canvas with least two colors or at least 10 distinct charachters
51-
* Criteria 3: The script should not call the save, restore, or addEventListener methods of the rendering context.
52-
* Criteria 4: The script extracts an image withtoDataURL or with a single call togetImageData that specifies an area with a minimum size of 16px×16px
51+
* Criteria 1: The canvas element’s height and width properties must not be set below 16px
52+
* Criteria 2: Text must be written to canvas with least two colors or at least 10 distinct characters
53+
* Criteria 3: The script should not call the save, restore, or addEventListener methods of the rendering context.
54+
* Criteria 4: The script extracts an image with toDataURL or with a single call to getImageData that specifies an area with a minimum size of 16px×16px
5355
* @param canvasCalls
5456
* @see {@link http://randomwalker.info/publications/OpenWPM_1_million_site_tracking_measurement.pdf#page=12}
5557
*/
5658
export const sortCanvasCalls = (canvasCalls: BlacklightEvent[]) => {
57-
const CANVAS_READ_FUNCS = ['HTMLCanvasElement.toDataURL', 'CanvasRenderingContext2D.getImageData'];
58-
59-
const CANVAS_WRITE_FUNCS = ['CanvasRenderingContext2D.fillText', 'CanvasRenderingContext2D.strokeText'];
60-
const CANVAS_FP_DO_NOT_CALL_LIST = ['CanvasRenderingContext2D.save', 'CanvasRenderingContext2D.restore', 'HTMLCanvasElement.addEventListener'];
61-
62-
const cReads = new Map() as CanvasCallMap;
59+
const cReads = new Map() as CanvasCallMap;
6360
const cDataUrls = new Map() as CanvasCallMap;
64-
const cWrites = new Map() as CanvasCallMap;
65-
const cTexts = new Map() as CanvasCallMap;
66-
const cBanned = new Map() as CanvasCallMap;
67-
const cStyles = new Map() as CanvasCallMap;
61+
const cWrites = new Map() as CanvasCallMap;
62+
const cTexts = new Map() as CanvasCallMap;
63+
const cBanned = new Map() as CanvasCallMap;
64+
const cStyles = new Map() as CanvasCallMap;
65+
6866
for (const item of canvasCalls) {
6967
const { url, data } = item as JsInstrumentEvent;
7068
const url_host = new URL(url).hostname;
@@ -75,7 +73,7 @@ export const sortCanvasCalls = (canvasCalls: BlacklightEvent[]) => {
7573
}
7674

7775
if (CANVAS_READ_FUNCS.includes(symbol) && operation === 'call') {
78-
if (symbol === 'CanvasRenderingContext2D.getImageData' && isGetImageDataDimsTooSmall(data.arguments)) {
76+
if (symbol === 'CanvasRenderingContext2D.getImageData' && isImageTooSmall(data.arguments)) {
7977
continue;
8078
}
8179
if (symbol === 'HTMLCanvasElement.toDataURL') {
@@ -111,37 +109,31 @@ export const sortCanvasCalls = (canvasCalls: BlacklightEvent[]) => {
111109
* @see {@link sortCanvasCalls}
112110
* @param canvasCalls
113111
*/
114-
export const getCanvasFp = (
112+
export const getCanvasFingerprinters = (
115113
canvasCalls
116114
): {
117115
fingerprinters: string[];
118116
texts: any;
119117
styles: any;
120118
data_url: any;
121119
} => {
120+
const fingerprinters: Set<string> = new Set();
122121
const { cDataUrls, cReads, cWrites, cBanned, cTexts, cStyles } = sortCanvasCalls(canvasCalls);
123122

124-
const fingerprinters: Set<string> = new Set();
125123
for (const [script_url, url_hosts] of cReads.entries()) {
126-
if (fingerprinters.has(script_url)) {
127-
continue;
128-
}
124+
if (fingerprinters.has(script_url)) continue;
129125

130126
const rwIntersection = new Set([...url_hosts].filter(x => cWrites.has(script_url) && cWrites.get(script_url).has(x)));
131-
132-
if (rwIntersection.size < 1) {
133-
continue;
134-
}
127+
if (rwIntersection.size < 1) continue;
128+
135129
for (const canvasRwVisit of rwIntersection.values()) {
136130
if (cBanned.has(script_url) && cBanned.get(script_url).has(canvasRwVisit)) {
137-
// console.log(
138-
// `Ignoring script ${script_url} from url_host ${canvasRwVisit}`
139-
// );
140131
continue;
141132
}
142133
fingerprinters.add(script_url);
143134
}
144135
}
136+
145137
return {
146138
data_url: serializeCanvasCallMap(cDataUrls),
147139
fingerprinters: Array.from(fingerprinters),
@@ -150,12 +142,12 @@ export const getCanvasFp = (
150142
};
151143
};
152144

153-
export const getCanvasFontFp = jsCalls => {
154-
const CANVAS_FONT = ['CanvasRenderingContext2D.measureText', 'CanvasRenderingContext2D.font'];
145+
export const getCanvasFontFingerprinters = jsCalls => {
155146
const font_shorthand =
156147
/^\s*(?=(?:(?:[-a-z]+\s*){0,2}(italic|oblique))?)(?=(?:(?:[-a-z]+\s*){0,2}(small-caps))?)(?=(?:(?:[-a-z]+\s*){0,2}(bold(?:er)?|lighter|[1-9]00))?)(?:(?:normal|\1|\2|\3)\s*){0,3}((?:xx?-)?(?:small|large)|medium|smaller|larger|[.\d]+(?:\%|in|[cem]m|ex|p[ctx]))(?:\s*\/\s*(normal|[.\d]+(?:\%|in|[cem]m|ex|p[ctx])))?\s*([-_\{\}\(\)\&!\',\*\.\"\sa-zA-Z0-9]+?)\s*$/g;
157148
const textMeasure = new Map() as Map<string, any>;
158149
const canvasFont = new Map() as CanvasCallMap;
150+
159151
for (const item of jsCalls) {
160152
const script_url = getScriptUrl(item);
161153
const { symbol, value } = item.data;

src/collector.ts

+11-21
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ import { join } from 'path';
55
import puppeteer, { Browser, Page, PuppeteerLifeCycleEvent, KnownDevices, PuppeteerLaunchOptions } from 'puppeteer';
66
import PuppeteerHar from 'puppeteer-har';
77
import { getDomain, getSubdomain, parse } from 'tldts';
8-
import { captureBrowserCookies, clearCookiesCache, setupHttpCookieCapture } from './cookie-collector';
8+
import { captureBrowserCookies, clearCookiesCache, setupHttpCookieCapture } from './inspectors/cookies';
99

10-
import { getLogger } from './logger';
10+
import { getLogger } from './helpers/logger';
1111
import { generateReport } from './parser';
1212
import { defaultPuppeteerBrowserOptions, savePageContent } from './pptr-utils/default';
1313
import { dedupLinks, getLinks, getSocialLinks } from './pptr-utils/get-links';
@@ -16,7 +16,7 @@ import { setupBlacklightInspector } from './inspectors/inspector';
1616
import { setupKeyLoggingInspector } from './inspectors/key-logging';
1717
import { setupSessionRecordingInspector } from './inspectors/session-recording';
1818
import { setUpThirdPartyTrackersInspector } from './inspectors/third-party-trackers';
19-
import { clearDir, closeBrowser } from './utils';
19+
import { clearDir, closeBrowser } from './helpers/utils';
2020

2121
export type CollectorOptions = Partial<typeof DEFAULT_OPTIONS>;
2222

@@ -152,7 +152,7 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
152152
}
153153

154154
// record all requested hosts
155-
await page.on('request', request => {
155+
page.on('request', request => {
156156
const l = parse(request.url());
157157
// note that hosts may appear as first and third party depending on the path
158158
if (FIRST_PARTY.domain === l.domain) {
@@ -249,10 +249,9 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
249249
}
250250
}
251251
}
252+
252253
await fillForms(page);
253-
// console.log('... done with fillForms');
254254
await autoScroll(page);
255-
// console.log('... done with autoScroll');
256255

257256
let subDomainLinks = [];
258257
if (getSubdomain(output.uri_dest) !== 'www') {
@@ -266,7 +265,6 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
266265
output.browsing_history = [output.uri_dest].concat(browse_links.map(l => l.href));
267266
console.log('About to browse more links');
268267

269-
// try {
270268
for (const link of output.browsing_history.slice(1)) {
271269
logger.log('info', `browsing now to ${link}`, { type: 'Browser' });
272270
if (didBrowserDisconnect) {
@@ -292,13 +290,9 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
292290
pageIndex++;
293291
}
294292

295-
// console.log('saving cookies');
296293
await captureBrowserCookies(page, args.outDir);
297-
// console.log('... done saving cookies');
298294
if (args.captureHar) {
299-
// console.log('saving har');
300295
await har.stop();
301-
// console.log('... done saving har');
302296
}
303297

304298
await closeBrowser(browser);
@@ -321,8 +315,8 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
321315
}
322316
}
323317
}
318+
324319
// generate report
325-
// console.log('generating report');
326320
const fpRequests = Array.from(hosts.requests.first_party);
327321
const tpRequests = Array.from(hosts.requests.third_party);
328322
const incorrectTpAssignment = tpRequests.filter((f: string) => getDomain(f) === REDIRECTED_FIRST_PARTY.domain);
@@ -381,23 +375,19 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
381375
return acc;
382376
}, {});
383377

384-
// console.log('writing inspection.json');
385378
const json_dump = JSON.stringify({ ...output, reports }, null, 2);
386379
writeFileSync(join(args.outDir, 'inspection.json'), json_dump);
387380
if (args.outDir.includes('bl-tmp')) {
388381
clearDir(args.outDir, false);
389382
}
390-
return { status: 'success', ...output, reports };
383+
return {
384+
status: 'success',
385+
...output,
386+
reports,
387+
};
391388
} finally {
392-
// close browser and clear tmp dir
393389
if (browser && !didBrowserDisconnect) {
394390
await closeBrowser(browser);
395391
}
396-
// if (typeof userDataDir !== 'undefined') {
397-
// clearDir(userDataDir, false);
398-
// }
399-
// if (args.outDir.includes('bl-tmp')) {
400-
// clearDir(args.outDir, false);
401-
// }
402392
}
403393
};

src/logger.ts renamed to src/helpers/logger.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import path from 'path';
22
import { setGracefulCleanup, tmpNameSync } from 'tmp';
3-
setGracefulCleanup();
43

54
// https://stackoverflow.com/a/45211015/1407622
65
import { createLogger, format, transports } from 'winston';
76

7+
setGracefulCleanup();
8+
89
export const getLogger = ({ outDir = '', quiet = false }) => {
910
const log_transports = [];
1011
log_transports.push(

src/helpers/responses.ts

Whitespace-only changes.

0 commit comments

Comments
 (0)