diff --git a/local/README.md b/local/README.md new file mode 100644 index 0000000..a98ce38 --- /dev/null +++ b/local/README.md @@ -0,0 +1,11 @@ +# Local +This is where scraping scripts that can only be ran locally and not in a remote enviorment go. + +## Set up (OSX) +For these scripts to run sucessfully it requires an already running instance of chrome in debug mode, as well a tab logged into the target site to bypass auth/recaptcha. + + +Run the following command with no other chrome process running to get Chrome running in debug mode: + +```bash +/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222 \ No newline at end of file diff --git a/local/package-lock.json b/local/package-lock.json index c4bbea7..a33fe10 100644 --- a/local/package-lock.json +++ b/local/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "axios": "^1.6.8", - "puppeteer": "^22.4.0" + "puppeteer": "^23.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", @@ -1279,18 +1279,19 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.1.0.tgz", - "integrity": "sha512-xloWvocjvryHdUjDam/ZuGMh7zn4Sn3ZAaV4Ah2e2EwEt90N3XphZlSsU3n0VDc1F7kggCjMuH0UuxfPQ5mD9w==", - "dependencies": { - "debug": "4.3.4", - "extract-zip": "2.0.1", - "progress": "2.0.3", - "proxy-agent": "6.4.0", - "semver": "7.6.0", - "tar-fs": "3.0.5", - "unbzip2-stream": "1.4.3", - "yargs": "17.7.2" + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.3.1.tgz", + "integrity": "sha512-uK7o3hHkK+naEobMSJ+2ySYyXtQkBxIH8Gn4MK9ciePjNV+Pf+PgY/W7iPzn2MTjl3stcYB5AlcTmPYw7AXDwA==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.3.6", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.4.0", + "semver": "^7.6.3", + "tar-fs": "^3.0.6", + "unbzip2-stream": "^1.4.3", + "yargs": "^17.7.2" }, "bin": { "browsers": "lib/cjs/main-cli.js" @@ -1326,7 +1327,8 @@ "node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", - "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -1446,15 +1448,17 @@ "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", "optional": true, "dependencies": { "@types/node": "*" } }, "node_modules/agent-base": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", - "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "license": "MIT", "dependencies": { "debug": "^4.3.4" }, @@ -1518,6 +1522,7 @@ "version": "0.13.4", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", "dependencies": { "tslib": "^2.0.1" }, @@ -1556,7 +1561,8 @@ "node_modules/b4a": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", - "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==" + "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==", + "license": "Apache-2.0" }, "node_modules/babel-jest": { "version": "29.7.0", @@ -1751,38 +1757,51 @@ "dev": true }, "node_modules/bare-events": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.2.1.tgz", - "integrity": "sha512-9GYPpsPFvrWBkelIhOhTWtkeZxVxZOdb3VnFTCzlOo3OjvmTvzLoZFUT8kNFACx0vJej6QPney1Cf9BvzCNE/A==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", + "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", + "license": "Apache-2.0", "optional": true }, "node_modules/bare-fs": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.2.1.tgz", - "integrity": "sha512-+CjmZANQDFZWy4PGbVdmALIwmt33aJg8qTkVjClU6X4WmZkTPBDxRHiBn7fpqEWEfF3AC2io++erpViAIQbSjg==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", + "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", + "license": "Apache-2.0", "optional": true, "dependencies": { "bare-events": "^2.0.0", - "bare-os": "^2.0.0", "bare-path": "^2.0.0", - "streamx": "^2.13.0" + "bare-stream": "^2.0.0" } }, "node_modules/bare-os": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.2.0.tgz", - "integrity": "sha512-hD0rOPfYWOMpVirTACt4/nK8mC55La12K5fY1ij8HAdfQakD62M+H4o4tpfKzVGLgRDTuk3vjA4GqGXXCeFbag==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.0.tgz", + "integrity": "sha512-v8DTT08AS/G0F9xrhyLtepoo9EJBJ85FRSMbu1pQUlAf6A8T0tEEQGMVObWeqpjhSPXsE0VGlluFBJu2fdoTNg==", + "license": "Apache-2.0", "optional": true }, "node_modules/bare-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.0.tgz", - "integrity": "sha512-DIIg7ts8bdRKwJRJrUMy/PICEaQZaPGZ26lsSx9MJSwIhSrcdHn7/C8W+XmnG/rKi6BaRcz+JO00CjZteybDtw==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "license": "Apache-2.0", "optional": true, "dependencies": { "bare-os": "^2.1.0" } }, + "node_modules/bare-stream": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.3.tgz", + "integrity": "sha512-tiDAH9H/kP+tvNO5sczyn9ZAA7utrSMobyDchsnyyXBuUe2FSQWbxhtuHB8jwpHYYevVo2UJpcmvvjrbHboUUQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.18.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1800,12 +1819,14 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/basic-ftp": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "license": "MIT", "engines": { "node": ">=10.0.0" } @@ -1903,6 +1924,7 @@ "url": "https://feross.org/support" } ], + "license": "MIT", "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -1912,6 +1934,7 @@ "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", "engines": { "node": "*" } @@ -1982,12 +2005,14 @@ } }, "node_modules/chromium-bidi": { - "version": "0.5.12", - "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.5.12.tgz", - "integrity": "sha512-sZMgEBWKbupD0Q7lyFu8AWkrE+rs5ycE12jFkGwIgD/VS8lDPtelPlXM7LYaq4zrkZ/O2L3f4afHUHL0ICdKog==", + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.6.4.tgz", + "integrity": "sha512-8zoq6ogmhQQkAKZVKO2ObFTl4uOkqoX1PlKQX3hZQ5E9cbUotcAb7h4pTNVAGGv8Z36PF3CtdOriEp/Rz82JqQ==", + "license": "Apache-2.0", "dependencies": { "mitt": "3.0.1", - "urlpattern-polyfill": "10.0.0" + "urlpattern-polyfill": "10.0.0", + "zod": "3.23.8" }, "peerDependencies": { "devtools-protocol": "*" @@ -2195,14 +2220,6 @@ "node": ">=8" } }, - "node_modules/cross-fetch": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", - "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", - "dependencies": { - "node-fetch": "^2.6.12" - } - }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2221,14 +2238,16 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", "engines": { "node": ">= 14" } }, "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", + "license": "MIT", "dependencies": { "ms": "2.1.2" }, @@ -2268,6 +2287,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -2295,9 +2315,10 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1249869", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1249869.tgz", - "integrity": "sha512-Ctp4hInA0BEavlUoRy9mhGq0i+JSo/AwVyX2EFgZmV1kYB+Zq+EMBAn52QWu6FbRr10hRb6pBl420upbp4++vg==" + "version": "0.0.1330662", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1330662.tgz", + "integrity": "sha512-pzh6YQ8zZfz3iKlCvgzVCu22NdpZ8hNmwU6WnQjNVquh0A9iVosPtNLWDwaWVGyrntQlltPFztTMK5Cg6lfCuw==", + "license": "BSD-3-Clause" }, "node_modules/diff-sequences": { "version": "29.6.3", @@ -2335,6 +2356,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", "dependencies": { "once": "^1.4.0" } @@ -2375,6 +2397,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", "dependencies": { "esprima": "^4.0.1", "estraverse": "^5.2.0", @@ -2407,6 +2430,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", "engines": { "node": ">=4.0" } @@ -2415,6 +2439,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" } @@ -2483,6 +2508,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -2507,7 +2533,8 @@ "node_modules/fast-fifo": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", - "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", @@ -2528,6 +2555,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", "dependencies": { "pend": "~1.2.0" } @@ -2593,6 +2621,7 @@ "version": "11.2.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "license": "MIT", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -2661,6 +2690,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", "dependencies": { "pump": "^3.0.0" }, @@ -2675,6 +2705,7 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "license": "MIT", "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -2749,6 +2780,7 @@ "version": "7.0.2", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -2758,9 +2790,10 @@ } }, "node_modules/https-proxy-agent": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", - "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "license": "MIT", "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -2795,7 +2828,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "BSD-3-Clause" }, "node_modules/import-fresh": { "version": "3.3.0", @@ -2860,6 +2894,7 @@ "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "license": "MIT", "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -4602,7 +4637,8 @@ "node_modules/jsbn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", - "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "license": "MIT" }, "node_modules/jsesc": { "version": "2.5.2", @@ -4637,6 +4673,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "license": "MIT", "dependencies": { "universalify": "^2.0.0" }, @@ -4689,6 +4726,7 @@ "version": "7.18.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", "engines": { "node": ">=12" } @@ -4785,7 +4823,8 @@ "node_modules/mitt": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", - "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" }, "node_modules/ms": { "version": "2.1.2", @@ -4802,29 +4841,11 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", "engines": { "node": ">= 0.4.0" } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -4933,18 +4954,19 @@ } }, "node_modules/pac-proxy-agent": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz", - "integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", + "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", + "license": "MIT", "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.0.2", "debug": "^4.3.4", "get-uri": "^6.0.1", "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.2", - "pac-resolver": "^7.0.0", - "socks-proxy-agent": "^8.0.2" + "https-proxy-agent": "^7.0.5", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.4" }, "engines": { "node": ">= 14" @@ -4954,6 +4976,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -5026,7 +5049,8 @@ "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" }, "node_modules/picocolors": { "version": "1.0.0", @@ -5096,6 +5120,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", "engines": { "node": ">=0.4.0" } @@ -5117,6 +5142,7 @@ "version": "6.4.0", "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "license": "MIT", "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -5140,39 +5166,45 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "node_modules/puppeteer": { - "version": "22.4.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.4.0.tgz", - "integrity": "sha512-tR+JsDbA2qD1DqRX4F9k9SxQhk6UzcaCN+Qux7+WrDceS7wcR7tlFmMNB8+g8zE4Fmr/iRTOtf5wNnTW9cGUFQ==", + "version": "23.2.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-23.2.0.tgz", + "integrity": "sha512-MP7kLOdCfx1BJaGN5sgRo5fTYwAyGrlwWtrNphjKcwv/HO91+m90gbbwpRHbGl0rCvrmylq6vljn+zrjukniVg==", "hasInstallScript": true, + "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.1.0", - "cosmiconfig": "9.0.0", - "puppeteer-core": "22.4.0" + "@puppeteer/browsers": "2.3.1", + "chromium-bidi": "0.6.4", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1330662", + "puppeteer-core": "23.2.0", + "typed-query-selector": "^2.12.0" }, "bin": { - "puppeteer": "lib/esm/puppeteer/node/cli.js" + "puppeteer": "lib/cjs/puppeteer/node/cli.js" }, "engines": { "node": ">=18" } }, "node_modules/puppeteer-core": { - "version": "22.4.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.4.0.tgz", - "integrity": "sha512-MZttAbttrxi6O/B//rY6zQihjFe/vXeCLb5YvKH2xG6yrcVESo0Hc5/Cv49omwZyZzAJ1BK8BnDeatDsj+3hMw==", + "version": "23.2.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.2.0.tgz", + "integrity": "sha512-OFyPp2oolGSesx6ZrpmorE5tCaCKY1Z5e/h8f6sB0NpiezenB72jdWBdOrvBO/bUXyq14XyGJsDRUsv0ZOPdZA==", + "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.1.0", - "chromium-bidi": "0.5.12", - "cross-fetch": "4.0.0", - "debug": "4.3.4", - "devtools-protocol": "0.0.1249869", - "ws": "8.16.0" + "@puppeteer/browsers": "2.3.1", + "chromium-bidi": "0.6.4", + "debug": "^4.3.6", + "devtools-protocol": "0.0.1330662", + "typed-query-selector": "^2.12.0", + "ws": "^8.18.0" }, "engines": { "node": ">=18" @@ -5197,7 +5229,8 @@ "node_modules/queue-tick": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", - "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "license": "MIT" }, "node_modules/react-is": { "version": "18.2.0", @@ -5269,12 +5302,10 @@ } }, "node_modules/semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -5282,17 +5313,6 @@ "node": ">=10" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5339,15 +5359,17 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" } }, "node_modules/socks": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.1.tgz", - "integrity": "sha512-B6w7tkwNid7ToxjZ08rQMT8M9BJAf8DKx8Ft4NivzH0zBUfd6jldGcisJn/RLgxcX3FPNDdNQCUEMMT79b+oCQ==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "license": "MIT", "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -5358,13 +5380,14 @@ } }, "node_modules/socks-proxy-agent": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz", - "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==", + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "license": "MIT", "dependencies": { - "agent-base": "^7.0.2", + "agent-base": "^7.1.1", "debug": "^4.3.4", - "socks": "^2.7.1" + "socks": "^2.8.3" }, "engines": { "node": ">= 14" @@ -5392,7 +5415,8 @@ "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" }, "node_modules/stack-utils": { "version": "2.0.6", @@ -5416,12 +5440,14 @@ } }, "node_modules/streamx": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.16.1.tgz", - "integrity": "sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==", + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.19.0.tgz", + "integrity": "sha512-5z6CNR4gtkPbwlxyEqoDGDmWIzoNJqCBt4Eac1ICP9YaIT08ct712cFj0u1rx4F8luAuL+3Qc+RFIdI4OX00kg==", + "license": "MIT", "dependencies": { - "fast-fifo": "^1.1.0", - "queue-tick": "^1.0.1" + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" }, "optionalDependencies": { "bare-events": "^2.2.0" @@ -5518,9 +5544,10 @@ } }, "node_modules/tar-fs": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.5.tgz", - "integrity": "sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "license": "MIT", "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -5534,6 +5561,7 @@ "version": "3.1.7", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -5554,10 +5582,20 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz", + "integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" }, "node_modules/tmpl": { "version": "1.0.5", @@ -5586,11 +5624,6 @@ "node": ">=8.0" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/ts-jest": { "version": "29.1.2", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.2.tgz", @@ -5635,9 +5668,10 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", + "license": "0BSD" }, "node_modules/type-detect": { "version": "4.0.8", @@ -5660,6 +5694,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", @@ -5677,6 +5717,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "license": "MIT", "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -5692,6 +5733,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", "engines": { "node": ">= 10.0.0" } @@ -5729,7 +5771,8 @@ "node_modules/urlpattern-polyfill": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", - "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==" + "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==", + "license": "MIT" }, "node_modules/v8-to-istanbul": { "version": "9.2.0", @@ -5754,20 +5797,6 @@ "makeerror": "1.0.12" } }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5848,9 +5877,10 @@ } }, "node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "license": "MIT", "engines": { "node": ">=10.0.0" }, @@ -5875,11 +5905,6 @@ "node": ">=10" } }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", @@ -5909,6 +5934,7 @@ "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", "dependencies": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" @@ -5925,6 +5951,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/local/package.json b/local/package.json index b88a191..bdbcd01 100644 --- a/local/package.json +++ b/local/package.json @@ -12,6 +12,6 @@ }, "dependencies": { "axios": "^1.6.8", - "puppeteer": "^22.4.0" + "puppeteer": "^23.2.0" } } diff --git a/local/src/index.ts b/local/src/index.ts index 59977c1..1a56e01 100644 --- a/local/src/index.ts +++ b/local/src/index.ts @@ -6,6 +6,10 @@ const main = async () =>{ let jobCount = 0; const link = process.env.LOCAL_SCRAPER_SITEA; const jobApiEndpoint = process.env.JOB_API_ENDPOINT; + + // Chrome needs to be launched with remote debugging port enabled + const browserURL = "http://localhost:9222" + if (!link) { console.error('LOCAL_SCRAPER_SITEA environment variable isnt set'); return; @@ -14,11 +18,15 @@ const main = async () =>{ console.error('JOB_API_ENDPOINT environment variable isnt set'); return; } - const browser = await puppeteer.launch({ headless: false }); + const connectOptions = { + browserURL: browserURL, + protocolTimeout: 60000, // 60 seconds + } + const browser = await puppeteer.connect(connectOptions); const jobService = new JobService(jobApiEndpoint); await scrap(browser, link, jobCount, jobService); console.log('Scraping complete'); - await browser.close(); + await browser.disconnect(); } main().catch(error => { diff --git a/local/src/scraper/sitea.ts b/local/src/scraper/sitea.ts index 1c5ad55..31f88a9 100644 --- a/local/src/scraper/sitea.ts +++ b/local/src/scraper/sitea.ts @@ -25,9 +25,22 @@ const isUrlADup = async (page: Page, counter: number = 0): Promise => { export const scrap = async(browser : Browser, link : string, jobCount : number, jobService: JobService) => { let interestingJobs: Job[] = []; - try{ - const page = await browser.newPage(); - await page.goto(link); + let page: Page | undefined; + + try{ + const tabs = await browser.pages(); + for (const tab of tabs) { + if (tab.url().includes(link)) { + page = tab; + break; + } + } + if (!page) { + throw new Error('Page with matching link not found'); + } + + console.log(`Scraping started for ${link}`); + await page.waitForSelector('.slider_container'); while (true) { @@ -83,7 +96,7 @@ export const scrap = async(browser : Browser, link : string, jobCount : number, if (nextButton) { await Promise.all([ nextButton.click(), - page.waitForNavigation({ waitUntil: 'networkidle0' }), + page.waitForNavigation(), ]); } else { break; // Exit the loop if there is no "Next" button