diff --git a/package-lock.json b/package-lock.json index d140551..8c39d44 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,6 @@ "license": "MIT", "dependencies": { "@notionhq/client": "2.2.3", - "axios": "^1.6.8", "chalk": "^4.1.2", "commander": "^9.2.0", "cosmiconfig": "^8.0.0", @@ -1510,29 +1509,6 @@ "node": ">= 4.0.0" } }, - "node_modules/axios": { - "version": "1.6.8", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", - "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "node_modules/axios/node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/babel-code-frame": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", @@ -4688,25 +4664,6 @@ "dev": true, "license": "ISC" }, - "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -10391,11 +10348,6 @@ "dev": true, "license": "ISC" }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", diff --git a/package.json b/package.json index 4136d52..6057622 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "//note: ts-node": "really is a runtime dependency", "dependencies": { "@notionhq/client": "2.2.3", - "axios": "^1.6.8", "chalk": "^4.1.2", "commander": "^9.2.0", "cosmiconfig": "^8.0.0", diff --git a/src/images.ts b/src/images.ts index aa42b8a..0b51e7e 100644 --- a/src/images.ts +++ b/src/images.ts @@ -1,6 +1,5 @@ import * as fs from "fs-extra"; import FileType, { FileTypeResult } from "file-type"; -import axios from "axios"; import { makeImagePersistencePlan } from "./MakeImagePersistencePlan"; import { warning, logDebug, verbose, info } from "./log"; import { ListBlockChildrenResponseResult } from "notion-to-md/build/types"; @@ -150,10 +149,15 @@ async function processImageBlock( async function readPrimaryImage(imageSet: ImageSet) { // In Mar 2024, we started having a problem getting a particular gif from imgur using // node-fetch. Switching to axios resolved it. I don't know why. - const response = await axios.get(imageSet.primaryUrl, { - responseType: "arraybuffer", - }); - imageSet.primaryBuffer = Buffer.from(response.data, "utf-8"); + // Then, in Apr 2025, we started getting 429 responses from imgur through axios, + // so we switched to node's built-in fetch (different than the node-fetch package). + // Just a guess, but probably imgur keeps locking down what it suspects as code running + // to scrape images. + // Apparently, imgur is getting to be more and more of a liability, + // so we should probably stop using it. + const response = await fetch(imageSet.primaryUrl); + const arrayBuffer = await response.arrayBuffer(); + imageSet.primaryBuffer = Buffer.from(arrayBuffer); imageSet.fileType = await FileType.fromBuffer(imageSet.primaryBuffer); }