fix: add retry mechanism to fetch.js to handle rate limiting

- Implemented a retry mechanism with exponential backoff for both GitHub and Medium API requests to handle HTTP 429 (Too Many Requests) errors.
- Improved error handling to ensure more graceful recovery from transient errors.
- Added logging for better visibility into retry attempts and errors.

Author: AmrElsayyad

fetch.js

@@ -18,6 +18,48 @@ const ERR = {
     "The request to Medium didn't succeed. Check if Medium username in your .env file is correct."
 };
 
+async function fetchWithRetry(options, data, retries = 5, delay = 5000) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await new Promise((resolve, reject) => {
+        const req = https.request(options, res => {
+          let responseData = "";
+
+          res.on("data", d => {
+            responseData += d;
+          });
+
+          res.on("end", () => {
+            if (res.statusCode === 200) {
+              resolve(responseData);
+            } else if (res.statusCode === 429 && i < retries - 1) {
+              console.log(`Rate limited. Retrying in ${delay}ms...`);
+              setTimeout(() => resolve(fetchWithRetry(options, data, retries - 1, delay)), delay);
+            } else {
+              reject(new Error(`${options.path} request failed with status code: ${res.statusCode}`));
+            }
+          });
+        });
+
+        req.on("error", error => {
+          reject(error);
+        });
+
+        if (data) {
+          req.write(data);
+        }
+        req.end();
+      });
+    } catch (error) {
+      if (i === retries - 1) {
+        throw error;
+      }
+      console.log(`Error fetching data. Retrying in ${delay}ms...`);
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+}
+
 if (USE_GITHUB_DATA === "true") {
   if (GITHUB_USERNAME === undefined) {
     throw new Error(ERR.noUserName);
@@ -69,31 +111,16 @@ if (USE_GITHUB_DATA === "true") {
     }
   };
 
-  const req = https.request(default_options, res => {
-    let data = "";
-
-    console.log(`statusCode: ${res.statusCode}`);
-    if (res.statusCode !== 200) {
-      throw new Error(ERR.requestFailed);
-    }
-
-    res.on("data", d => {
-      data += d;
-    });
-    res.on("end", () => {
-      fs.writeFile("./public/profile.json", data, function (err) {
+  fetchWithRetry(default_options, data)
+    .then(responseData => {
+      fs.writeFile("./public/profile.json", responseData, function (err) {
         if (err) return console.log(err);
         console.log("saved file to public/profile.json");
       });
+    })
+    .catch(error => {
+      console.error("Error:", error);
     });
-  });
-
-  req.on("error", error => {
-    throw error;
-  });
-
-  req.write(data);
-  req.end();
 }
 
 if (MEDIUM_USERNAME !== undefined) {
@@ -106,29 +133,22 @@ if (MEDIUM_USERNAME !== undefined) {
     method: "GET"
   };
 
-  const req = https.request(options, res => {
-    console.log(`statusCode: ${res.statusCode}`);
-    if (res.statusCode !== 200) {
-      throw new Error(ERR.requestMediumFailed);
-    }
-
-    feed2json.fromStream(res, url, {}, (err, json) => {
-      if (err) {
-        console.error("Error converting feed to JSON:", err);
-        return;
-      }
+  fetchWithRetry(options)
+    .then(responseData => {
+      feed2json.fromString(responseData, url, {}, (err, json) => {
+        if (err) {
+          console.error("Error converting feed to JSON:", err);
+          return;
+        }
 
-      const mediumData = JSON.stringify(json, null, 2);
-      fs.writeFile("./public/blogs.json", mediumData, function (err) {
-        if (err) return console.error(err);
-        console.log("Saved file to public/blogs.json");
+        const mediumData = JSON.stringify(json, null, 2);
+        fs.writeFile("./public/blogs.json", mediumData, function (err) {
+          if (err) return console.error(err);
+          console.log("Saved file to public/blogs.json");
+        });
       });
+    })
+    .catch(error => {
+      console.error("Error:", error);
     });
-  });
-
-  req.on("error", error => {
-    throw error;
-  });
-
-  req.end();
-}
+}