doc: Use unstable_cache for dependents caching (#535)

* doc: Use unstable_cache for dependents caching

* chore: Error logging

* chore: Add logging

* chore: Parallelise crawling and add perf logs

Author: franky47

packages/docs/src/app/(pages)/_landing/bundle-size.tsx

@@ -10,6 +10,6 @@ export async function BundleSize() {
     return prettyBytes(size)
   } catch (error) {
     console.error(error)
-    return 'less than 4KB'
+    return 'less than 5KB'
   }
 }

packages/docs/src/app/(pages)/_landing/dependents/crawler.ts

@@ -1,4 +1,5 @@
 import * as cheerio from 'cheerio'
+import { unstable_cache } from 'next/cache'
 
 enum PackageId {
   nuqs = 'UGFja2FnZS00MjczNzAxNTA5',
@@ -12,45 +13,57 @@ export type Result = {
   avatarID: string
 }
 
-export async function crawlDependents() {
-  const allResults: Result[] = []
-  let url = `https://github.com/47ng/nuqs/network/dependents?package_id=${PackageId.nuqs}`
-  while (true) {
-    const { results, nextPage } = await crawlDependentsPage(url)
-    allResults.push(...results)
-    if (nextPage === null) {
-      break
-    }
-    url = nextPage
+export const crawlDependents = unstable_cache(
+  _crawlDependents,
+  ['crawlDependents'],
+  {
+    revalidate: 86_400
   }
-  url = `https://github.com/47ng/nuqs/network/dependents?package_id=${PackageId.nextUseQueryState}`
-  while (true) {
-    const { results, nextPage } = await crawlDependentsPage(url)
-    allResults.push(...results)
-    if (nextPage === null) {
-      break
-    }
-    url = nextPage
-  }
-  return allResults
+)
+
+async function _crawlDependents() {
+  const tick = performance.now()
+  const allResults: Result[] = []
+  await Promise.allSettled([
+    crawlPackageDependents(PackageId.nuqs, allResults),
+    crawlPackageDependents(PackageId.nextUseQueryState, allResults)
+  ])
+  const out = allResults
     .sort((a, b) => b.stars - a.stars)
     .filter(
       // remove duplicates by repo
       (result, index, self) =>
         index === self.findIndex(r => r.repo === result.repo)
     )
     .slice(0, 100)
+  console.log(`Dependents crawled in ${performance.now() - tick}ms`)
+  return out
+}
+
+async function crawlPackageDependents(pkgId: string, allResults: Result[]) {
+  let url = `https://github.com/47ng/nuqs/network/dependents?package_id=${pkgId}`
+  while (true) {
+    const { results, nextPage } = await crawlDependentsPage(url)
+    allResults.push(...results)
+    if (nextPage === null) {
+      return
+    }
+    url = nextPage
+  }
 }
 
 async function crawlDependentsPage(url: string) {
+  const tick = performance.now()
   const pkg =
     new URLSearchParams(url.split('?')[1]).get('package_id') === PackageId.nuqs
       ? 'nuqs'
       : 'next-usequerystate'
   const html = await fetch(url, {
     cache: 'no-store'
   }).then(res => res.text())
+  const endOfFetch = performance.now()
   const $ = cheerio.load(html)
+  const endOfParse = performance.now()
   const results: Result[] = []
   $('[data-test-id="dg-repo-pkg-dependent"]').each((index, element) => {
     const img = $(element).find('img').attr('src') // ?.replace('s=40', 's=64')
@@ -83,6 +96,13 @@ async function crawlDependentsPage(url: string) {
   })
   const nextButton = $('div.paginate-container a:contains(Next)')
   const nextPage = nextButton?.attr('href') ?? null
+  console.log(
+    'Crawled page %s (fetch: %s, parse: %s, extract: %s)',
+    url,
+    (endOfFetch - tick).toFixed(2),
+    (endOfParse - endOfFetch).toFixed(2),
+    (performance.now() - endOfParse).toFixed(2)
+  )
   return { results, nextPage }
 }