refactor: move to cron based caching strategy

.env.test

@@ -7,5 +7,5 @@ HOST=localhost
 LOG_LEVEL=info
 APP_KEY=25St7k_qqnrC1z5tKGuhiUG0H1oH8A7q
 SESSION_DRIVER=memory
-
+DB_CONNECTION=test
 GITHUB_TOKEN=xxx

adonisrc.ts

@@ -12,8 +12,8 @@ export default defineConfig({
   */
   commands: [
     () => import('@adonisjs/core/commands'),
-    () => import('@adonisjs/cache/commands'),
     () => import('@adonisjs/lucid/commands'),
+    () => import('@adonisjs/cache/commands'),
   ],
 
   /*
@@ -35,12 +35,12 @@ export default defineConfig({
     () => import('@adonisjs/core/providers/vinejs_provider'),
     () => import('@adonisjs/core/providers/edge_provider'),
     () => import('@adonisjs/session/session_provider'),
-    () => import('@adonisjs/vite/vite_provider'),
     () => import('@adonisjs/shield/shield_provider'),
+    () => import('@adonisjs/vite/vite_provider'),
     () => import('@adonisjs/static/static_provider'),
-    () => import('@adonisjs/cache/cache_provider'),
-    () => import('@adonisjs/inertia/inertia_provider'),
     () => import('@adonisjs/lucid/database_provider'),
+    () => import('@adonisjs/inertia/inertia_provider'),
+    () => import('@adonisjs/cache/cache_provider'),
     () => import('./providers/app_provider.js'),
   ],
 
@@ -52,7 +52,14 @@ export default defineConfig({
   | List of modules to import before starting the application.
   |
   */
-  preloads: [() => import('#start/routes'), () => import('#start/kernel')],
+  preloads: [
+    () => import('#start/routes'),
+    () => import('#start/kernel'),
+    {
+      file: () => import('#start/scheduler'),
+      environment: ['web'],
+    },
+  ],
 
   /*
   |--------------------------------------------------------------------------

app/models/package_stats.ts

@@ -0,0 +1,15 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column } from '@adonisjs/lucid/orm'
+
+export default class PackageStats extends BaseModel {
+  @column({ isPrimary: true }) declare id: number
+
+  @column() declare packageName: string
+  @column() declare weeklyDownloads: number
+  @column() declare githubStars: number
+  @column.dateTime() declare firstReleaseAt: DateTime | null
+  @column.dateTime() declare lastReleaseAt: DateTime | null
+
+  @column.dateTime({ autoCreate: true }) declare createdAt: DateTime
+  @column.dateTime({ autoCreate: true, autoUpdate: true }) declare updatedAt: DateTime
+}

app/services/packages_data_refresher.ts

@@ -0,0 +1,95 @@
+import pLimit from 'p-limit'
+import { DateTime } from 'luxon'
+import logger from '@adonisjs/core/services/logger'
+
+import type { PackageInfo } from '#types/main'
+import PackageStats from '#models/package_stats'
+import type { PackageFetcher } from './package_fetcher.js'
+
+export class PackagesDataRefresher {
+  constructor(
+    protected packageFetcher: PackageFetcher,
+    protected packagesList: PackageInfo[],
+  ) {}
+
+  /**
+   * Get the first and last release dates from cache or fetch it from npm
+   */
+  async #getReleasesDates(pkg: PackageInfo) {
+    if (!pkg.npm) return { firstReleaseAt: '', lastReleaseAt: '' }
+
+    return this.packageFetcher.fetchReleaseDates(pkg.npm!).catch((err) => {
+      logger.error({ err }, `Cannot fetch releases dates for ${pkg.npm}`)
+      return { firstReleaseAt: '', lastReleaseAt: '' }
+    })
+  }
+
+  /**
+   * Get the package downloads from cache or fetch it from npm
+   */
+  async #getPackageDownloads(pkg: PackageInfo) {
+    if (!pkg.npm) return { downloads: 0 }
+
+    return this.packageFetcher.fetchPackageDownloads(pkg.npm!).catch((err) => {
+      logger.error({ err }, `Cannot fetch npm info for ${pkg.npm}`)
+      return { downloads: 0 }
+    })
+  }
+
+  /**
+   * Get the github stars from cache or fetch it from github
+   */
+  async #getGithubStars(pkg: PackageInfo) {
+    if (!pkg.repo) return { stars: 0 }
+
+    return this.packageFetcher.fetchGithubStars(pkg.repo).catch((err) => {
+      logger.error({ err }, `Cannot fetch github repo info for ${pkg.repo}`)
+      return { stars: 0 }
+    })
+  }
+
+  /**
+   * Get stats about a single package
+   */
+  async #fetchPackageStats(pkg: PackageInfo) {
+    logger.debug(`Fetching stats for ${pkg.name}`)
+
+    const [npmStats, ghStats, releases] = await Promise.all([
+      this.#getPackageDownloads(pkg),
+      this.#getGithubStars(pkg),
+      this.#getReleasesDates(pkg),
+    ])
+
+    pkg.downloads = npmStats.downloads
+    pkg.stars = ghStats.stars
+    pkg.firstReleaseAt = releases.firstReleaseAt
+    pkg.lastReleaseAt = releases.lastReleaseAt
+
+    return pkg
+  }
+
+  /**
+   * Fetch stats for all packages, either from cache or from npm/github
+   */
+  async refresh() {
+    logger.debug('Refreshing packages stats')
+
+    const limit = pLimit(10)
+    let packages = [...this.packagesList]
+
+    packages = await Promise.all(packages.map((pkg) => limit(() => this.#fetchPackageStats(pkg))))
+
+    await PackageStats.updateOrCreateMany(
+      'packageName',
+      packages.map((pkg) => ({
+        packageName: pkg.name,
+        githubStars: pkg.stars,
+        weeklyDownloads: pkg.downloads,
+        firstReleaseAt: pkg.firstReleaseAt ? DateTime.fromISO(pkg.firstReleaseAt) : null,
+        lastReleaseAt: pkg.lastReleaseAt ? DateTime.fromISO(pkg.lastReleaseAt) : null,
+      })),
+    )
+
+    logger.debug('Packages stats refreshed')
+  }
+}

app/services/packages_fetcher.ts

@@ -1,7 +1,7 @@
-import pLimit from 'p-limit'
 import cache from '@adonisjs/cache/services/main'
 import logger from '@adonisjs/core/services/logger'
 
+import PackageStats from '#models/package_stats'
 import { categories } from '../../content/categories.js'
 import { MarkdownRenderer } from './markdown_renderer.js'
 import type { PackageFetcher } from './package_fetcher.js'
@@ -15,68 +15,13 @@ export class PackagesFetcher {
     protected packagesList: PackageInfo[],
   ) {}
 
-  /**
-   * Creates a cache key with the given prefix and today's date
-   */
-  #createCacheKey(prefix: string) {
-    const today = new Date()
-    const todayKey = `${today.getFullYear()}-${today.getMonth() + 1}-${today.getDate()}`
-    return `${prefix}:${todayKey}`
-  }
-
-  /**
-   * Get the first and last release dates from cache or fetch it from npm
-   */
-  async #getReleasesDates(pkg: PackageInfo) {
-    if (!pkg.npm) return { firstReleaseAt: '', lastReleaseAt: '' }
-
-    const cacheKey = this.#createCacheKey(`npm:package:releases:${pkg.npm}`)
-
-    return await cache
-      .getOrSet(cacheKey, async () => this.packageFetcher.fetchReleaseDates(pkg.npm!))
-      .catch((err) => {
-        logger.error({ err }, `Cannot fetch releases dates for ${pkg.npm}`)
-        return { firstReleaseAt: '', lastReleaseAt: '' }
-      })
-  }
-
-  /**
-   * Get the package downloads from cache or fetch it from npm
-   */
-  async #getPackageDownloads(pkg: PackageInfo) {
-    if (!pkg.npm) return { downloads: 0 }
-
-    const cacheKey = this.#createCacheKey(`npm:package:downloads:${pkg.npm}`)
-    return await cache
-      .getOrSet(cacheKey, () => this.packageFetcher.fetchPackageDownloads(pkg.npm!))
-      .catch((err) => {
-        logger.error({ err }, `Cannot fetch npm info for ${pkg.npm}`)
-        return { downloads: 0 }
-      })
-  }
-
-  /**
-   * Get the github stars from cache or fetch it from github
-   */
-  async #getGithubStars(pkg: PackageInfo) {
-    if (!pkg.repo) return { stars: 0 }
-
-    const cacheKey = this.#createCacheKey(`github:repo:stars:${pkg.repo}`)
-    return cache
-      .getOrSet(cacheKey, () => this.packageFetcher.fetchGithubStars(pkg.repo))
-      .catch((err) => {
-        logger.error({ err }, `Cannot fetch github repo info for ${pkg.repo}`)
-        return { stars: 0 }
-      })
-  }
-
   /**
    * Get the github readme from cache or fetch it from github
    */
   async #getPackageReadme(pkg: PackageInfo) {
     if (!pkg.repo) return ''
 
-    const cacheKey = this.#createCacheKey(`github:repo:readme:${pkg.repo}`)
+    const cacheKey = `github:repo:readme:${pkg.repo}`
     const [repo, branch] = pkg.repo.split('#')
     return cache
       .getOrSet(cacheKey, () => this.packageFetcher.fetchReadme(repo, branch))
@@ -86,26 +31,6 @@ export class PackagesFetcher {
       })
   }
 
-  /**
-   * Get stats about a single package
-   */
-  async #fetchPackageStats(pkg: PackageInfo) {
-    logger.debug(`Fetching stats for ${pkg.name}`)
-
-    const [npmStats, ghStats, releases] = await Promise.all([
-      this.#getPackageDownloads(pkg),
-      this.#getGithubStars(pkg),
-      this.#getReleasesDates(pkg),
-    ])
-
-    pkg.downloads = npmStats.downloads
-    pkg.stars = ghStats.stars
-    pkg.firstReleaseAt = releases.firstReleaseAt
-    pkg.lastReleaseAt = releases.lastReleaseAt
-
-    return pkg
-  }
-
   /**
    * Sort packages based on PackagesFilters
    */
@@ -131,18 +56,35 @@ export class PackagesFetcher {
     })
   }
 
+  /**
+   * Merge raw package .yml data with the stats fetched from npm/github stored
+   * on our database
+   */
+  #mergePackageStatsAndInfo(pkg: PackageInfo, stats: PackageStats) {
+    return {
+      ...pkg,
+      firstReleaseAt: stats.firstReleaseAt?.toISODate() || undefined,
+      lastReleaseAt: stats.lastReleaseAt?.toISODate() || undefined,
+      stars: stats.githubStars,
+      downloads: stats.weeklyDownloads,
+    }
+  }
+
   /**
    * Fetch stats for all packages, either from cache or from npm/github
    */
   async fetchPackages(options: PackagesFilters = {}) {
+    const categoriesWithCount = this.#getCategories(this.packagesList)
+
     /**
-     * Get list of packages with their npm/GitHub stats
+     * Get packages list with stats
      */
-    const limit = pLimit(10)
-    const categoriesWithCount = this.#getCategories(this.packagesList)
-    let packages = [...this.packagesList]
+    const stats = await PackageStats.all()
 
-    packages = await Promise.all(packages.map((pkg) => limit(() => this.#fetchPackageStats(pkg))))
+    let packages = [...this.packagesList].map((pkg) => {
+      const info = stats.find((info) => info.packageName === pkg.name)
+      return this.#mergePackageStatsAndInfo(pkg, info!)
+    })
 
     /**
      * Filter them based on the given options
@@ -174,23 +116,25 @@ export class PackagesFetcher {
     return {
       packages,
       categories: categoriesWithCount,
-      meta: {
-        pages: totalPage,
-        total: this.packagesList.length,
-        currentPage: page,
-      },
+      meta: { pages: totalPage, total: this.packagesList.length, currentPage: page },
     }
   }
 
+  /**
+   * Fetch a single package with its readme
+   */
   async fetchPackage(name: string) {
     const pkg = this.packagesList.find((pkg_) => pkg_.name === name)
     if (!pkg) {
       throw new Error(`Cannot find package ${name}`)
     }
 
-    const stats = await this.#fetchPackageStats(pkg)
+    const stats = await PackageStats.findByOrFail('packageName', name)
     const readme = await this.#getPackageReadme(pkg)
 
-    return { package: stats, readme: this.#markdownRenderer.render(readme) }
+    return {
+      package: this.#mergePackageStatsAndInfo(pkg, stats),
+      readme: this.#markdownRenderer.render(readme),
+    }
   }
 }

commands/build_packages.ts

@@ -6,11 +6,16 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import type { CommandOptions } from '@adonisjs/core/types/ace'
 
+import { PackageFetcher } from '#services/package_fetcher'
+import { PackagesDataRefresher } from '#services/packages_data_refresher'
+
 export default class BuildPackages extends BaseCommand {
   static commandName = 'build:packages'
   static description = 'Create a big packages.json file with all the packages.'
 
-  static options: CommandOptions = {}
+  static options: CommandOptions = {
+    startApp: true,
+  }
 
   #contentFolder = join(getDirname(import.meta.url), '../content')
   #distFolder = join(getDirname(import.meta.url), '../content/build/')
@@ -45,6 +50,8 @@ export default class BuildPackages extends BaseCommand {
     await mkdir(this.#distFolder, { recursive: true })
     await writeFile(distFile, JSON.stringify(packages, null, 2))
 
+    await new PackagesDataRefresher(new PackageFetcher(), packages as any).refresh()
+
     this.logger.success(`Packages file created successfully at "${distFile}"`)
   }
 }

config/cache.ts

@@ -26,26 +26,19 @@ const cacheConfig = defineConfig({
   ttl: '1d',
 
   /**
-   * Grace period is 2 hours. Meaning, we gonna keep serving the
-   * old value for 2 hours, if we are unable to fetch the new
+   * Grace period is 14 days. Meaning, we gonna keep serving the
+   * old value for 2 weeks, if we are unable to fetch the new
    * value from the API (Rate limit, GitHub/npm down etc...?)
    */
   gracePeriod: {
     enabled: true,
-    duration: '2h',
+    duration: '14d',
     fallbackDuration: '5m',
   },
 
   stores: {
     cache: store()
-      .useL1Layer(
-        drivers.memory({
-          /**
-           * Keep only 50MB of cache in memory
-           */
-          maxSize: 50 * 1024 * 1024,
-        }),
-      )
+      .useL1Layer(drivers.memory({ maxSize: 50 * 1024 * 1024 }))
       .useL2Layer(drivers.database({ connectionName: 'sqlite' })),
 
     test: store().useL1Layer(drivers.memory({})),