important commit, cache.clear() WORKS, detect database change in anot…

…her process and clear cache on page request, set Date to string in DbMonth type

app/[[...month]]/page.tsx

@@ -5,13 +5,12 @@ import LineChartMultiple from '@/components/charts/line-chart-multiple';
 import Heading from '@/components/heading';
 import NewOldCompaniesSection from '@/components/new-old-companies-section';
 
-import { getNewOldCompaniesForMonth } from '@/modules/database/select/company';
-import { getNewOldCompaniesCountForAllMonths } from '@/modules/database/select/line-chart';
+import { getNewOldCompaniesForMonthCached } from '@/modules/database/select/company';
+import { clearCacheIfDatabaseUpdated } from '@/modules/database/select/is-updated';
+import { getNewOldCompaniesCountForAllMonthsCached } from '@/modules/database/select/line-chart';
 import { getAllMonths } from '@/modules/database/select/month';
-import { getStatistics } from '@/modules/database/select/statistics';
-import { getCacheDatabase, getDynamicCacheKey, getOrComputeValue } from '@/libs/keyv';
+import { getStatisticsCached } from '@/modules/database/select/statistics';
 import { isValidMonthNameWithDb } from '@/utils/validation';
-import { CACHE_KEYS_DATABASE } from '@/constants/cache';
 import { METADATA } from '@/constants/metadata';
 
 import { MonthQueryParam } from '@/types/website';
@@ -20,25 +19,11 @@ export interface Props extends MonthQueryParam {}
 
 const { title } = METADATA;
 
-const {
-  getStatisticsCacheKey,
-  getNewOldCompaniesCountForAllMonthsCacheKey,
-  getNewOldCompaniesForMonthCacheKey,
-} = CACHE_KEYS_DATABASE;
-
 const IndexPage: FC<Props> = async ({ params }) => {
-  const statistics = await getOrComputeValue(
-    () => getCacheDatabase().get(getStatisticsCacheKey),
-    (value) => getCacheDatabase().set(getStatisticsCacheKey, value),
-    getStatistics
-  );
+  await clearCacheIfDatabaseUpdated();
 
-  // ! must run AT REQUEST time, big problem with stale data
-  const lineChartMultipleData = await getOrComputeValue(
-    () => getCacheDatabase().get(getNewOldCompaniesCountForAllMonthsCacheKey),
-    (value) => getCacheDatabase().set(getNewOldCompaniesCountForAllMonthsCacheKey, value),
-    getNewOldCompaniesCountForAllMonths
-  );
+  const statistics = await getStatisticsCached();
+  const lineChartMultipleData = await getNewOldCompaniesCountForAllMonthsCached();
 
   const allMonths = getAllMonths();
 
@@ -48,17 +33,7 @@ const IndexPage: FC<Props> = async ({ params }) => {
 
   if (!isValidMonthNameWithDb(selectedMonth)) return notFound();
 
-  const newOldCompanies = await getOrComputeValue(
-    () =>
-      getCacheDatabase().get(getDynamicCacheKey(getNewOldCompaniesForMonthCacheKey, selectedMonth)),
-    (value) =>
-      getCacheDatabase().set(
-        getDynamicCacheKey(getNewOldCompaniesForMonthCacheKey, selectedMonth),
-        value
-      ),
-    getNewOldCompaniesForMonth,
-    selectedMonth
-  );
+  const newOldCompanies = await getNewOldCompaniesForMonthCached(selectedMonth);
 
   const { monthsCount, companiesCount, commentsCount, firstMonth, lastMonth } = statistics ?? {
     firstMonth: {},

app/month/[[...month]]/page.tsx

@@ -5,20 +5,17 @@ import BarChartSimple from '@/components/charts/bar-chart-simple';
 import CompaniesCommentsTable from '@/components/companies-comments-table';
 import Heading from '@/components/heading';
 
-import { getNewOldCompaniesForMonth } from '@/modules/database/select/company';
+import { getNewOldCompaniesForMonthCached } from '@/modules/database/select/company';
+import { clearCacheIfDatabaseUpdated } from '@/modules/database/select/is-updated';
 import { getAllMonths } from '@/modules/database/select/month';
 import { getBarChartSimpleData } from '@/modules/transform/bar-chart';
 import { getCompanyTableData } from '@/modules/transform/table';
-import { getCacheDatabase, getDynamicCacheKey, getOrComputeValue } from '@/libs/keyv';
 import { isValidMonthNameWithDb } from '@/utils/validation';
-import { CACHE_KEYS_DATABASE } from '@/constants/cache';
 
 import { MonthQueryParam } from '@/types/api';
 
 export interface Props extends MonthQueryParam {}
 
-const { getNewOldCompaniesForMonthCacheKey } = CACHE_KEYS_DATABASE;
-
 const CurrentMonthPage: FC<Props> = async ({ params }) => {
   const allMonths = getAllMonths();
 
@@ -27,18 +24,9 @@ const CurrentMonthPage: FC<Props> = async ({ params }) => {
 
   if (!isValidMonthNameWithDb(selectedMonth)) return notFound();
 
-  const newOldCompanies = await getOrComputeValue(
-    () =>
-      getCacheDatabase().get(getDynamicCacheKey(getNewOldCompaniesForMonthCacheKey, selectedMonth)),
-    (value) =>
-      getCacheDatabase().set(
-        getDynamicCacheKey(getNewOldCompaniesForMonthCacheKey, selectedMonth),
-        value
-      ),
-    getNewOldCompaniesForMonth,
-    selectedMonth
-  );
+  await clearCacheIfDatabaseUpdated();
 
+  const newOldCompanies = await getNewOldCompaniesForMonthCached(selectedMonth);
   const companyTableData = getCompanyTableData(newOldCompanies.allCompanies);
   const barChartSimpleData = getBarChartSimpleData(newOldCompanies.allCompanies);
 

constants/cache.ts

@@ -2,4 +2,5 @@ export const CACHE_KEYS_DATABASE = {
   getNewOldCompaniesCountForAllMonthsCacheKey: 'getNewOldCompaniesCountForAllMonthsCacheKey',
   getStatisticsCacheKey: 'getStatisticsCacheKey',
   getNewOldCompaniesForMonthCacheKey: 'getNewOldCompaniesForMonthCacheKey',
+  getUpdatedAtCacheKey: 'getUpdatedAtCacheKey',
 } as const;

docs/working-notes/notes4.md

@@ -36,5 +36,7 @@ if (cachedResult) return cachedResult;
   const dbResult = func(...args);
   await getCacheDatabase().set(key, dbResult);
 NIJE ISTI PROCESS u scheduler script, cli script, i u next.js page
+cache.clear() zove u drugoj aplikaciji - procesu iako je isti import // glavna POENTA
 event?
-get number of months from db, check in next.js page and cache.clear() // to
+get number of months from db, check in next.js page and cache.clear() // to
+compare cached number of months and db number of months

libs/datetime.ts

@@ -11,6 +11,8 @@ import { format as formatTz, toZonedTime } from 'date-fns-tz';
 
 import { SERVER_CONFIG } from '@/config/server';
 
+export type DateUnion = Date | string | number;
+
 const { appTimeZone, appDateTimeFormat } = SERVER_CONFIG;
 
 export const DATETIME = {
@@ -36,10 +38,10 @@ export const convertMonthNameToDate = (monthString: string): Date =>
  * Format to 'dd/MM/yyyy HH:mm:ss' to Belgrade time zone.
  * @example 05/11/2024 14:30:01
  */
-export const humanFormat = (date: Date): string =>
+export const humanFormat = (date: DateUnion): string =>
   formatTz(date, appDateTimeFormat, { timeZone: appTimeZone });
 
-export const getAppTime = (dateTime: Date): Date => toZonedTime(dateTime, appTimeZone);
+export const getAppTime = (dateTime: DateUnion): Date => toZonedTime(dateTime, appTimeZone);
 
 export const getAppNow = (): Date => getAppTime(new Date());
 

libs/keyv.ts

@@ -1,4 +1,4 @@
-import fs from 'fs';
+// import fs from 'fs';
 
 import Keyv, { KeyvStoreAdapter } from 'keyv';
 import KeyvFile from 'keyv-file';
@@ -48,10 +48,7 @@ const _databaseFileAdapter = new KeyvFile({ filename: cacheDatabaseFilePath });
 const databaseLruAdapter = new KeyvLruManagedTtl({ max: cacheDatabaseLruItems });
 CacheDatabaseInstance.setAdapter(databaseLruAdapter);
 
-// export const getCacheDatabase = CacheDatabaseInstance.getCacheDatabase;
-
-const keyvDb = new Keyv({ store: _databaseFileAdapter });
-export const getCacheDatabase = (): Keyv => keyvDb;
+export const getCacheDatabase = CacheDatabaseInstance.getCacheDatabase;
 
 /*-------------------------------- http cache ------------------------------*/
 
@@ -80,23 +77,18 @@ CacheHttpInstance.setAdapter(httpLruAdapter);
 
 export const getCacheHttp = CacheHttpInstance.getCacheHttp;
 
-export const getOrComputeValue = async <T, A extends any[]>(
-  getCache: () => Promise<T | undefined>,
-  setCache: (value: T) => Promise<boolean>,
-  computeValue: (...args: A) => T,
+export const cacheDatabaseWrapper = async <T, A extends any[]>(
+  key: string,
+  func: (...args: A) => T,
   ...args: A
 ): Promise<T> => {
   if (!cacheDatabaseDisabled) {
-    const cachedResult = await getCache();
-    console.log('cachedResult', (cachedResult as any)?.forMonth);
-
+    const cachedResult = await getCacheDatabase().get<T>(key);
     if (cachedResult) return cachedResult;
   }
 
-  const dbResult = computeValue(...args);
-  await setCache(dbResult);
-
-  console.log('dbResult', (dbResult as any)?.forMonth);
+  const dbResult = func(...args);
+  await getCacheDatabase().set(key, dbResult);
 
   return dbResult;
 };

modules/database/select/company.ts

@@ -1,9 +1,13 @@
 import { getDb } from '@/modules/database/schema';
 import { getMonthByName, getMonthPairByName } from '@/modules/database/select/month';
 import { convertCompanyRowType, withCommentsQuery } from '@/modules/database/select/utils';
+import { cacheDatabaseWrapper, getDynamicCacheKey } from '@/libs/keyv';
+import { CACHE_KEYS_DATABASE } from '@/constants/cache';
 
 import { CompanyWithCommentsAsStrings, MonthPair, NewOldCompanies } from '@/types/database';
 
+const { getNewOldCompaniesForMonthCacheKey } = CACHE_KEYS_DATABASE;
+
 /** Compare two specific months by name. */
 
 export const getNewOldCompaniesForTwoMonths = (monthPair: MonthPair): NewOldCompanies => {
@@ -89,3 +93,10 @@ export const getNewOldCompaniesForMonth = (monthName: string): NewOldCompanies =
 
   return newOldCompanies;
 };
+
+export const getNewOldCompaniesForMonthCached = (monthName: string) =>
+  cacheDatabaseWrapper(
+    getDynamicCacheKey(getNewOldCompaniesForMonthCacheKey, monthName),
+    getNewOldCompaniesForMonth,
+    monthName
+  );

modules/database/select/is-updated.ts

@@ -0,0 +1,63 @@
+import { cacheDatabaseWrapper, getCacheDatabase } from '@/libs/keyv';
+import logger from '@/libs/winston';
+import { CACHE_KEYS_DATABASE } from '@/constants/cache';
+import { getAllMonths } from './month';
+
+import { DbMonth } from '@/types/database';
+
+const { getUpdatedAtCacheKey } = CACHE_KEYS_DATABASE;
+
+export const getUpdatedAt = (): string[] => getAllMonths().map((month) => month.updatedAt);
+
+export const getUpdatedAtCached = () => cacheDatabaseWrapper(getUpdatedAtCacheKey, getUpdatedAt);
+
+export const findUpdatedMonth = (
+  allMonths: DbMonth[],
+  updatedAtArray: string[]
+): DbMonth | undefined => {
+  const updatedAtSet = new Set(updatedAtArray);
+
+  const dbMonth = allMonths.find((month) => !updatedAtSet.has(month.updatedAt));
+  if (dbMonth) return dbMonth;
+
+  const allMonthsUpdatedAtSet = new Set(allMonths.map((month) => month.updatedAt));
+  const missingUpdatedAt = updatedAtArray.find(
+    (updatedAt) => !allMonthsUpdatedAtSet.has(updatedAt)
+  );
+  if (missingUpdatedAt) {
+    return {
+      name: 'missing-in-db',
+      threadId: 'missing-in-db',
+      createdAtOriginal: new Date().toISOString(),
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date(missingUpdatedAt).toISOString(),
+    };
+  }
+
+  return undefined;
+};
+
+export const getUpdatedMonth = async (): Promise<DbMonth | undefined> => {
+  const allMonths = getAllMonths();
+  const updatedAtArrayCached = await getUpdatedAtCached();
+
+  const updatedMonth = findUpdatedMonth(allMonths, updatedAtArrayCached);
+
+  return updatedMonth;
+};
+
+/** This must run on every request, to detect change. */
+
+export const clearCacheIfDatabaseUpdated = async (): Promise<DbMonth | undefined> => {
+  const updatedMonth = await getUpdatedMonth();
+
+  if (updatedMonth) {
+    logger.info('Database changed, clearing cache, updatedMonth:', updatedMonth);
+    await getCacheDatabase().clear();
+
+    // populate cache again
+    await getUpdatedAtCached();
+  }
+
+  return updatedMonth;
+};

modules/database/select/line-chart.ts

@@ -1,7 +1,11 @@
 import { getDb } from '@/modules/database/schema';
+import { cacheDatabaseWrapper } from '@/libs/keyv';
+import { CACHE_KEYS_DATABASE } from '@/constants/cache';
 
 import { LineChartMultipleData } from '@/types/charts';
 
+const { getNewOldCompaniesCountForAllMonthsCacheKey } = CACHE_KEYS_DATABASE;
+
 export const getNewOldCompaniesCountForAllMonths = (): LineChartMultipleData[] => {
   const query = `
     WITH OrderedMonths AS (
@@ -81,3 +85,9 @@ export const getNewOldCompaniesCountForAllMonths = (): LineChartMultipleData[] =
 
   return result;
 };
+
+export const getNewOldCompaniesCountForAllMonthsCached = () =>
+  cacheDatabaseWrapper(
+    getNewOldCompaniesCountForAllMonthsCacheKey,
+    getNewOldCompaniesCountForAllMonths
+  );

modules/database/select/statistics.ts

@@ -1,8 +1,12 @@
 import { getDb } from '@/modules/database/schema';
 import { getFirstMonth, getLastMonth } from '@/modules/database/select/month';
+import { cacheDatabaseWrapper } from '@/libs/keyv';
+import { CACHE_KEYS_DATABASE } from '@/constants/cache';
 
 import { Statistics } from '@/types/database';
 
+const { getStatisticsCacheKey } = CACHE_KEYS_DATABASE;
+
 export const getStatistics = (): Statistics => {
   const counts = getDb()
     .prepare<[], Statistics>(
@@ -24,3 +28,5 @@ export const getStatistics = (): Statistics => {
 
   return statistics;
 };
+
+export const getStatisticsCached = () => cacheDatabaseWrapper(getStatisticsCacheKey, getStatistics);

modules/database/select/utils.ts

@@ -55,13 +55,6 @@ export const withCommentsQuery = (
     `;
 
 export const convertCompanyRowType = (row: CompanyWithCommentsAsStrings): CompanyWithComments => ({
-  company: {
-    name: row.name,
-    commentId: row.commentId,
-    monthName: row.monthName,
-    createdAtOriginal: new Date(row.createdAtOriginal),
-    createdAt: new Date(row.createdAt),
-    updatedAt: new Date(row.updatedAt),
-  },
+  company: row,
   comments: JSON.parse(row.comments) as DbCompany[],
 });