Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add file cache, increase test coverage #22

Merged
merged 30 commits into from
Feb 10, 2024
Merged
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
4b41139
Add local file cache to store Yahoo Finance data between runs
dickwolff Jan 9, 2024
97bb6d9
Add readme part about cache
dickwolff Jan 9, 2024
069e737
Console logging format
dickwolff Jan 9, 2024
c209846
Fix cache set and parse
dickwolff Jan 11, 2024
bf85ff5
Small renames
dickwolff Jan 11, 2024
7872413
Remove debug logging
dickwolff Jan 11, 2024
95c085b
Merge branch 'main' into feature/Add-File-Cache
dickwolff Jan 24, 2024
4841194
Update caching strategy description
dickwolff Jan 24, 2024
97f87a0
Merge branch 'main' into feature/Add-File-Cache
dickwolff Feb 2, 2024
72a3b82
Merge branch 'main' into feature/Add-File-Cache
dickwolff Feb 9, 2024
91c9ce7
Return cache size on load
dickwolff Feb 9, 2024
e22bb11
Preload cache on converter create
dickwolff Feb 9, 2024
a666468
Fix cache retrieval
dickwolff Feb 9, 2024
0f25f12
Bump gitversion
dickwolff Feb 9, 2024
bd4a3f3
Add some more files to coverage ignore
dickwolff Feb 9, 2024
538b716
Don't run test converters workflow on all pushes
dickwolff Feb 9, 2024
739da43
Reverse .env.sample change
dickwolff Feb 9, 2024
ec378fb
Remove obsolete symbol check
dickwolff Feb 9, 2024
beb2d6d
Fix unitPrice calculation for etoro
dickwolff Feb 9, 2024
4907212
Fix unit price calculation
dickwolff Feb 10, 2024
5d476f9
Add more validations in existing tests
dickwolff Feb 10, 2024
aee2d44
Add tests for finpension and schwab
dickwolff Feb 10, 2024
9effd28
Test cleanup
dickwolff Feb 10, 2024
4d2324c
Test cleanup
dickwolff Feb 10, 2024
6d2ecf8
Add tests for DEGIRO v2
dickwolff Feb 10, 2024
a3b03e8
Fix final degiro test
dickwolff Feb 10, 2024
ce50f59
Move caching to run local section
dickwolff Feb 10, 2024
2187cf7
Add cache purging support to Docker container
dickwolff Feb 10, 2024
befe516
Merge branch 'main' into feature/Add-File-Cache
dickwolff Feb 10, 2024
1730d30
Add percentage to coverage badge
dickwolff Feb 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Preload cache on converter create
  • Loading branch information
dickwolff committed Feb 9, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit e22bb111cabef9e05174b864d551574b5b75f7d4
10 changes: 6 additions & 4 deletions src/converter.ts
Original file line number Diff line number Diff line change
@@ -11,8 +11,7 @@ import { SwissquoteConverter } from "./converters/swissquoteConverter";
import { FinpensionConverter } from "./converters/finpensionConverter";
import { YahooFinanceService } from "./yahooFinanceService";


export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {
export async function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {

// Verify if Ghostolio account ID is set (because without it there can be no valid output).
if (!process.env.GHOSTFOLIO_ACCOUNT_ID) {
@@ -22,7 +21,7 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri
const converterTypeLc = converterType.toLocaleLowerCase();

// Determine convertor type.
const converter = createConverter(converterTypeLc);
const converter = await createConverter(converterTypeLc);

// Map the file to a Ghostfolio import.
converter.readAndProcessFile(inputFilePath, (result: GhostfolioExport) => {
@@ -41,10 +40,13 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri
}, (error) => errorCallback(error));
}

function createConverter(converterType: string): AbstractConverter {
async function createConverter(converterType: string): Promise<AbstractConverter> {

const yahooFinanceService = new YahooFinanceService();

const cacheSize = await yahooFinanceService.loadCache();
console.log(`[i] Restored ${cacheSize[0]} ISIN-symbol pairs and ${cacheSize[1]} symbols from cache..`);

let converter: AbstractConverter;

switch (converterType) {
54 changes: 28 additions & 26 deletions src/yahooFinanceService.ts
Original file line number Diff line number Diff line change
@@ -29,9 +29,6 @@ export class YahooFinanceService {

// Retrieve prefered exchange postfix if set in .env
this.preferedExchangePostfix = process.env.DEGIRO_PREFERED_EXCHANGE_POSTFIX;

// Preload the cache from disk.
this.preloadCache().then((cacheSize) => console.log(`\n[i] Restored ${cacheSize[0]} ISIN-symbol pairs and ${cacheSize[1]} symbols from cache..`));
}

/**
@@ -53,7 +50,7 @@ export class YahooFinanceService {
if (symbol) {

const symbolMatch = this.symbolCache.has(symbol);

// If a match was found, return the security.
if (symbolMatch) {
this.logDebug(`Retrieved symbol ${symbol} from cache!`, progress);
@@ -134,6 +131,33 @@ export class YahooFinanceService {
return null;
}

/**
* Load the cache with ISIN and symbols.
*
* @returns The size of the loaded cache
*/
public async loadCache(): Promise<[number, number]> {

// Verify if there is data in the ISIN-Symbol cache. If so, restore to the local variable.
const isinSymbolCacheExist = await cacache.get.info(cachePath, "isinSymbolCache");
if (isinSymbolCacheExist) {
const cache = await cacache.get(cachePath, "isinSymbolCache");
const cacheAsJson = JSON.parse(cache.data.toString(), this.mapReviver);
this.isinSymbolCache = cacheAsJson;
}

// Verify if there is data in the Symbol cache. If so, restore to the local variable.
const symbolCacheExists = await cacache.get.info(cachePath, "symbolCache");
if (symbolCacheExists) {
const cache = await cacache.get(cachePath, "symbolCache");
const cacheAsJson = JSON.parse(cache.data.toString(), this.mapReviver);
this.symbolCache = cacheAsJson;
}

// Return cache sizes.
return [this.isinSymbolCache.size, this.symbolCache.size];
}

/**
* Get symbols for a security by a given key.
*
@@ -237,28 +261,6 @@ export class YahooFinanceService {
return symbolMatch;
}

private async preloadCache(): Promise<[number, number]> {

// Verify if there is data in the ISIN-Symbol cache. If so, restore to the local variable.
const isinSymbolCacheExist = await cacache.get.info(cachePath, "isinSymbolCache");
if (isinSymbolCacheExist) {
const cache = await cacache.get(cachePath, "isinSymbolCache");
const cacheAsJson = JSON.parse(cache.data.toString(), this.mapReviver);
this.isinSymbolCache = cacheAsJson;
}

// Verify if there is data in the Symbol cache. If so, restore to the local variable.
const symbolCacheExists = await cacache.get.info(cachePath, "symbolCache");
if (symbolCacheExists) {
const cache = await cacache.get(cachePath, "symbolCache");
const cacheAsJson = JSON.parse(cache.data.toString(), this.mapReviver);
this.symbolCache = cacheAsJson;
}

// Return cache sizes.
return [this.isinSymbolCache.size, this.symbolCache.size];
}

private async saveInCache(isin?: string, symbol?: string, value?: any) {

// Save ISIN-value combination to cache if given.