From 84c058d964c76cc87369ce5b09c469625a1059bc Mon Sep 17 00:00:00 2001 From: Pavan Rikhi Date: Sat, 3 Aug 2024 02:06:23 -0400 Subject: [PATCH] [GH#14] Allow Aliasing of Commodity Symbols Add a new `commodity-aliases` configuration file option that allows you to rename journal symbols into AlphaVantage tickers. We convert these aliased commodities to the proper symbols, then convert the resulting symbols in the price directives back to their aliased versions before writing them out. Closes GH#14 --- README.md | 42 +++++++++++++++++--- app/Main.hs | 48 +++++++++++++++++++---- hledger-stockquotes.cabal | 3 +- package.yaml | 1 + src/Hledger/StockQuotes.hs | 78 +++++++++++++++++++++++++++++++++++--- 5 files changed, 153 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4350b75..578e626 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ exists! ### Excluding Commodities By default, we query AlphaVantage for all non-USD commodities included in your -journal file. We do not currently support AlphaVantage's FOREX or Crypto API -routes, so if you have those commodities, `stockquotes` will print an error -when fetching them. You can exclude commodities by passing them as arguments to +journal file. We do not currently support AlphaVantage's FOREX API route, so if +you have those commodities, `stockquotes` will print an error when fetching +them. You can exclude commodities by passing them as arguments to `hledger-stockquotes`: ```sh @@ -88,23 +88,53 @@ ranges that would be queried instead of making requests to AlphaVantage. `$XDG_CONFIG_HOME/hledger-stockquotes/config.yaml`(`$XDG_CONFIG_HOME` is usually `~/.config/`). -You can set the `api-key`, `rate-limit`, `cryptocurrencies`, & `exclude` -options via this file: +You can set the `api-key`, `rate-limit`, `cryptocurrencies`, `exclude`, & +`commodity-aliases` options via this file: ```yaml rate-limit: false api-key: DeAdBeEf9001 -crypto-currencies: +cryptocurrencies: - BTC - XMR exclude: - USD - AUTO +commodity-aliases: + MY_BTC_CURRENCY: BTC + 401K_VTSAX: VTSAX ``` CLI flags & environmental variables will override config file settings. +### Aliases + +By specifying the `commedity-aliases` option in your configuration file, +you can rename the commodities used in your journal to the commodities +expected by AlphaVantage. + +Keys in the map should be your journal commities while their values are the +AlphaVantage ticker symbols: + +```yaml +commodity-aliases: + MY_VTSAX: VTSAX + MY_BTC_CURRENCY: BTC +``` + +Renaming is done after commodity exclusion, but before bucketing them into +equities & cryptocurrencies so the `exclude` list should use your symbols while +the `cryptocurrencies` list should use AlphaVantage's: + +```code +journal -> exclude -> commodity-aliases -> cryptocurrencies +``` + +Specifying aliases via command line options or environmental variable +is not currently supported. + + ### Additional Documentation The `--help` flag provides more thorough documentation on all available flags: diff --git a/app/Main.hs b/app/Main.hs index 6b6956a..4541cf4 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -4,6 +4,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TupleSections #-} {-# LANGUAGE ViewPatterns #-} module Main where @@ -16,7 +17,6 @@ import Data.Aeson , withObject , (.:?) ) -import Data.List (partition) import Data.Maybe (fromMaybe) import Data.Time ( Day @@ -61,6 +61,7 @@ import Paths_hledger_stockquotes (version) import Web.AlphaVantage import qualified Data.ByteString.Lazy as LBS +import qualified Data.Map as M import qualified Data.Text as T @@ -81,6 +82,7 @@ main = do cfg commodities cryptoCurrencies + aliases start end rateLimit @@ -94,11 +96,14 @@ main = do <> " to " <> showDate end let (stocks, cryptos) = - partition (`notElem` cryptoCurrencies) commodities + unaliasAndBucketCommodities commodities cryptoCurrencies aliases putStrLn "Querying Stocks:" forM_ stocks $ \commodity -> putStrLn $ "\t" <> T.unpack commodity putStrLn "Querying CryptoCurrencies:" forM_ cryptos $ \commodity -> putStrLn $ "\t" <> T.unpack commodity + let reAliased = map fst $ reAliasCommodities (fmap (,()) $ stocks <> cryptos) commodities aliases + putStrLn "Writing Prices for:" + forM_ reAliased $ \commodity -> putStrLn $ "\t" <> T.unpack commodity where showDate :: Day -> String showDate = formatTime defaultTimeLocale "%Y-%m-%d" @@ -118,6 +123,7 @@ data AppConfig = AppConfig , excludedCurrencies :: [String] , cryptoCurrencies :: [T.Text] , dryRun :: Bool + , aliases :: M.Map T.Text T.Text } deriving (Show, Eq) @@ -155,6 +161,7 @@ mergeArgsEnvCfg ConfigFile {..} Args {..} = do else concatMap (T.splitOn "," . T.pack) argCryptoCurrencies outputFile = argOutputFile dryRun = argDryRun + aliases = fromMaybe M.empty cfgAliases return AppConfig {..} @@ -163,6 +170,7 @@ data ConfigFile = ConfigFile , cfgRateLimit :: Maybe Bool , cfgExcludedCurrencies :: Maybe [String] , cfgCryptoCurrencies :: Maybe [String] + , cfgAliases :: Maybe (M.Map T.Text T.Text) } deriving (Show, Eq) @@ -173,6 +181,7 @@ instance FromJSON ConfigFile where cfgRateLimit <- o .:? "rate-limit" cfgExcludedCurrencies <- o .:? "exclude" cfgCryptoCurrencies <- o .:? "cryptocurrencies" + cfgAliases <- o .:? "commodity-aliases" return ConfigFile {..} @@ -191,7 +200,7 @@ loadConfigFile = do else return defaultConfig where defaultConfig :: ConfigFile - defaultConfig = ConfigFile Nothing Nothing Nothing Nothing + defaultConfig = ConfigFile Nothing Nothing Nothing Nothing Nothing data Args = Args @@ -350,15 +359,40 @@ specify them in a YAML configuration file. We attempt to parse a configuration file in $XDG_CONFIG_HOME/hledger-stockquotes/config.yaml. It currently supports the following top-level keys: -- `api-key`: (string) Your AlphaVantage API Key -- `cryptocurrencies`: (list of string) Cryptocurrencies to Fetch -- `exclude`: (list of strings) Currencies to Exclude -- `rate-limit`: (bool) Obey AlphaVantage's Rate Limit +- `api-key`: (string) Your AlphaVantage API Key +- `cryptocurrencies`: (list of string) Cryptocurrencies to Fetch +- `exclude`: (list of strings) Currencies to Exclude +- `rate-limit`: (bool) Obey AlphaVantage's Rate Limit +- `commodity-aliases`: (map of strings) Rename journal commodities before + querying AlphaVantage Environmental variables will overide any config file options, and CLI flags will override both environmental variables & config file options. +ALIASES + +By specifying the `commedity-aliases` option in your configuration file, +you can rename the commodities used in your journal to the commodities +expected by AlphaVantage. + +Keys in the map should be your journal commities while their values are the +AlphaVantage ticker symbols: + + commodity-aliases: + MY_VTSAX: VTSAX + MY_BTC_CURRENCY: BTC + +Renaming is done after commodity exclusion, but before bucketing them into +equities & cryptocurrencies so the `exclude` list should use your symbols +while the `cryptocurrencies` list should use AlphaVantage's: + + journal -> exclude -> aliases -> cryptocurrencies + +Specifying aliases via command line options or environmental variables is +not currently supported. + + USAGE EXAMPLES Fetch prices for all commodities in the default journal file: diff --git a/hledger-stockquotes.cabal b/hledger-stockquotes.cabal index 9a5684a..3134e00 100644 --- a/hledger-stockquotes.cabal +++ b/hledger-stockquotes.cabal @@ -1,6 +1,6 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.35.0. +-- This file has been generated from package.yaml by hpack version 0.36.0. -- -- see: https://github.com/sol/hpack @@ -73,6 +73,7 @@ executable hledger-stockquotes , base >=4.7 && <5 , bytestring <1 , cmdargs >=0.6 && <1 + , containers <1 , directory <2 , hledger-stockquotes , raw-strings-qq <2 diff --git a/package.yaml b/package.yaml index bd1f46f..512ae95 100644 --- a/package.yaml +++ b/package.yaml @@ -88,6 +88,7 @@ executables: - aeson >= 1 && < 3 - bytestring < 1 - cmdargs >= 0.6 && < 1 + - containers < 1 - directory < 2 - raw-strings-qq < 2 - safe-exceptions diff --git a/src/Hledger/StockQuotes.hs b/src/Hledger/StockQuotes.hs index e647a22..85bb430 100644 --- a/src/Hledger/StockQuotes.hs +++ b/src/Hledger/StockQuotes.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE LambdaCase #-} -{-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -10,6 +8,8 @@ module Hledger.StockQuotes ( getCommoditiesAndDateRange , fetchPrices , makePriceDirectives + , unaliasAndBucketCommodities + , reAliasCommodities ) where import Control.Concurrent (threadDelay) @@ -19,7 +19,7 @@ import Control.Exception , try ) import Data.List.Split (chunksOf) -import Data.Maybe (catMaybes) +import Data.Maybe (catMaybes, fromMaybe) import Data.Text.Encoding (encodeUtf8) import Data.Time ( Day @@ -31,6 +31,13 @@ import Data.Time , toGregorian ) import Hledger + ( CommoditySymbol + , Journal (..) + , Transaction (..) + , definputopts + , readJournalFile + , runExceptT + ) import Safe.Foldable ( maximumMay , minimumMay @@ -51,7 +58,9 @@ import Web.AlphaVantage import qualified Data.ByteString.Lazy as LBS import qualified Data.ByteString.Lazy.Char8 as LC import qualified Data.List as L +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as M +import qualified Data.Set as S import qualified Data.Text as T @@ -93,6 +102,8 @@ fetchPrices -- ^ Commodities to Fetch -> [T.Text] -- ^ Commodities to Classify as Cryptocurrencies + -> M.Map T.Text T.Text + -- ^ Map of aliases to transform journal commodities -> Day -- ^ Start of Price Range -> Day @@ -100,9 +111,9 @@ fetchPrices -> Bool -- ^ Rate Limit Requests -> IO [(CommoditySymbol, [(Day, Prices)])] -fetchPrices cfg symbols cryptoCurrencies start end rateLimit = do +fetchPrices cfg symbols cryptoCurrencies aliases start end rateLimit = do let (stockSymbols, cryptoSymbols) = - L.partition (`notElem` cryptoCurrencies) symbols + unaliasAndBucketCommodities symbols cryptoCurrencies aliases genericAction = map FetchStock stockSymbols <> map FetchCrypto cryptoSymbols if rateLimit @@ -148,6 +159,63 @@ fetchPrices cfg symbols cryptoCurrencies start end rateLimit = do logError = hPutStrLn stderr +-- | Given a list of commodities from a journal, a list a cryptocurrencies, +-- and a map of aliases, return the a list of AlphaVantage equities +-- & cryptocurencies. +unaliasAndBucketCommodities + :: [CommoditySymbol] + -- ^ Journal symbols + -> [T.Text] + -- ^ Cryptocurrency symbols + -> M.Map T.Text T.Text + -- ^ Aliases + -> ([CommoditySymbol], [CommoditySymbol]) +unaliasAndBucketCommodities symbols cryptoCurrencies aliases = + L.partition (`notElem` cryptoCurrencies) $ + S.toList $ + S.fromList $ + map transformAliases symbols + where + transformAliases :: T.Text -> T.Text + transformAliases original = + fromMaybe original $ M.lookup original aliases + + +-- | Given a list of paired unaliased symbols, the original journal +-- commodities, and the map of aliases, generate a new list of paired +-- symbols that reflects the commodities in the original journal. +-- +-- Pairs with symbols in the journal but not in the aliases will be +-- unaltered. Pairs with aliases only in the journal will return only alias +-- items. Pairs for multiple aliases with return a set of items for each +-- alias. Pairs with symbols and aliases in the journal will return both +-- sets of items. +reAliasCommodities + :: [(CommoditySymbol, a)] + -- ^ Unaliased pairs of symbols + -> [CommoditySymbol] + -- ^ Original symbols from the journal + -> M.Map T.Text T.Text + -- ^ Aliases + -> [(CommoditySymbol, a)] +reAliasCommodities symbolPairs journalSymbols aliases = + concatMap reAlias symbolPairs + where + reAlias :: (CommoditySymbol, a) -> [(CommoditySymbol, a)] + reAlias s@(cs, a) = case M.lookup cs reverseAliases of + Nothing -> + [s] + Just revAliases -> + map (,a) $ filter (`elem` journalSymbols) $ NE.toList revAliases + reverseAliases :: M.Map T.Text (NE.NonEmpty T.Text) + reverseAliases = + let journalSymbolPairs = map (\s -> (s, NE.singleton s)) journalSymbols + in M.fromListWith (<>) + . (<> journalSymbolPairs) + . map (\(k, v) -> (v, NE.singleton k)) + $ M.assocs aliases + + -- | Types of AlphaVantage requests we make. Unified under one type so we -- write a generic fetching function that can be rate limited. data AlphaRequest