diff --git a/coinmetrics/_data_collection.py b/coinmetrics/_data_collection.py index dd46275..06a4efd 100644 --- a/coinmetrics/_data_collection.py +++ b/coinmetrics/_data_collection.py @@ -23,6 +23,7 @@ ) from coinmetrics._utils import get_file_path_or_buffer from coinmetrics._models import AssetChainsData, CoinMetricsAPIModel, TransactionTrackerData +from coinmetrics._catalogs import convert_catalog_dtypes, _expand_df from importlib import import_module from concurrent.futures import ThreadPoolExecutor, Executor from tqdm import tqdm @@ -362,12 +363,6 @@ class TransactionTrackerDataCollection(DataCollection): API_RETURN_MODEL = TransactionTrackerData -class CatalogV2DataCollection(DataCollection): - """ - This class will be used to implement functionality specific to catalog-v2 endpoints. - """ - - class ParallelDataCollection(DataCollection): """ This class will be used as an extension of the normal data collection, but all functions will run in parallel, @@ -1009,3 +1004,121 @@ def parse_date(date_input: Union[datetime, date, str]) -> datetime: except ValueError: continue raise ValueError(f"Unrecognized date format for string: {date_input}") + + +class CatalogV2DataCollection(DataCollection): + """ + This class is used to implement functionality specific to catalog-v2 endpoints. + """ + + def __init__( + self, + data_retrieval_function: DataRetrievalFuncType, + endpoint: str, + url_params: Dict[str, UrlParamTypes], + csv_export_supported: bool = True, + columns_to_store: List[str] = [], + client: Optional[CoinMetricsClient] = None, + metric_type: Optional[str] = None, + iterable_col: Optional[str] = None, + iterable_key: Optional[str] = None, + explode_on: Optional[str] = None, + assign_to: Optional[str] = None, + nested_catalog_columns: List[str] = ["min_time", "max_time"] + ): + super().__init__( + data_retrieval_function=data_retrieval_function, + endpoint=endpoint, + url_params=url_params, + csv_export_supported=csv_export_supported, + columns_to_store=columns_to_store, + client=client + ) + # *-metrics data + self.metric_type = metric_type + # column where nested catalog fields live e.g. "frequencies" + self.iterable_col = iterable_col + # grain for each value in iterable_col e.g. "frequency" + self.iterable_key = iterable_key + self.explode_on = explode_on + self.assign_to = assign_to + # fields in nested dicts in catalog response + self.nested_catalog_columns = nested_catalog_columns + + def to_dataframe( + self, + header: Optional[List[str]] = None, + dtype_mapper: Optional[Dict[str, Any]] = None, + optimize_pandas_types: Optional[bool] = True + ) -> DataFrameType: + """ + Transforms catalog data in list form into a dataframe + :return: DataFrame + """ + df = pd.DataFrame(self) + + # catalog data with no nested data + if self.iterable_col is None or not isinstance(self.iterable_col, str) or not isinstance(self.iterable_key, str): + return convert_catalog_dtypes(df) + + # for *-metrics and market-* types, add frequency (depth if orderbook) + if isinstance(self.iterable_key, str): + self.nested_catalog_columns = [self.iterable_key] + self.nested_catalog_columns + + def _assign_column(df_: DataFrameType, col_name: str, values: Iterable[Any]) -> DataFrameType: + return df_.assign(**{col_name: values}) + + if self.metric_type is not None: + # for *-metrics datatypes + mapper = df[self.metric_type].to_dict() + + def _assign_metric(x: DataFrameType) -> Any: + try: + return x[self.assign_to] + except TypeError: + return None + + df = df.explode(self.explode_on).assign( + metrics=lambda x: pd.Series(x[self.explode_on]) + ) + df[self.assign_to] = df[self.explode_on].apply(_assign_metric) + df_metrics = df.dropna(subset=[self.explode_on]).metrics.apply( + pd.Series + ) + df_metrics[self.metric_type] = df_metrics.index.map(mapper) + df_metrics = df_metrics.explode(self.iterable_col) + + # expand min/max time, heights, hash + for column in self.nested_catalog_columns: + df_metrics = _assign_column( + df_metrics, + column, + _expand_df( + key=column, iterable=df_metrics[self.iterable_col] + ) + ) + df_metrics = df_metrics.drop([self.iterable_col], axis=1) + + df = ( + df.drop(["metrics"], axis=1).merge( + df_metrics, on=["metric", self.metric_type], how="left" + ).reset_index(drop=True) + ) + df = convert_catalog_dtypes(df) + return df + else: + # for market-* data types + df = df.explode(self.iterable_col) + + # expand metadata (min/max time) + for column in self.nested_catalog_columns: + df = _assign_column( + df, + column, + _expand_df( + key=column, iterable=df[self.iterable_col] + ) + ) + df = df.drop([self.iterable_col], axis=1) + + return convert_catalog_dtypes(df) diff --git a/coinmetrics/_utils.py b/coinmetrics/_utils.py index 038b452..0714d42 100644 --- a/coinmetrics/_utils.py +++ b/coinmetrics/_utils.py @@ -5,7 +5,7 @@ from logging import getLogger from os.path import expanduser from time import sleep -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Set from coinmetrics._typing import FilePathOrBuffer, UrlParamTypes logger = getLogger("cm_client_utils") @@ -115,3 +115,14 @@ def wrapper(*args: List[Any], **kwargs: Dict[str, Any]) -> Any: return wrapper return retry_wrapper + + +def get_keys_from_catalog(d: Dict[str, str]) -> Set[str]: + keys = [] + for k, v in d.items(): + if isinstance(v, list): + for nested_dict in v: + keys.extend(get_keys_from_catalog(nested_dict)) + else: + keys.append(k) + return set(keys) diff --git a/coinmetrics/api_client.py b/coinmetrics/api_client.py index 3cad0b4..d189910 100644 --- a/coinmetrics/api_client.py +++ b/coinmetrics/api_client.py @@ -43,7 +43,7 @@ CatalogMiningPoolTipsData, CatalogTransactionTrackerData, CatalogMarketContractPrices, - CatalogMarketImpliedVolatility + CatalogMarketImpliedVolatility, ) from importlib import import_module @@ -2205,7 +2205,14 @@ def catalog_market_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-candles", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/market-candles", + params, + iterable_col="frequencies", + iterable_key="frequency", + client=self + ) def catalog_market_orderbooks_v2( self, @@ -2261,7 +2268,14 @@ def catalog_market_orderbooks_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-orderbooks", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/market-orderbooks", + params, + iterable_key="depth", + iterable_col="depths", + client=self + ) def catalog_market_quotes_v2( self, @@ -2729,6 +2743,8 @@ def catalog_market_metrics_v2( """ :param markets: Comma separated list of markets. By default all markets are returned. :type markets: Optional[Union[str, List[str]]] + :param metrics: Comma separated list of metrics. By default all metrics are returned. + :type metrics: Optional[Union[str, List[str]]] :param exchange: Unique name of an exchange. :type exchange: Optional[str] :param market_type: Type of markets. @@ -2767,7 +2783,17 @@ def catalog_market_metrics_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/market-metrics", + params, + metric_type='market', + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_full_market_trades_v2( self, @@ -2879,7 +2905,14 @@ def catalog_full_market_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-candles", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/market-candles", + params, + iterable_col="frequencies", + iterable_key="frequency", + client=self + ) def catalog_full_market_orderbooks_v2( self, @@ -2935,7 +2968,14 @@ def catalog_full_market_orderbooks_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-orderbooks", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/market-orderbooks", + params, + iterable_key="depth", + iterable_col="depths", + client=self + ) def catalog_full_market_quotes_v2( self, @@ -3403,6 +3443,8 @@ def catalog_full_market_metrics_v2( """ :param markets: Comma separated list of markets. By default all markets are returned. :type markets: Optional[Union[str, List[str]]] + :param metrics: Comma separated list of metrics. By default all metrics are returned. + :type metrics: Optional[Union[str, List[str]]] :param exchange: Unique name of an exchange. :type exchange: Optional[str] :param market_type: Type of markets. @@ -3441,7 +3483,17 @@ def catalog_full_market_metrics_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/market-metrics", + params, + metric_type='market', + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_asset_metrics_v2( self, @@ -3481,7 +3533,18 @@ def catalog_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/asset-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/asset-metrics", + params, + metric_type="asset", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + nested_catalog_columns=["frequency", "min_time", "max_time", "min_height", "max_height", "min_hash", "max_hash", "community"], + client=self + ) def catalog_full_asset_metrics_v2( self, @@ -3521,7 +3584,18 @@ def catalog_full_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/asset-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/asset-metrics", + params, + metric_type="asset", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + nested_catalog_columns=["frequency", "min_time", "max_time", "min_height", "max_height", "min_hash", "max_hash", "community"], + client=self + ) def catalog_exchange_metrics_v2( self, @@ -3561,7 +3635,17 @@ def catalog_exchange_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/exchange-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/exchange-metrics", + params, + metric_type="exchange", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_full_exchange_metrics_v2( self, @@ -3601,7 +3685,17 @@ def catalog_full_exchange_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/exchange-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/exchange-metrics", + params, + metric_type="exchange", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_exchange_asset_metrics_v2( self, @@ -3641,7 +3735,17 @@ def catalog_exchange_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/exchange-asset-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/exchange-asset-metrics", + params, + metric_type="exchange_asset", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_full_exchange_asset_metrics_v2( self, @@ -3681,7 +3785,17 @@ def catalog_full_exchange_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/exchange-asset-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/exchange-asset-metrics", + params, + metric_type="exchange_asset", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_pair_metrics_v2( self, @@ -3721,7 +3835,17 @@ def catalog_pair_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/pair-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/pair-metrics", + params, + metric_type="pair", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_full_pair_metrics_v2( self, @@ -3761,7 +3885,17 @@ def catalog_full_pair_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/pair-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/pair-metrics", + params, + metric_type="pair", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_institution_metrics_v2( self, @@ -3801,7 +3935,17 @@ def catalog_institution_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/institution-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/institution-metrics", + params, + metric_type="institution", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_full_institution_metrics_v2( self, @@ -3841,7 +3985,17 @@ def catalog_full_institution_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/institution-metrics", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/institution-metrics", + params, + metric_type="institution", + iterable_col="frequencies", + iterable_key="frequency", + explode_on="metrics", + assign_to="metric", + client=self + ) def catalog_pair_candles_v2( self, @@ -3869,7 +4023,14 @@ def catalog_pair_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/pair-candles", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/pair-candles", + params, + iterable_col="frequencies", + iterable_key="frequency", + client=self + ) def catalog_index_candles_v2( self, @@ -3897,7 +4058,14 @@ def catalog_index_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/index-candles", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/index-candles", + params, + iterable_col="frequencies", + iterable_key="frequency", + client=self + ) def catalog_index_levels_v2( self, @@ -3925,7 +4093,14 @@ def catalog_index_levels_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/index-levels", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-v2/index-levels", + params, + iterable_col="frequencies", + iterable_key="frequency", + client=self + ) def catalog_asset_chains_v2( self, @@ -4093,7 +4268,14 @@ def catalog_full_index_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/index-candles", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/index-candles", + params, + iterable_col="frequencies", + iterable_key="frequency", + client=self + ) def catalog_full_index_levels_v2( self, @@ -4121,7 +4303,12 @@ def catalog_full_index_levels_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/index-levels", params, client=self) + return CatalogV2DataCollection( + self._get_data, + "/catalog-all-v2/index-levels", + params, + client=self + ) def catalog_full_asset_chains_v2( self, diff --git a/test/test_catalogs_v2.py b/test/test_catalogs_v2.py index d00a6a1..4cc61c7 100644 --- a/test/test_catalogs_v2.py +++ b/test/test_catalogs_v2.py @@ -2,6 +2,7 @@ import pytest from coinmetrics.api_client import CoinMetricsClient +from coinmetrics._utils import get_keys_from_catalog import os client = CoinMetricsClient(str(os.environ.get("CM_API_KEY")), verbose=True) @@ -17,6 +18,10 @@ def test_catalogv2_market_trades() -> None: catalog_market_trades = client.catalog_market_trades_v2(page_size=10).first_page() assert len(catalog_market_trades) == 10 assert all(['market' in catalog for catalog in catalog_market_trades]) + df_market_trades = client.catalog_market_trades_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_trades = client.catalog_market_trades_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_trades[0]) + assert set(df_market_trades.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -24,6 +29,10 @@ def test_catalogv2_full_market_trades() -> None: catalog_market_trades = client.catalog_full_market_trades_v2(page_size=10).first_page() assert len(catalog_market_trades) == 10 assert all(['market' in catalog for catalog in catalog_market_trades]) + df_market_trades = client.catalog_full_market_trades_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_trades = client.catalog_full_market_trades_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_trades[0]) + assert set(df_market_trades.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -32,6 +41,10 @@ def test_catalogv2_full_market_candles() -> None: assert len(catalog_market_candles) == 10 assert all(['market' in catalog for catalog in catalog_market_candles]) assert all(['frequencies' in catalog for catalog in catalog_market_candles]) + df_market_candles = client.catalog_full_market_candles_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_candles = client.catalog_full_market_candles_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_candles[0]) + assert set(df_market_candles.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -40,6 +53,10 @@ def test_catalogv2_market_candles() -> None: assert len(catalog_market_candles) == 10 assert all(['market' in catalog for catalog in catalog_market_candles]) assert all(['frequencies' in catalog for catalog in catalog_market_candles]) + df_market_candles = client.catalog_market_candles_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_candles = client.catalog_market_candles_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_candles[0]) + assert set(df_market_candles.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -49,6 +66,10 @@ def test_catalogv2_contract_prices() -> None: assert all(['market' in catalog for catalog in catalog_contract_prices]) assert all(['min_time' in catalog for catalog in catalog_contract_prices]) assert all(['max_time' in catalog for catalog in catalog_contract_prices]) + df_market_contract_prices = client.catalog_market_contract_prices_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_dataframe() + list_market_contract_prices = client.catalog_market_contract_prices_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_list() + catalog_fields = get_keys_from_catalog(list_market_contract_prices[0]) + assert set(df_market_contract_prices.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -58,6 +79,10 @@ def test_catalogv2_full_contract_prices() -> None: assert all(['market' in catalog for catalog in catalog_contract_prices]) assert all(['min_time' in catalog for catalog in catalog_contract_prices]) assert all(['max_time' in catalog for catalog in catalog_contract_prices]) + df_market_contract_prices = client.catalog_full_market_contract_prices_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_dataframe() + list_market_contract_prices = client.catalog_full_market_contract_prices_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_list() + catalog_fields = get_keys_from_catalog(list_market_contract_prices[0]) + assert set(df_market_contract_prices.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -67,6 +92,15 @@ def test_catalogv2_full_market_funding_rates() -> None: assert all(['market' in catalog for catalog in catalog_funding_rates]) assert all(['min_time' in catalog for catalog in catalog_funding_rates]) assert all(['max_time' in catalog for catalog in catalog_funding_rates]) + df_market_funding_rates = client.catalog_full_market_funding_rates_v2( + markets='binance-1000BONKUSDC-future' + ).to_dataframe() + list_market_funding_rates = client.catalog_full_market_funding_rates_v2( + markets='binance-1000BONKUSDC-future' + ).to_list() + catalog_fields = get_keys_from_catalog(list_market_funding_rates[0]) + assert set(df_market_funding_rates.columns) == catalog_fields + @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_catalogv2_full_market_funding_rates_predicted() -> None: @@ -75,6 +109,15 @@ def test_catalogv2_full_market_funding_rates_predicted() -> None: assert all(['market' in catalog for catalog in catalog_funding_rates]) assert all(['min_time' in catalog for catalog in catalog_funding_rates]) assert all(['max_time' in catalog for catalog in catalog_funding_rates]) + df_market_funding_rates_predicted = client.catalog_full_market_funding_rates_predicted_v2( + markets='bybit-10000000AIDOGEUSDT-future' + ).to_dataframe() + list_market_funding_rates_predicted = client.catalog_full_market_funding_rates_predicted_v2( + markets='bybit-10000000AIDOGEUSDT-future' + ).to_list() + catalog_fields = get_keys_from_catalog(list_market_funding_rates_predicted[0]) + assert set(df_market_funding_rates_predicted.columns) == catalog_fields + @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_catalogv2_market_funding_rates() -> None: @@ -83,6 +126,15 @@ def test_catalogv2_market_funding_rates() -> None: assert all(['market' in catalog for catalog in catalog_funding_rates]) assert all(['min_time' in catalog for catalog in catalog_funding_rates]) assert all(['max_time' in catalog for catalog in catalog_funding_rates]) + df_market_funding_rates = client.catalog_market_funding_rates_v2( + markets='binance-1000BONKUSDC-future' + ).to_dataframe() + list_market_funding_rates = client.catalog_market_funding_rates_v2( + markets='binance-1000BONKUSDC-future' + ).to_list() + catalog_fields = get_keys_from_catalog(list_market_funding_rates[0]) + assert set(df_market_funding_rates.columns) == catalog_fields + @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_catalogv2_market_funding_rates_predicted() -> None: @@ -91,6 +143,15 @@ def test_catalogv2_market_funding_rates_predicted() -> None: assert all(['market' in catalog for catalog in catalog_funding_rates]) assert all(['min_time' in catalog for catalog in catalog_funding_rates]) assert all(['max_time' in catalog for catalog in catalog_funding_rates]) + df_market_funding_rates_predicted = client.catalog_market_funding_rates_predicted_v2( + markets='bybit-10000000AIDOGEUSDT-future' + ).to_dataframe() + list_market_funding_rates_predicted = client.catalog_market_funding_rates_predicted_v2( + markets='bybit-10000000AIDOGEUSDT-future' + ).to_list() + catalog_fields = get_keys_from_catalog(list_market_funding_rates_predicted[0]) + assert set(df_market_funding_rates_predicted.columns) == catalog_fields + @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_catalogv2_market_greeks() -> None: @@ -99,6 +160,10 @@ def test_catalogv2_market_greeks() -> None: assert all(['market' in catalog for catalog in catalog_greeks]) assert all(['min_time' in catalog for catalog in catalog_greeks]) assert all(['max_time' in catalog for catalog in catalog_greeks]) + df_market_greeks = client.catalog_market_greeks_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_dataframe() + list_market_greeks = client.catalog_market_greeks_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_list() + catalog_fields = get_keys_from_catalog(list_market_greeks[0]) + assert set(df_market_greeks.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -108,6 +173,10 @@ def test_catalogv2_full_market_greeks() -> None: assert all(['market' in catalog for catalog in catalog_greeks]) assert all(['min_time' in catalog for catalog in catalog_greeks]) assert all(['max_time' in catalog for catalog in catalog_greeks]) + df_market_greeks = client.catalog_full_market_greeks_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_dataframe() + list_market_greeks = client.catalog_full_market_greeks_v2(markets="deribit-BTC-15OCT21-60000-C-option").to_list() + catalog_fields = get_keys_from_catalog(list_market_greeks[0]) + assert set(df_market_greeks.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -117,6 +186,12 @@ def test_catalogv2_full_market_implied_volatility() -> None: assert all(['market' in catalog for catalog in catalog_market_implied_vol]) assert all(['min_time' in catalog for catalog in catalog_market_implied_vol]) assert all(['max_time' in catalog for catalog in catalog_market_implied_vol]) + df_market_implied_volatility = client.catalog_full_market_implied_volatility_v2( + markets='binance-BNB-240620-550-C-option', page_size=10000).to_dataframe() + list_market_implied_volatility = client.catalog_full_market_implied_volatility_v2( + markets='binance-BNB-240620-550-C-option', page_size=10000).to_list() + catalog_fields = get_keys_from_catalog(list_market_implied_volatility[0]) + assert set(df_market_implied_volatility.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -126,6 +201,12 @@ def test_catalogv2_market_implied_volatility() -> None: assert all(['market' in catalog for catalog in catalog_market_implied_vol]) assert all(['min_time' in catalog for catalog in catalog_market_implied_vol]) assert all(['max_time' in catalog for catalog in catalog_market_implied_vol]) + df_market_implied_volatility = client.catalog_market_implied_volatility_v2( + markets='binance-BNB-240620-550-C-option', page_size=10000).to_dataframe() + list_market_implied_volatility = client.catalog_market_implied_volatility_v2( + markets='binance-BNB-240620-550-C-option', page_size=10000).to_list() + catalog_fields = get_keys_from_catalog(list_market_implied_volatility[0]) + assert set(df_market_implied_volatility.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -135,6 +216,10 @@ def test_catalogv2_market_liquidations() -> None: assert all(['market' in catalog for catalog in catalog_market_liquidations]) assert all(['min_time' in catalog for catalog in catalog_market_liquidations]) assert all(['max_time' in catalog for catalog in catalog_market_liquidations]) + df_market_liquidations = client.catalog_market_liquidations_v2(markets="binance-BTCUSDT-future").to_dataframe() + list_market_liquidations = client.catalog_market_liquidations_v2(markets="binance-BTCUSDT-future").to_list() + catalog_fields = get_keys_from_catalog(list_market_liquidations[0]) + assert set(df_market_liquidations.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -144,6 +229,10 @@ def test_catalogv2_full_market_liquidations() -> None: assert all(['market' in catalog for catalog in catalog_market_liquidations]) assert all(['min_time' in catalog for catalog in catalog_market_liquidations]) assert all(['max_time' in catalog for catalog in catalog_market_liquidations]) + df_market_liquidations = client.catalog_full_market_liquidations_v2(markets="binance-BTCUSDT-future").to_dataframe() + list_market_liquidations = client.catalog_full_market_liquidations_v2(markets="binance-BTCUSDT-future").to_list() + catalog_fields = get_keys_from_catalog(list_market_liquidations[0]) + assert set(df_market_liquidations.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -153,6 +242,10 @@ def test_catalogv2_full_market_openinterest() -> None: assert all(['market' in catalog for catalog in catalog_market_openinterest]) assert all(['min_time' in catalog for catalog in catalog_market_openinterest]) assert all(['max_time' in catalog for catalog in catalog_market_openinterest]) + df_market_open_interest = client.catalog_full_market_open_interest_v2(markets="binance-BTCUSDT-future").to_dataframe() + list_market_open_interest = client.catalog_full_market_open_interest_v2(markets="binance-BTCUSDT-future").to_list() + catalog_fields = get_keys_from_catalog(list_market_open_interest[0]) + assert set(df_market_open_interest.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -162,6 +255,10 @@ def test_catalogv2_market_openinterest() -> None: assert all(['market' in catalog for catalog in catalog_market_openinterest]) assert all(['min_time' in catalog for catalog in catalog_market_openinterest]) assert all(['max_time' in catalog for catalog in catalog_market_openinterest]) + df_market_open_interest = client.catalog_market_open_interest_v2(markets="binance-BTCUSDT-future").to_dataframe() + list_market_open_interest = client.catalog_market_open_interest_v2(markets="binance-BTCUSDT-future").to_list() + catalog_fields = get_keys_from_catalog(list_market_open_interest[0]) + assert set(df_market_open_interest.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -170,6 +267,10 @@ def test_catalogv2_market_orderbooks() -> None: assert len(catalog_market_orderbooks) == 10 assert all(['market' in catalog for catalog in catalog_market_orderbooks]) assert all(['depths' in catalog for catalog in catalog_market_orderbooks]) + df_market_orderbooks = client.catalog_market_orderbooks_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_orderbooks = client.catalog_market_orderbooks_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_orderbooks[0]) + assert set(df_market_orderbooks.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -179,6 +280,10 @@ def test_catalogv2_market_quotes() -> None: assert all(['market' in catalog for catalog in catalog_market_quotes]) assert all(['min_time' in catalog for catalog in catalog_market_quotes]) assert all(['max_time' in catalog for catalog in catalog_market_quotes]) + df_market_quotes = client.catalog_market_quotes_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_quotes = client.catalog_market_quotes_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_quotes[0]) + assert set(df_market_quotes.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -188,6 +293,10 @@ def test_catalogv2_full_market_quotes() -> None: assert all(['market' in catalog for catalog in catalog_market_quotes]) assert all(['min_time' in catalog for catalog in catalog_market_quotes]) assert all(['max_time' in catalog for catalog in catalog_market_quotes]) + df_market_quotes = client.catalog_full_market_quotes_v2(markets="coinbase-btc-usd-spot").to_dataframe() + list_market_quotes = client.catalog_full_market_quotes_v2(markets="coinbase-btc-usd-spot").to_list() + catalog_fields = get_keys_from_catalog(list_market_quotes[0]) + assert set(df_market_quotes.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -196,6 +305,11 @@ def test_catalogv2_full_market_orderbooks() -> None: assert len(catalog_market_orderbooks) == 10 assert all(['market' in catalog for catalog in catalog_market_orderbooks]) assert all(['depths' in catalog for catalog in catalog_market_orderbooks]) + list_market_orderbooks = client.catalog_full_market_orderbooks_v2(markets="coinbase-btc-usd-spot").to_list() + df_market_orderbooks = client.catalog_full_market_orderbooks_v2(markets="coinbase-btc-usd-spot").to_dataframe() + catalog_fields = get_keys_from_catalog(list_market_orderbooks[0]) + assert set(df_market_orderbooks.columns) == catalog_fields + @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_catalogv2_full_market_metrics() -> None: @@ -203,6 +317,14 @@ def test_catalogv2_full_market_metrics() -> None: assert len(catalog_market_metrics) == 10 assert all(['market' in catalog for catalog in catalog_market_metrics]) assert all(['metrics' in catalog for catalog in catalog_market_metrics]) + list_market_metrics = client.catalog_full_market_metrics_v2( + markets='coinbase-btc-usd-spot', metrics='liquidity_slippage_10K_ask_percent' + ).to_list() + df_market_metrics = client.catalog_full_market_metrics_v2( + markets='coinbase-btc-usd-spot', metrics='liquidity_slippage_10K_ask_percent' + ).to_dataframe() + catalog_fields = get_keys_from_catalog(list_market_metrics[0]) + assert set(df_market_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -211,6 +333,14 @@ def test_catalogv2_market_metrics() -> None: assert len(catalog_market_metrics) == 10 assert all(['market' in catalog for catalog in catalog_market_metrics]) assert all(['metrics' in catalog for catalog in catalog_market_metrics]) + list_market_metrics = client.catalog_market_metrics_v2( + markets='coinbase-btc-usd-spot', metrics='liquidity_slippage_10K_ask_percent' + ).to_list() + df_market_metrics = client.catalog_market_metrics_v2( + markets='coinbase-btc-usd-spot', metrics='liquidity_slippage_10K_ask_percent' + ).to_dataframe() + catalog_fields = get_keys_from_catalog(list_market_metrics[0]) + assert set(df_market_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -219,6 +349,10 @@ def test_catalogv2_asset_metrics() -> None: assert len(catalog_asset_metrics) == 10 assert all(['asset' in catalog for catalog in catalog_asset_metrics]) assert all(['metrics' in catalog for catalog in catalog_asset_metrics]) + list_asset_metrics = client.catalog_asset_metrics_v2(assets='btc', metrics='PriceUSD').to_list() + df_asset_metrics = client.catalog_asset_metrics_v2(assets='btc', metrics='PriceUSD').to_dataframe() + catalog_fields = get_keys_from_catalog(list_asset_metrics[0]) + assert set(df_asset_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -227,6 +361,10 @@ def test_catalogv2_full_asset_metrics() -> None: assert len(catalog_asset_metrics) == 10 assert all(['asset' in catalog for catalog in catalog_asset_metrics]) assert all(['metrics' in catalog for catalog in catalog_asset_metrics]) + list_asset_metrics = client.catalog_full_asset_metrics_v2(assets='btc', metrics='PriceUSD').to_list() + df_asset_metrics = client.catalog_full_asset_metrics_v2(assets='btc', metrics='PriceUSD').to_dataframe() + catalog_fields = get_keys_from_catalog(list_asset_metrics[0]) + assert set(df_asset_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -235,6 +373,15 @@ def test_catalogv2_exchange_metrics() -> None: assert len(catalog_exchange_metrics) == 10 assert all(['exchange' in catalog for catalog in catalog_exchange_metrics]) assert all(['metrics' in catalog for catalog in catalog_exchange_metrics]) + list_exchange_metrics = client.catalog_exchange_metrics_v2( + exchanges='binance', metrics='volume_reported_spot_usd_1h' + ).to_list() + df_exchange_metrics = client.catalog_exchange_metrics_v2( + exchanges='binance', + metrics='volume_reported_spot_usd_1h' + ).to_dataframe() + catalog_fields = get_keys_from_catalog(list_exchange_metrics[0]) + assert set(df_exchange_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -243,7 +390,15 @@ def test_catalogv2_full_exchange_metrics() -> None: assert len(catalog_exchange_metrics) == 10 assert all(['exchange' in catalog for catalog in catalog_exchange_metrics]) assert all(['metrics' in catalog for catalog in catalog_exchange_metrics]) - + list_exchange_metrics = client.catalog_full_exchange_metrics_v2( + exchanges='binance', metrics='volume_reported_spot_usd_1h' + ).to_list() + df_exchange_metrics = client.catalog_full_exchange_metrics_v2( + exchanges='binance', + metrics='volume_reported_spot_usd_1h' + ).to_dataframe() + catalog_fields = get_keys_from_catalog(list_exchange_metrics[0]) + assert set(df_exchange_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_catalogv2_exchange_asset_metrics() -> None: @@ -251,6 +406,16 @@ def test_catalogv2_exchange_asset_metrics() -> None: assert len(catalog_exchange_metrics) == 10 assert all(['exchange_asset' in catalog for catalog in catalog_exchange_metrics]) assert all(['metrics' in catalog for catalog in catalog_exchange_metrics]) + list_exchange_asset_metrics = client.catalog_exchange_asset_metrics_v2( + exchange_assets='binance-btc', + metrics='volume_reported_spot_usd_1h' + ).to_list() + df_exchange_asset_metrics = client.catalog_exchange_asset_metrics_v2( + exchange_assets='binance-btc', + metrics='volume_reported_spot_usd_1h' + ).to_dataframe() + catalog_fields = get_keys_from_catalog(list_exchange_asset_metrics[0]) + assert set(df_exchange_asset_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -259,14 +424,28 @@ def test_catalogv2_full_exchange_asset_metrics() -> None: assert len(catalog_exchange_metrics) == 10 assert all(['exchange_asset' in catalog for catalog in catalog_exchange_metrics]) assert all(['metrics' in catalog for catalog in catalog_exchange_metrics]) + list_exchange_asset_metrics = client.catalog_full_exchange_asset_metrics_v2( + exchange_assets='binance-btc', + metrics='volume_reported_spot_usd_1h' + ).to_list() + df_exchange_asset_metrics = client.catalog_full_exchange_asset_metrics_v2( + exchange_assets='binance-btc', + metrics='volume_reported_spot_usd_1h' + ).to_dataframe() + catalog_fields = get_keys_from_catalog(list_exchange_asset_metrics[0]) + assert set(df_exchange_asset_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) -def test_catalogv2_exchange_pair_metrics() -> None: +def test_catalogv2_pair_metrics() -> None: catalog_pair_metrics = client.catalog_pair_metrics_v2(page_size=10).first_page() assert len(catalog_pair_metrics) == 10 assert all(['pair' in catalog for catalog in catalog_pair_metrics]) assert all(['metrics' in catalog for catalog in catalog_pair_metrics]) + list_pair_metrics = client.catalog_pair_metrics_v2(pairs='btc-usd', metrics='volume_reported_spot_usd_1h').to_list() + df_pair_metrics = client.catalog_pair_metrics_v2(pairs='btc-usd', metrics='volume_reported_spot_usd_1h').to_dataframe() + catalog_fields = get_keys_from_catalog(list_pair_metrics[0]) + assert set(df_pair_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -275,6 +454,10 @@ def test_catalogv2_exchange_full_pair_metrics() -> None: assert len(catalog_pair_metrics) == 10 assert all(['pair' in catalog for catalog in catalog_pair_metrics]) assert all(['metrics' in catalog for catalog in catalog_pair_metrics]) + list_pair_metrics = client.catalog_full_pair_metrics_v2(pairs='btc-usd', metrics='volume_reported_spot_usd_1h').to_list() + df_pair_metrics = client.catalog_full_pair_metrics_v2(pairs='btc-usd', metrics='volume_reported_spot_usd_1h').to_dataframe() + catalog_fields = get_keys_from_catalog(list_pair_metrics[0]) + assert set(df_pair_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -282,6 +465,10 @@ def test_catalogv2_institution_pair_metrics() -> None: catalog_institution_metrics = client.catalog_institution_metrics_v2(page_size=10).first_page() assert all(['institution' in catalog for catalog in catalog_institution_metrics]) assert all(['metrics' in catalog for catalog in catalog_institution_metrics]) + list_institution_metrics = client.catalog_institution_metrics_v2(institutions='grayscale').to_list() + df_institution_metrics = client.catalog_institution_metrics_v2(institutions='grayscale').to_dataframe() + catalog_fields = get_keys_from_catalog(list_institution_metrics[0]) + assert set(df_institution_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -289,6 +476,10 @@ def test_catalogv2_full_institution_pair_metrics() -> None: catalog_institution_metrics = client.catalog_full_institution_metrics_v2(page_size=10).first_page() assert all(['institution' in catalog for catalog in catalog_institution_metrics]) assert all(['metrics' in catalog for catalog in catalog_institution_metrics]) + list_institution_metrics = client.catalog_full_institution_metrics_v2(institutions='grayscale').to_list() + df_institution_metrics = client.catalog_full_institution_metrics_v2(institutions='grayscale').to_dataframe() + catalog_fields = get_keys_from_catalog(list_institution_metrics[0]) + assert set(df_institution_metrics.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -299,6 +490,10 @@ def test_catalogv2_pair_candles() -> None: assert all(['pair' in catalog for catalog in catalog_full_pair_candles]) assert all(['frequencies' in catalog for catalog in catalog_pair_candles]) assert all(['frequencies' in catalog for catalog in catalog_full_pair_candles]) + df_candles = client.catalog_pair_candles_v2(pairs="btc-usd").to_dataframe() + list_candles = client.catalog_pair_candles_v2(pairs="btc-usd").to_list() + catalog_fields = get_keys_from_catalog(list_candles[0]) + assert set(df_candles.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) @@ -309,6 +504,10 @@ def test_catalogv2_index_candles() -> None: assert all(['index' in catalog for catalog in catalog_full_index_candles]) assert all(['frequencies' in catalog for catalog in catalog_index_candles]) assert all(['frequencies' in catalog for catalog in catalog_full_index_candles]) + df_candles = client.catalog_index_candles_v2(indexes="CMBI10").to_dataframe() + list_candles = client.catalog_index_candles_v2(indexes="CMBI10").to_list() + catalog_fields = get_keys_from_catalog(list_candles[0]) + assert set(df_candles.columns) == catalog_fields @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)