diff --git a/coinmetrics/_data_collection.py b/coinmetrics/_data_collection.py index a58e509..6394648 100644 --- a/coinmetrics/_data_collection.py +++ b/coinmetrics/_data_collection.py @@ -5,7 +5,6 @@ import requests import itertools - from dateutil.relativedelta import relativedelta from copy import deepcopy from gzip import GzipFile @@ -13,7 +12,7 @@ from logging import getLogger from time import sleep from datetime import datetime, timedelta, date, timezone -from typing import Any, Dict, Iterable, Iterator, List, Optional, cast, Type, Callable, Union, Generator, Tuple +from typing import Any, Dict, Iterable, Iterator, List, Optional, cast, Type, Callable, Union, Generator, Tuple, TYPE_CHECKING from dateutil.parser import isoparse from coinmetrics._typing import ( DataRetrievalFuncType, @@ -28,6 +27,9 @@ from concurrent.futures import ThreadPoolExecutor, Executor from tqdm import tqdm from collections import defaultdict +from coinmetrics._exceptions import CoinMetricsClientNotFoundError +if TYPE_CHECKING: + from coinmetrics.api_client import CoinMetricsClient orjson_found = True try: @@ -74,7 +76,8 @@ def __init__( endpoint: str, url_params: Dict[str, UrlParamTypes], csv_export_supported: bool = True, - columns_to_store: List[str] = [] + columns_to_store: List[str] = [], + client: Optional[CoinMetricsClient] = None ) -> None: self._csv_export_supported = csv_export_supported self._data_retrieval_function = data_retrieval_function @@ -83,6 +86,7 @@ def __init__( self._next_page_token: Optional[str] = "" self._current_data_iterator: Optional[Iterator[Any]] = None self._columns_to_store = columns_to_store + self._client = client def first_page(self) -> List[Dict[str, Any]]: return cast( @@ -298,7 +302,8 @@ def parallel(self, executor: Optional[Callable[[Any], Executor]] = None, max_workers: Optional[int] = None, progress_bar: Optional[bool] = None, - time_increment: Optional[Union[relativedelta, timedelta]] = None + time_increment: Optional[Union[relativedelta, timedelta]] = None, + height_increment: Optional[int] = None ) -> "ParallelDataCollection": """ This method will convert the DataCollection into a ParallelDataCollection - enabling the ability to split @@ -322,7 +327,9 @@ def parallel(self, executor=executor, max_workers=max_workers, progress_bar=progress_bar, - time_increment=time_increment) + time_increment=time_increment, + height_increment=height_increment + ) class AssetChainsDataCollection(DataCollection): @@ -360,8 +367,11 @@ class ParallelDataCollection(DataCollection): """ TIME = "time" - _VALID_PARALLELIZATION_PARAMS = {'exchanges', 'assets', 'indexes', 'metrics', 'markets', 'institutions', - 'defi_protocols', 'exchange_assets', 'pairs'} + _VALID_PARALLELIZATION_PARAMS = { + 'exchanges', 'assets', 'indexes', 'metrics', 'markets', 'institutions', + 'defi_protocols', 'exchange_assets', 'pairs', 'txid', 'accounts', + 'block_hashes', 'heights', 'sub_accounts' + } _ENDPOINT_FIRST_PARAM_DICT = { 'blockchain-metadata/tags': 'type', 'blockchain-v2/{asset}/accounts': 'asset', @@ -435,12 +445,16 @@ class ParallelDataCollection(DataCollection): 'reference-data/pair-metrics': 'metrics', 'reference-data/institution-metrics': 'metrics'} - def __init__(self, parent_data_collection: DataCollection, - parallelize_on: Optional[Union[str, List[str]]] = None, - executor: Optional[Callable[..., Executor]] = None, - max_workers: Optional[int] = None, - progress_bar: Optional[bool] = None, - time_increment: Optional[Union[relativedelta, timedelta]] = None): + def __init__( + self, + parent_data_collection: DataCollection, + parallelize_on: Optional[Union[str, List[str]]] = None, + executor: Optional[Callable[..., Executor]] = None, + max_workers: Optional[int] = None, + progress_bar: Optional[bool] = None, + time_increment: Optional[Union[relativedelta, timedelta]] = None, + height_increment: Optional[int] = None + ): """ :param parallelize_on: What parameter to parallelize on. By default will use the primary query parameter in the endpoint the user is calling. For example - if the user is calling `.get_market_candles(assets="...") it will @@ -455,9 +469,12 @@ def __init__(self, parent_data_collection: DataCollection, :param time_increment: Optionally, can split the data collections by time_increment. This feature splits data collections further by time increment. So if you split by MONTH this will split a year long request into 12 smaller requests. If there is no "start_time" in the request it will raise a ValueError + :param height_increment: Optionally, can split the data collections by height_increment. This feature splits + data collections further by block height increment. If there is no "start_height" in the request it will raise a ValueError """ super().__init__(parent_data_collection._data_retrieval_function, parent_data_collection._endpoint, - parent_data_collection._url_params, parent_data_collection._csv_export_supported) + parent_data_collection._url_params, parent_data_collection._csv_export_supported, + client=parent_data_collection._client) self._parallelize_on = self._get_parallelize_on(parallelize_on) self._executor: Callable[..., Executor] = executor if executor else ThreadPoolExecutor # type: ignore self._max_workers = max_workers if max_workers else 10 @@ -466,7 +483,11 @@ def __init__(self, parent_data_collection: DataCollection, self._max_workers = 10 self._progress_bar = progress_bar if progress_bar is not None else True self._time_increment = time_increment - if self._time_increment is not None: + self._height_increment = height_increment + if self._time_increment is not None and self._height_increment is not None: + raise ValueError("time_increment and height_increment are mutually exclusive") + + elif (self._time_increment is not None) or (self._height_increment is not None): self._url_params.update({"end_inclusive": False}) def get_parallel_datacollections(self) -> List[DataCollection]: @@ -509,7 +530,6 @@ def get_parallel_datacollections(self) -> List[DataCollection]: keys = list(query_items_dict.keys()) for values_combo in itertools.product(*query_items_dict.values()): combinations.append(dict(zip(keys, values_combo))) - for combo in combinations: new_params = self._url_params.copy() new_params.update(combo) @@ -522,46 +542,114 @@ def get_parallel_datacollections(self) -> List[DataCollection]: data_collections = self._add_time_dimension_to_data_collections(data_collections=data_collections) return data_collections - def _add_time_dimension_to_data_collections(self, data_collections: List[DataCollection]) -> List[DataCollection]: + def _add_time_dimension_to_data_collections( + self, + data_collections: List[DataCollection] + ) -> List[DataCollection]: """ - Helper function to help create all possible combinations of time + parallelized parameters. Takes a list of - of data collections and returns a larger a list of dataframe over the time range. + Helper function to help create all possible combinations of time or height + parallelized parameters. Takes a list of + of data collections and returns a larger a list of dataframe over the time or height range. :param data_collections: List[DataCollection] list of data collections to be expanded :return: List[DataCollections] All combinations of the original data collections, over the specified time_increment """ - def generate_date_ranges(start: datetime, end: datetime, increment: Union[timedelta, relativedelta]) -> Generator[Tuple[datetime, datetime], None, None]: + def generate_ranges( + start: Union[datetime, int], + end: Union[datetime, int], + increment: Union[timedelta, relativedelta, int] + ) -> Generator[Tuple[datetime | int, datetime | Any | int], None, None]: + # code below can be simplified but is expanded for mypy checks current = start - if isinstance(increment, timedelta): - while current < end: - next_date = current + increment - if next_date > end: - next_date = end - yield (current, next_date) - current = next_date - elif isinstance(increment, relativedelta): - while current < end: - next_date = current + increment - if next_date > end: - next_date = end - yield (current, next_date) - current = next_date + if ( + isinstance(start, datetime) + and isinstance(end, datetime) + and isinstance(increment, (timedelta, relativedelta)) + ): + if isinstance(end, datetime) and isinstance(current, datetime): + while current < end: + if isinstance(current, datetime) and isinstance(increment, (timedelta, relativedelta)): + next_ = current + increment + if next_ > end: + next_ = end + yield (current, next_) + current = next_ + elif ( + isinstance(start, int) + and isinstance(end, int) + and isinstance(increment, int) + ): + if isinstance(current, int) and isinstance(end, int): + while current < end: # type: ignore + if isinstance(current, int) and isinstance(increment, int): + next_ = current + increment # type: ignore + if next_ > end: # type: ignore + next_ = end # type: ignore + yield (current, next_) + current = next_ else: - raise ValueError("Unsupported increment type") + raise ValueError("Unsupported combination of types for start, end, or increment") - if not self._time_increment: + if not self._time_increment and not self._height_increment: return data_collections - if not self._url_params.get("start_time"): - raise ValueError("No start_time specified, cannot use time_increment feature") - start_time = self.parse_date(cast(datetime, self._url_params.get("start_time"))) - end_time: datetime = self.parse_date(cast(datetime, self._url_params.get("end_time"))) if self._url_params.get( - "end_time") else datetime.today() full_data_collections = [] - for start, end in generate_date_ranges(start_time, end_time, increment=self._time_increment): - for data_collection in data_collections: - new_data_collection = deepcopy(data_collection) - new_data_collection._url_params.update({"start_time": start, "end_time": end}) - full_data_collections.append(new_data_collection) + if self._height_increment and isinstance(self._height_increment, int): + if self._url_params.get("start_height") and isinstance(self._url_params.get("start_height"), (int, str)): + start_height = int(self._url_params.get("start_height")) # type: ignore + else: + start_height = 0 + if self._url_params.get("end_height") and isinstance(self._url_params.get("end_height"), (int, str)): + end_height = int(self._url_params.get("end_height")) # type: ignore + else: + if self._url_params.get("asset"): + asset = str(self._url_params.get("asset")) + else: + raise ValueError( + """ + Parameter "asset" not found in request. + Note: Parallel height increment only works on a single asset. + Consider breaking query into asset-by-asset chunks (e.g. .parallel('assets').parallel(height_increment=height_increment)) + """ + ) + if self._client is not None: + block_data = self._client.get_list_of_blocks_v2(asset=asset, paging_from='end', page_size=1).first_page() + if block_data: + end_height = int(block_data[0]['height']) + else: + raise Exception(f"End height for asset {asset} not found.") + else: + raise CoinMetricsClientNotFoundError + + for start, end in generate_ranges( + start_height, + end_height, + increment=self._height_increment + ): + for data_collection in data_collections: + new_data_collection = deepcopy(data_collection) + new_data_collection._url_params.update( + {"start_height": start, "end_height": end} + ) + full_data_collections.append(new_data_collection) + elif self._time_increment and isinstance(self._time_increment, (timedelta, relativedelta)): + if not self._url_params.get("start_time"): + raise ValueError("No start_time specified, cannot use time_increment feature") + else: + start_time = self.parse_date( + cast(datetime, self._url_params.get("start_time")) + ) + end_time = self.parse_date( + cast(datetime, self._url_params.get("end_time")) + ) if self._url_params.get( + "end_time") else datetime.today() + for start, end in generate_ranges( + start_time, + end_time, + increment=self._time_increment + ): + for data_collection in data_collections: + new_data_collection = deepcopy(data_collection) + new_data_collection._url_params.update({"start_time": start, "end_time": end}) + full_data_collections.append(new_data_collection) return full_data_collections def to_list(self) -> List[Dict[str, Any]]: diff --git a/coinmetrics/_exceptions.py b/coinmetrics/_exceptions.py index 6703e56..b281b9e 100644 --- a/coinmetrics/_exceptions.py +++ b/coinmetrics/_exceptions.py @@ -50,3 +50,10 @@ def __init__(self, response: Response, *args: Any, **kwargs: Any): def __str__(self) -> str: return self.msg + + +class CoinMetricsClientNotFoundError(Exception): + """Raised when a CoinMetricsClient instance is not found.""" + def __init__(self, message="CoinMetricsClient not found"): + self.message = message + super().__init__(self.message) diff --git a/coinmetrics/api_client.py b/coinmetrics/api_client.py index f479690..0a70e8a 100644 --- a/coinmetrics/api_client.py +++ b/coinmetrics/api_client.py @@ -2014,7 +2014,7 @@ def catalog_market_trades_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-trades", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-trades", params, client=self) def catalog_market_candles_v2( self, @@ -2070,7 +2070,7 @@ def catalog_market_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-candles", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-candles", params, client=self) def catalog_market_orderbooks_v2( self, @@ -2126,7 +2126,7 @@ def catalog_market_orderbooks_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-orderbooks", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-orderbooks", params, client=self) def catalog_market_quotes_v2( self, @@ -2182,7 +2182,7 @@ def catalog_market_quotes_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-quotes", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-quotes", params, client=self) def catalog_market_funding_rates_v2( self, @@ -2238,7 +2238,7 @@ def catalog_market_funding_rates_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-funding-rates", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-funding-rates", params, client=self) def catalog_market_funding_rates_predicted_v2( self, @@ -2294,7 +2294,7 @@ def catalog_market_funding_rates_predicted_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-funding-rates-predicted", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-funding-rates-predicted", params, client=self) def catalog_market_contract_prices_v2( self, @@ -2350,7 +2350,7 @@ def catalog_market_contract_prices_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-contract-prices", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-contract-prices", params, client=self) def catalog_market_implied_volatility_v2( self, @@ -2406,7 +2406,7 @@ def catalog_market_implied_volatility_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-implied-volatility", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-implied-volatility", params, client=self) def catalog_market_greeks_v2( self, @@ -2462,7 +2462,7 @@ def catalog_market_greeks_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-greeks", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-greeks", params, client=self) def catalog_market_open_interest_v2( self, @@ -2518,7 +2518,7 @@ def catalog_market_open_interest_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-openinterest", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-openinterest", params, client=self) def catalog_market_liquidations_v2( self, @@ -2574,7 +2574,7 @@ def catalog_market_liquidations_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-liquidations", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-liquidations", params, client=self) def catalog_market_metrics_v2( self, @@ -2632,7 +2632,7 @@ def catalog_market_metrics_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/market-metrics", params, client=self) def catalog_full_market_trades_v2( self, @@ -2688,7 +2688,7 @@ def catalog_full_market_trades_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-trades", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-trades", params, client=self) def catalog_full_market_candles_v2( self, @@ -2744,7 +2744,7 @@ def catalog_full_market_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-candles", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-candles", params, client=self) def catalog_full_market_orderbooks_v2( self, @@ -2800,7 +2800,7 @@ def catalog_full_market_orderbooks_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-orderbooks", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-orderbooks", params, client=self) def catalog_full_market_quotes_v2( self, @@ -2856,7 +2856,7 @@ def catalog_full_market_quotes_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-quotes", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-quotes", params, client=self) def catalog_full_market_funding_rates_v2( self, @@ -2912,7 +2912,7 @@ def catalog_full_market_funding_rates_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-funding-rates", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-funding-rates", params, client=self) def catalog_full_market_funding_rates_predicted_v2( self, @@ -2968,7 +2968,7 @@ def catalog_full_market_funding_rates_predicted_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-funding-rates-predicted", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-funding-rates-predicted", params, client=self) def catalog_full_market_contract_prices_v2( self, @@ -3024,7 +3024,7 @@ def catalog_full_market_contract_prices_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-contract-prices", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-contract-prices", params, client=self) def catalog_full_market_implied_volatility_v2( self, @@ -3080,7 +3080,7 @@ def catalog_full_market_implied_volatility_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-implied-volatility", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-implied-volatility", params, client=self) def catalog_full_market_greeks_v2( self, @@ -3136,7 +3136,7 @@ def catalog_full_market_greeks_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-greeks", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-greeks", params, client=self) def catalog_full_market_open_interest_v2( self, @@ -3192,7 +3192,7 @@ def catalog_full_market_open_interest_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-openinterest", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-openinterest", params, client=self) def catalog_full_market_liquidations_v2( self, @@ -3248,7 +3248,7 @@ def catalog_full_market_liquidations_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-liquidations", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-liquidations", params, client=self) def catalog_full_market_metrics_v2( self, @@ -3306,7 +3306,7 @@ def catalog_full_market_metrics_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/market-metrics", params, client=self) def catalog_asset_metrics_v2( self, @@ -3346,7 +3346,7 @@ def catalog_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/asset-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/asset-metrics", params, client=self) def catalog_full_asset_metrics_v2( self, @@ -3386,7 +3386,7 @@ def catalog_full_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/asset-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/asset-metrics", params, client=self) def catalog_exchange_metrics_v2( self, @@ -3426,7 +3426,7 @@ def catalog_exchange_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/exchange-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/exchange-metrics", params, client=self) def catalog_full_exchange_metrics_v2( self, @@ -3466,7 +3466,7 @@ def catalog_full_exchange_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/exchange-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/exchange-metrics", params, client=self) def catalog_exchange_asset_metrics_v2( self, @@ -3506,7 +3506,7 @@ def catalog_exchange_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/exchange-asset-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/exchange-asset-metrics", params, client=self) def catalog_full_exchange_asset_metrics_v2( self, @@ -3546,7 +3546,7 @@ def catalog_full_exchange_asset_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/exchange-asset-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/exchange-asset-metrics", params, client=self) def catalog_pair_metrics_v2( self, @@ -3586,7 +3586,7 @@ def catalog_pair_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/pair-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/pair-metrics", params, client=self) def catalog_full_pair_metrics_v2( self, @@ -3626,7 +3626,7 @@ def catalog_full_pair_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/pair-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/pair-metrics", params, client=self) def catalog_institution_metrics_v2( self, @@ -3666,7 +3666,7 @@ def catalog_institution_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/institution-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/institution-metrics", params, client=self) def catalog_full_institution_metrics_v2( self, @@ -3706,7 +3706,7 @@ def catalog_full_institution_metrics_v2( "next_page_token": next_page_token, "format": format, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/institution-metrics", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/institution-metrics", params, client=self) def catalog_pair_candles_v2( self, @@ -3734,7 +3734,7 @@ def catalog_pair_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/pair-candles", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/pair-candles", params, client=self) def catalog_index_candles_v2( self, @@ -3762,7 +3762,7 @@ def catalog_index_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/index-candles", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/index-candles", params, client=self) def catalog_index_levels_v2( self, @@ -3790,7 +3790,7 @@ def catalog_index_levels_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/index-levels", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/index-levels", params, client=self) def catalog_asset_chains_v2( self, @@ -3818,7 +3818,7 @@ def catalog_asset_chains_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/asset-chains", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/asset-chains", params, client=self) def catalog_mempool_feerates_v2( self, @@ -3846,7 +3846,7 @@ def catalog_mempool_feerates_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/mempool-feerates", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/mempool-feerates", params, client=self) def catalog_mining_pool_tips_summaries_v2( self, @@ -3874,7 +3874,7 @@ def catalog_mining_pool_tips_summaries_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/mining-pool-tips-summary", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/mining-pool-tips-summary", params, client=self) def catalog_transaction_tracker_assets_v2( self, @@ -3902,7 +3902,7 @@ def catalog_transaction_tracker_assets_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-v2/transaction-tracker", params) + return CatalogV2DataCollection(self._get_data, "/catalog-v2/transaction-tracker", params, client=self) def catalog_full_pair_candles_v2( self, @@ -3930,7 +3930,7 @@ def catalog_full_pair_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/pair-candles", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/pair-candles", params, client=self) def catalog_full_index_candles_v2( self, @@ -3958,7 +3958,7 @@ def catalog_full_index_candles_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/index-candles", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/index-candles", params, client=self) def catalog_full_index_levels_v2( self, @@ -3986,7 +3986,7 @@ def catalog_full_index_levels_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/index-levels", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/index-levels", params, client=self) def catalog_full_asset_chains_v2( self, @@ -4014,7 +4014,7 @@ def catalog_full_asset_chains_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/asset-chains", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/asset-chains", params, client=self) def catalog_full_mempool_feerates_v2( self, @@ -4042,7 +4042,7 @@ def catalog_full_mempool_feerates_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/mempool-feerates", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/mempool-feerates", params, client=self) def catalog_full_mining_pool_tips_summaries_v2( self, @@ -4070,7 +4070,7 @@ def catalog_full_mining_pool_tips_summaries_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/mining-pool-tips-summary", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/mining-pool-tips-summary", params, client=self) def catalog_full_transaction_tracker_assets_v2( self, @@ -4098,7 +4098,7 @@ def catalog_full_transaction_tracker_assets_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/transaction-tracker", params) + return CatalogV2DataCollection(self._get_data, "/catalog-all-v2/transaction-tracker", params, client=self) def get_asset_alerts( self, @@ -4152,7 +4152,7 @@ def get_asset_alerts( "timezone": timezone, "include_heartbeats": include_heartbeats } - return DataCollection(self._get_data, "timeseries/asset-alerts", params) + return DataCollection(self._get_data, "timeseries/asset-alerts", params, client=self) def get_defi_balance_sheets( self, @@ -4205,7 +4205,7 @@ def get_defi_balance_sheets( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, "timeseries/defi-balance-sheets", params) + return DataCollection(self._get_data, "timeseries/defi-balance-sheets", params, client=self) def get_asset_chains( self, @@ -4251,7 +4251,7 @@ def get_asset_chains( "end_inclusive": end_inclusive, "timezone": timezone, } - return AssetChainsDataCollection(self._get_data, "timeseries/asset-chains", params) + return AssetChainsDataCollection(self._get_data, "timeseries/asset-chains", params, client=self) def get_asset_metrics( self, @@ -4353,7 +4353,7 @@ def get_asset_metrics( "ignore_forbidden_errors": ignore_forbidden_errors, "ignore_unsupported_errors": ignore_unsupported_errors } - return DataCollection(self._get_data, "timeseries/asset-metrics", params) + return DataCollection(self._get_data, "timeseries/asset-metrics", params, client=self) def get_exchange_metrics( self, @@ -4423,7 +4423,7 @@ def get_exchange_metrics( "sort": sort, "limit_per_exchange": limit_per_exchange, } - return DataCollection(self._get_data, "timeseries/exchange-metrics", params) + return DataCollection(self._get_data, "timeseries/exchange-metrics", params, client=self) def get_exchange_asset_metrics( self, @@ -4565,7 +4565,7 @@ def get_pair_metrics( "sort": sort, "limit_per_pair": limit_per_pair, } - return DataCollection(self._get_data, "timeseries/pair-metrics", params) + return DataCollection(self._get_data, "timeseries/pair-metrics", params, client=self) def get_pair_candles( self, @@ -4628,7 +4628,7 @@ def get_pair_candles( "timezone": timezone, "limit_per_pair": limit_per_pair, } - return DataCollection(self._get_data, "timeseries/pair-candles", params) + return DataCollection(self._get_data, "timeseries/pair-candles", params, client=self) def get_institution_metrics( self, @@ -4698,7 +4698,7 @@ def get_institution_metrics( "sort": sort, "limit_per_institution": limit_per_institution, } - return DataCollection(self._get_data, "timeseries/institution-metrics", params) + return DataCollection(self._get_data, "timeseries/institution-metrics", params, client=self) def get_index_candles( self, @@ -4752,7 +4752,7 @@ def get_index_candles( "timezone": timezone, "limit_per_index": limit_per_index, } - return DataCollection(self._get_data, "timeseries/index-candles", params) + return DataCollection(self._get_data, "timeseries/index-candles", params, client=self) def get_index_levels( self, @@ -4810,7 +4810,7 @@ def get_index_levels( "limit_per_index": limit_per_index, "include_verification": include_verification } - return DataCollection(self._get_data, "timeseries/index-levels", params) + return DataCollection(self._get_data, "timeseries/index-levels", params, client=self) def get_index_constituents( self, @@ -4860,7 +4860,7 @@ def get_index_constituents( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, "timeseries/index-constituents", params) + return DataCollection(self._get_data, "timeseries/index-constituents", params, client=self) def get_market_metrics( self, @@ -4925,7 +4925,7 @@ def get_market_metrics( "limit_per_market": limit_per_market, "sort": sort, } - return DataCollection(self._get_data, "timeseries/market-metrics", params) + return DataCollection(self._get_data, "timeseries/market-metrics", params, client=self) def get_market_candles( self, @@ -4980,7 +4980,7 @@ def get_market_candles( "timezone": timezone, "limit_per_market": limit_per_market, } - return DataCollection(self._get_data, "timeseries/market-candles", params) + return DataCollection(self._get_data, "timeseries/market-candles", params, client=self) def get_market_trades( self, @@ -5035,7 +5035,7 @@ def get_market_trades( "limit_per_market": limit_per_market, "min_confirmations": min_confirmations, } - return DataCollection(self._get_data, "timeseries/market-trades", params) + return DataCollection(self._get_data, "timeseries/market-trades", params, client=self) def get_market_open_interest( self, @@ -5090,7 +5090,7 @@ def get_market_open_interest( "timezone": timezone, "limit_per_market": limit_per_market, } - return DataCollection(self._get_data, "timeseries/market-openinterest", params) + return DataCollection(self._get_data, "timeseries/market-openinterest", params, client=self) def get_market_liquidations( self, @@ -5141,7 +5141,7 @@ def get_market_liquidations( "timezone": timezone, "limit_per_market": limit_per_market, } - return DataCollection(self._get_data, "timeseries/market-liquidations", params) + return DataCollection(self._get_data, "timeseries/market-liquidations", params, client=self) def get_market_funding_rates( self, @@ -5192,7 +5192,7 @@ def get_market_funding_rates( "timezone": timezone, "limit_per_market": limit_per_market, } - return DataCollection(self._get_data, "timeseries/market-funding-rates", params) + return DataCollection(self._get_data, "timeseries/market-funding-rates", params, client=self) def get_predicted_market_funding_rates( self, @@ -5243,7 +5243,7 @@ def get_predicted_market_funding_rates( "timezone": timezone, "limit_per_market": limit_per_market, } - return DataCollection(self._get_data, "timeseries/market-funding-rates-predicted", params) + return DataCollection(self._get_data, "timeseries/market-funding-rates-predicted", params, client=self) def get_market_orderbooks( self, @@ -5302,7 +5302,7 @@ def get_market_orderbooks( "depth_limit": depth_limit, "timezone": timezone, } - return DataCollection(self._get_data, "timeseries/market-orderbooks", params) + return DataCollection(self._get_data, "timeseries/market-orderbooks", params, client=self) def get_market_quotes( self, @@ -5361,7 +5361,7 @@ def get_market_quotes( "limit_per_market": limit_per_market, "include_one_sided": include_one_sided, } - return DataCollection(self._get_data, "timeseries/market-quotes", params) + return DataCollection(self._get_data, "timeseries/market-quotes", params, client=self) def get_market_contract_prices( self, @@ -5529,7 +5529,7 @@ def get_market_greeks( "timezone": timezone, "limit_per_market": limit_per_market, } - return DataCollection(self._get_data, "timeseries/market-greeks", params) + return DataCollection(self._get_data, "timeseries/market-greeks", params, client=self) def get_mining_pool_tips_summary( self, @@ -5623,7 +5623,7 @@ def get_mempool_feerates( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, "timeseries/mempool-feerates", params) + return DataCollection(self._get_data, "timeseries/mempool-feerates", params, client=self) def get_stream_asset_metrics( self, @@ -5922,7 +5922,7 @@ def get_list_of_blocks( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, f"blockchain/{asset}/blocks", params) + return DataCollection(self._get_data, f"blockchain/{asset}/blocks", params, client=self) def get_list_of_accounts( self, @@ -5987,7 +5987,7 @@ def get_list_of_accounts( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, f"blockchain/{asset}/accounts", params) + return DataCollection(self._get_data, f"blockchain/{asset}/accounts", params, client=self) def get_list_of_transactions( self, @@ -6190,7 +6190,7 @@ def get_list_of_blocks_v2( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, f"blockchain-v2/{asset}/blocks", params) + return DataCollection(self._get_data, f"blockchain-v2/{asset}/blocks", params, client=self) def get_list_of_accounts_v2( self, @@ -6255,7 +6255,7 @@ def get_list_of_accounts_v2( "end_inclusive": end_inclusive, "timezone": timezone, } - return DataCollection(self._get_data, f"blockchain-v2/{asset}/accounts", params) + return DataCollection(self._get_data, f"blockchain-v2/{asset}/accounts", params, client=self) def get_list_of_sub_accounts_v2( self, @@ -6731,7 +6731,7 @@ def get_list_of_balance_updates_for_account_v2( "paging_from": paging_from, "next_page_token": next_page_token, } - return DataCollection(self._get_data, f"blockchain-v2/{asset}/accounts/{account}/balance-updates", params) + return DataCollection(self._get_data, f"blockchain-v2/{asset}/accounts/{account}/balance-updates", params, client=self) def get_transaction_tracker( self, @@ -6852,7 +6852,7 @@ def get_taxonomy_assets( "paging_from": paging_from, "version": version, } - return DataCollection(self._get_data, "/taxonomy/assets", params) + return DataCollection(self._get_data, "/taxonomy/assets", params, client=self) def get_taxonomy_assets_metadata( self, @@ -6892,7 +6892,7 @@ def get_taxonomy_assets_metadata( "paging_from": paging_from, "version": version, } - return DataCollection(self._get_data, "/taxonomy-metadata/assets", params) + return DataCollection(self._get_data, "/taxonomy-metadata/assets", params, client=self) def get_asset_profiles( self, @@ -6918,7 +6918,7 @@ def get_asset_profiles( "page_size": page_size, "paging_from": paging_from, } - return DataCollection(self._get_data, "/profile/assets", params) + return DataCollection(self._get_data, "/profile/assets", params, client=self) def reference_data_asset_metrics( self, @@ -7474,7 +7474,7 @@ def get_snapshots_of_asset_metric_constituents( "page_size": page_size, "paging_from": paging_from, } - return DataCollection(self._get_data, "/constituent-snapshots/asset-metrics", params) + return DataCollection(self._get_data, "/constituent-snapshots/asset-metrics", params, client=self) def get_timeframes_of_asset_metric_constituents( self, @@ -7522,7 +7522,7 @@ def get_timeframes_of_asset_metric_constituents( "page_size": page_size, "paging_from": paging_from, } - return DataCollection(self._get_data, "/constituent-timeframes/asset-metrics", params) + return DataCollection(self._get_data, "/constituent-timeframes/asset-metrics", params, client=self) def blockchain_metadata_tags( self, @@ -7546,7 +7546,7 @@ def blockchain_metadata_tags( "page_size": page_size, "next_page_token": next_page_token, } - return DataCollection(self._get_data, "blockchain-metadata/tags", params) + return DataCollection(self._get_data, "blockchain-metadata/tags", params, client=self) def blockchain_metadata_tagged_entities( self, @@ -7578,7 +7578,7 @@ def blockchain_metadata_tagged_entities( "page_size": page_size, "next_page_token": next_page_token, } - return DataCollection(self._get_data, "blockchain-metadata/tagged-entities", params) + return DataCollection(self._get_data, "blockchain-metadata/tagged-entities", params, client=self) def _get_data(self, url: str, params: Dict[str, Any]) -> DataReturnType: if params: diff --git a/test/test_blockchain_methods.py b/test/test_blockchain_methods.py index 5910064..537797c 100644 --- a/test/test_blockchain_methods.py +++ b/test/test_blockchain_methods.py @@ -13,18 +13,6 @@ "CM_API_KEY not set, tests will not run" ) -@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) -def test_get_full_transaction_for_block() -> None: - asset = "btc" - block_hash = "0000000000000000000334e8637314d72d86a533c71f48da23e85e70a82cd38a" - transaction_id = "29d401526b06d55749034c10c3ee7ffd9ecab942c9b6852c963fa61103552729" - transaction = client.get_full_transaction_for_block( - asset=asset, block_hash=block_hash, txid=transaction_id - ) - as_list = transaction - print(as_list) - assert len(as_list) >= 1 - @pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) def test_get_blockchain_list_of_account_balance_updates() -> None: @@ -34,14 +22,5 @@ def test_get_blockchain_list_of_account_balance_updates() -> None: assert updates[0]['block_hash'] == "0000000000000000002c7505ef2272e0677fa53d68d633f8e076ed42dd3380e6" -@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) -def test_get_full_block_v1() -> None: - asset = "eth" - block_hash = "0x27a2bd0fd3b3298dd8004c18aaad83374bdc5dbd36eac46bfe00772d88dba7cf" - full_block = client.get_full_block(asset=asset, block_hash=block_hash) - assert full_block['block_hash'] == block_hash.strip("0x") - assert full_block['height'] == '15442095' - - if __name__ == "__main__": pytest.main() diff --git a/test/test_parallel_datacollections.py b/test/test_parallel_datacollections.py index 17b83ed..5e03edd 100644 --- a/test/test_parallel_datacollections.py +++ b/test/test_parallel_datacollections.py @@ -281,6 +281,18 @@ def test_parse_date() -> None: assert pdc.parse_date(t_naive_s) == datetime.datetime(year=2024, month=1, day=2, hour=0, minute=3, second=4) assert pdc.parse_date(t_naive_sz) == datetime.datetime(year=2024, month=1, day=2, hour=0, minute=3, second=4) + +@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP) +def test_height_increment() -> None: + df_blocks = client.get_list_of_blocks_v2( + asset="btc", + start_height=10, + end_height=100, + end_inclusive=False + ).parallel(height_increment=10).to_dataframe() + assert len(df_blocks) == 90 + + if __name__ == '__main__': pytest.main()