Skip to content

Commit

Permalink
PLAT-1167 remove column hardcoding
Browse files Browse the repository at this point in the history 
This MR removes the hardcoding for columns when calling to_dataframe for reference-data endpoints.
  • Loading branch information
@victoreram
victoreram committed Dec 10, 2024
1 parent adcf70f commit d7c8086
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 64 deletions.
16 changes: 13 additions & 3 deletions coinmetrics/_data_collection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,9 @@ def __init__(
url_params: Dict[str, UrlParamTypes],
csv_export_supported: bool = True,
columns_to_store: List[str] = [],
client: Optional[CoinMetricsClient] = None
client: Optional[CoinMetricsClient] = None,
optimize_pandas_types: Optional[bool] = True,
dtype_mapper: Optional[Dict[str, Any]] = None
) -> None:
self._csv_export_supported = csv_export_supported
self._data_retrieval_function = data_retrieval_function
Expand All @@ -89,6 +91,8 @@ def __init__(
self._current_data_iterator: Optional[Iterator[Any]] = None
self._columns_to_store = columns_to_store
self._client = client
self._optimize_pandas_types = optimize_pandas_types
self._dtype_mapper = dtype_mapper

def first_page(self) -> List[Dict[str, Any]]:
return cast(
Expand Down Expand Up @@ -245,7 +249,7 @@ def to_dataframe(
self,
header: Optional[List[str]] = None,
dtype_mapper: Optional[Dict[str, Any]] = None,
optimize_pandas_types: Optional[bool] = True,
optimize_pandas_types: Optional[bool] = None,
) -> DataFrameType:
"""
Outputs a pandas dataframe.
Expand All @@ -259,6 +263,10 @@ def to_dataframe(
:return: Data in a pandas dataframe
:rtype: DataFrameType
"""
if optimize_pandas_types is None:
optimize_pandas_types = self._optimize_pandas_types
if dtype_mapper is None:
dtype_mapper = self._dtype_mapper
if pd is None:
logger.info("Pandas not found; Returning None")
return None
Expand Down Expand Up @@ -305,7 +313,9 @@ def to_dataframe(
if dtype_mapper is None:
return pd.DataFrame(self)
else:
return pd.DataFrame(self).astype(dtype=dtype_mapper)
df = pd.DataFrame(self)
dtype_mapper = {key: value for key, value in dtype_mapper.items() if key in df.columns}
return df.astype(dtype_mapper)

def parallel(self,
parallelize_on: Optional[Union[str, List[str]]] = None,
Expand Down
156 changes: 95 additions & 61 deletions coinmetrics/api_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6990,10 +6990,15 @@ def reference_data_asset_metrics(
self._get_data,
"reference-data/asset-metrics",
params,
columns_to_store=[
'metric', 'full_name', 'description', 'product', 'category',
'subcategory', 'unit', 'data_type', 'type'
]
optimize_pandas_types=False,
dtype_mapper={
"product": "category",
"category": "category",
"subcategory": "category",
"unit": "category",
"data_type": "category",
"type": "category"
}
)

def reference_data_markets(
Expand Down Expand Up @@ -7054,14 +7059,41 @@ def reference_data_markets(
self._get_data,
"reference-data/markets",
params,
columns_to_store=[
'market', 'exchange', 'base', 'quote', 'pair', 'symbol', 'type', 'size_asset', 'margin_asset',
'strike', 'option_contract_type', 'is_european', 'contract_size', 'tick_size', 'multiplier_size',
'listing', 'expiration', 'settlement_price', 'pool_config_id', 'contract_address', 'fee',
'price_includes_fee', 'variable_fee', 'base_address', 'quote_address', 'status',
'order_amount_increment', 'order_amount_min', 'order_amount_max', 'order_price_increment',
'order_price_min', 'order_price_max', 'order_size_min', 'order_taker_fee', 'order_maker_fee',
'margin_trading_enabled', 'experimental', 'base_native', 'quote_native']
optimize_pandas_types=False,
dtype_mapper={
"exchange": "category",
"base": "category",
"quote": "category",
"pair": "category",
"type": "category",
"size_asset": "category",
"margin_asset": "category",
"strike": "Float64",
"option_contract_type": "category",
"is_european": "boolean",
"contract_size": "Float64",
"tick_size": "Float64",
"multiplier_size": "Float64",
"expiration": "datetime64[ns, UTC]",
"listing": "datetime64[ns, UTC]",
"settlement_price": "Float64",
"pool_config_id": "Int64",
"fee": "Float64",
"price_includes_fee": "boolean",
"variable_fee": "boolean",
"order_amount_increment": "Float64",
"order_amount_min": "Float64",
"order_amount_max": "Float64",
"order_price_increment": "Float64",
"order_price_min": "Float64",
"order_price_max": "Float64",
"order_taker_fee": "Float64",
"order_maker_fee": "Float64",
"margin_trading_enabled": "boolean",
"experimental": "boolean",
"base_native": "category",
"quote_native": "category"
}
)

def reference_data_exchange_metrics(
Expand Down Expand Up @@ -7094,10 +7126,15 @@ def reference_data_exchange_metrics(
self._get_data,
"reference-data/exchange-metrics",
params,
columns_to_store=[
'metric', 'full_name', 'description', 'product', 'category',
'subcategory', 'unit', 'data_type', 'type'
]
optimize_pandas_types=False,
dtype_mapper={
"product": "category",
"category": "category",
"subcategory": "category",
"unit": "category",
"data_type": "category",
"type": "category"
}
)

def reference_data_exchange_asset_metrics(
Expand Down Expand Up @@ -7130,10 +7167,15 @@ def reference_data_exchange_asset_metrics(
self._get_data,
"reference-data/exchange-asset-metrics",
params,
columns_to_store=[
'metric', 'full_name', 'description', 'product', 'category',
'subcategory', 'unit', 'data_type', 'type'
]
optimize_pandas_types=False,
dtype_mapper={
"product": "category",
"category": "category",
"subcategory": "category",
"unit": "category",
"data_type": "category",
"type": "category"
}
)

def reference_data_pair_metrics(
Expand Down Expand Up @@ -7166,10 +7208,15 @@ def reference_data_pair_metrics(
self._get_data,
"reference-data/pair-metrics",
params,
columns_to_store=[
'metric', 'full_name', 'description', 'product', 'category',
'subcategory', 'unit', 'data_type', 'type'
]
optimize_pandas_types=False,
dtype_mapper={
"product": "category",
"category": "category",
"subcategory": "category",
"unit": "category",
"data_type": "category",
"type": "category"
}
)

def reference_data_institution_metrics(
Expand Down Expand Up @@ -7202,10 +7249,15 @@ def reference_data_institution_metrics(
self._get_data,
"reference-data/institution-metrics",
params,
columns_to_store=[
'metric', 'full_name', 'description', 'product', 'category',
'subcategory', 'unit', 'data_type', 'type'
]
optimize_pandas_types=False,
dtype_mapper={
"product": "category",
"category": "category",
"subcategory": "category",
"unit": "category",
"data_type": "category",
"type": "category"
}
)

def reference_data_assets(
Expand Down Expand Up @@ -7242,7 +7294,7 @@ def reference_data_assets(
self._get_data,
"reference-data/assets",
params,
columns_to_store=['asset', 'full_name']
optimize_pandas_types=False
)

def reference_data_exchanges(
Expand Down Expand Up @@ -7279,7 +7331,7 @@ def reference_data_exchanges(
self._get_data,
"reference-data/exchanges",
params,
columns_to_store=['exchange', 'full_name']
optimize_pandas_types=False
)

def reference_data_indexes(
Expand Down Expand Up @@ -7316,7 +7368,7 @@ def reference_data_indexes(
self._get_data,
"reference-data/indexes",
params,
columns_to_store=["index", "full_name"]
optimize_pandas_types=False
)

def reference_data_pairs(
Expand Down Expand Up @@ -7353,7 +7405,7 @@ def reference_data_pairs(
self._get_data,
"reference-data/pairs",
params,
columns_to_store=["pair", "full_name"]
optimize_pandas_types=False
)

def reference_data_market_metrics(
Expand Down Expand Up @@ -7390,10 +7442,15 @@ def reference_data_market_metrics(
self._get_data,
"reference-data/market-metrics",
params,
columns_to_store=[
'metric', 'full_name', 'description', 'product', 'category',
'subcategory', 'unit', 'data_type', 'type'
]
optimize_pandas_types=False,
dtype_mapper={
"product": "category",
"category": "category",
"subcategory": "category",
"unit": "category",
"data_type": "category",
"type": "category"
}
)

def security_master_assets(
Expand Down Expand Up @@ -7430,11 +7487,7 @@ def security_master_assets(
self._get_data,
"security-master/assets",
params,
columns_to_store=[
'asset', 'code', 'description', 'overview', 'website', 'whitepaper',
'consensus_mechanism', 'decimals', 'creation_date', 'type', 'parent_asset',
'pricing_asset', 'erc20_token_contract', 'fiat'
]
optimize_pandas_types=False
)

def security_master_markets(
Expand Down Expand Up @@ -7487,26 +7540,7 @@ def security_master_markets(
self._get_data,
"security-master/markets",
params,
columns_to_store=[
'market', 'code', 'pair', 'trades_min_time', 'trades_max_time',
'orderbooks_min_time', 'orderbooks_max_time',
'quotes_min_time', 'quotes_max_time',
'funding_rates_min_time', 'funding_rates_max_time',
'openinterest_min_time', 'openinterest_max_time',
'liquidations_min_time', 'liquidations_max_time',
'exchange', 'base', 'quote', 'symbol', 'type',
'size_asset', 'margin_asset', 'strike', 'option_contract_type',
'is_european', 'contract_size', 'tick_size', 'multiplier_size',
'listing', 'expiration', 'settlement_price', 'pool_config_id',
'contract_address', 'fee', 'price_includes_fee', 'variable_fee',
'base_address', 'quote_address', 'status', 'order_amount_increment',
'order_amount_min', 'order_amount_max', 'order_price_increment',
'order_price_min', 'order_price_max', 'order_size_min',
'order_taker_fee', 'order_maker_fee', 'margin_trading_enabled',
'experimental', 'price_open', 'price_close', 'price_high',
'price_low', 'vwap', 'volume', 'candle_usd_volume', 'candle_trades_count',
'base_native', 'quote_native'
]
optimize_pandas_types=False
)

def get_snapshots_of_asset_metric_constituents(
Expand Down

0 comments on commit d7c8086

Please sign in to comment.