diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..493bf2d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,42 @@ +name: Build and Deploy to Cloud Run + +on: + pull_request: + branches: + - main + types: + - closed + + +jobs: + build-and-deploy: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + environment: + name: production + url: ${{ steps.deploy.outputs.url }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + # Configure Workload Identity Federation and generate an access token. + # + # See https://github.com/google-github-actions/auth for more options, + # including authenticating via a JSON credentials file. + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_KEY }} + + - name: Configure Docker + run: gcloud auth configure-docker + + - name: Build Docker image + run: | + docker build -t gcr.io/dfg-analytics-insights-prod/us-department-treasury:latest . + + - name: Push Docker image to GCR + run: | + docker push gcr.io/dfg-analytics-insights-prod/us-department-treasury:latest diff --git a/.gitignore b/.gitignore index e69a119..0cc436c 100644 --- a/.gitignore +++ b/.gitignore @@ -161,5 +161,5 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/README.md b/README.md index 5ec2d6f..5d58e08 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,33 @@ # us-department-treasury -### us-department-treasury - yield curves and such +## Project Overview +This project automates the process of fetching, processing, and storing daily Treasury Par Yield Curve Rates data from +the US Department of the Treasury. It provides a reliable and up-to-date source of Treasury yield curve data for +financial analysis and research purposes. +### What it does -### Daily Treasury Par Yield Curve Rates - Website -https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value=2024 +1. Data Retrieval: The project downloads the latest Daily Treasury Par Yield Curve Rates data from the US Department of + the Treasury website. +2. Data Storage: The processed data is committed to a GitHub repository, creating a historical record of Treasury yield + curve rates. +3. Automated Execution: The entire process is automated to run daily, ensuring the dataset is always current. +### How it works -### Download Link for 2024 Data -https://home.treasury.gov/resource-center/data-chart-center/interest-rates/daily-treasury-rates.csv/2024/all?type=daily_treasury_yield_curve&field_tdr_date_value=2024&page&_format=csv +1. Scheduled Execution: The project is set up to run automatically using Google Cloud Run's scheduled tasks. +2. Data Download: The script uses the provided URL to download the latest CSV file containing Treasury Par Yield Curve +### us-department-treasury - yield curves and such +### Daily Treasury Par Yield Curve Rates - Website +https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value=2024 + +### Download Link for 2024 Data + +https://home.treasury.gov/resource-center/data-chart-center/interest-rates/daily-treasury-rates.csv/2024/all?type=daily_treasury_yield_curve&field_tdr_date_value=2024&page&_format=csv ## Nightly, download the Daily Treasury Par Yield Curve Rates into CSV @@ -24,35 +39,37 @@ Throw an error if NOT EXIST ## GCP Cloud Run - scheduled 1AM NYC - ############## + ## Reference dataset: QC + https://www.quantconnect.com/data/us-treasury-yield-curve https://github.com/QuantConnect/Lean.DataSource.USTreasury/blob/master/USTreasuryYieldCurveDataAlgorithm.cs https://github.com/QuantConnect/Lean.DataSource.USTreasury/ https://github.com/QuantConnect/Lean.DataSource.USTreasury/blob/master/USTreasuryYieldCurveRate.cs - - - ## Treasury Yield Curve Methodology -![Methdology](https://home.treasury.gov/policy-issues/financing-the-government/interest-rate-statistics/treasury-yield-curve-methodology) - - -Treasury Par Yield Curve Rates: These rates are commonly referred to as "Constant Maturity Treasury" rates, or CMTs. Yields are interpolated by the Treasury from the daily par yield curve. This curve, which relates the yield on a security to its time to maturity, is based on the closing market bid prices on the most recently auctioned Treasury securities in the over-the-counter market. These par yields are derived from indicative, bid-side market price quotations (not actual transactions) obtained by the Federal Reserve Bank of New York at or near 3:30 PM each trading day. The CMT yield values are read from the par yield curve at fixed maturities, currently 1, 2, 3, 4 and 6 months and 1, 2, 3, 5, 7, 10, 20, and 30 years. This method provides a par yield for a 10-year maturity, for example, even if no outstanding security has exactly 10 years remaining to maturity. +![Methdology](https://home.treasury.gov/policy-issues/financing-the-government/interest-rate-statistics/treasury-yield-curve-methodology) +Treasury Par Yield Curve Rates: These rates are commonly referred to as "Constant Maturity Treasury" rates, or CMTs. +Yields are interpolated by the Treasury from the daily par yield curve. This curve, which relates the yield on a +security to its time to maturity, is based on the closing market bid prices on the most recently auctioned Treasury +securities in the over-the-counter market. These par yields are derived from indicative, bid-side market price +quotations (not actual transactions) obtained by the Federal Reserve Bank of New York at or near 3:30 PM each trading +day. The CMT yield values are read from the par yield curve at fixed maturities, currently 1, 2, 3, 4 and 6 months and +1, 2, 3, 5, 7, 10, 20, and 30 years. This method provides a par yield for a 10-year maturity, for example, even if no +outstanding security has exactly 10 years remaining to maturity. # Docker cli + docker build -t gcr.io/dfg-analytics-insights-prod/us-department-treasury . docker push gcr.io/dfg-analytics-insights-prod/us-department-treasury - - - - ################################# + ## Sample Data + Date,"1 Mo","2 Mo","3 Mo","4 Mo","6 Mo","1 Yr","2 Yr","3 Yr","5 Yr","7 Yr","10 Yr","20 Yr","30 Yr" 10/07/2024,5.00,4.87,4.77,4.67,4.45,4.24,3.99,3.89,3.86,3.92,4.03,4.37,4.30 10/04/2024,5.01,4.88,4.73,4.68,4.45,4.20,3.93,3.84,3.81,3.88,3.98,4.33,4.26 @@ -71,18 +88,8 @@ Date,"1 Mo","2 Mo","3 Mo","4 Mo","6 Mo","1 Yr","2 Yr","3 Yr","5 Yr","7 Yr","10 Y 09/17/2024,5.05,5.05,4.95,4.87,4.55,3.99,3.59,3.45,3.44,3.53,3.65,4.02,3.96 09/16/2024,5.11,5.10,4.96,4.88,4.55,3.96,3.56,3.42,3.41,3.51,3.63,4.01,3.94 - - - - - - - - - - ## Screenshot -For: https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value=2024 +For: https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value=2024 ![US Treasury-Daily Treasury Par Yield Curve Rates](https://github.com/deerfieldgreen/us-department-treasury/blob/main/images/Screenshot%202024-10-08%20at%2014.41.25.png) \ No newline at end of file diff --git a/data/2021-daily-treasury-rates.pkl b/data/2021-daily-treasury-rates.pkl new file mode 100644 index 0000000..9e60ed2 Binary files /dev/null and b/data/2021-daily-treasury-rates.pkl differ diff --git a/data/2022-daily-treasury-rates.pkl b/data/2022-daily-treasury-rates.pkl new file mode 100644 index 0000000..4b9bb29 Binary files /dev/null and b/data/2022-daily-treasury-rates.pkl differ diff --git a/data/2023-daily-treasury-rates.pkl b/data/2023-daily-treasury-rates.pkl new file mode 100644 index 0000000..096a87d Binary files /dev/null and b/data/2023-daily-treasury-rates.pkl differ diff --git a/data/2024-daily-treasury-rates.csv b/data/2024-daily-treasury-rates.csv index fca23a3..a5bf8b3 100644 --- a/data/2024-daily-treasury-rates.csv +++ b/data/2024-daily-treasury-rates.csv @@ -1,4 +1,41 @@ Date,1 Mo,2 Mo,3 Mo,4 Mo,6 Mo,1 Yr,2 Yr,3 Yr,5 Yr,7 Yr,10 Yr,20 Yr,30 Yr +2024-12-06,4.57,4.5,4.42,4.42,4.34,4.19,4.1,4.05,4.03,4.09,4.15,4.42,4.34 +2024-12-05,4.59,4.53,4.46,4.46,4.38,4.23,4.15,4.1,4.07,4.12,4.17,4.43,4.33 +2024-12-04,4.65,4.54,4.47,4.46,4.38,4.23,4.13,4.09,4.07,4.13,4.19,4.45,4.35 +2024-12-03,4.66,4.56,4.49,4.48,4.4,4.27,4.17,4.13,4.11,4.17,4.23,4.5,4.4 +2024-12-02,4.75,4.63,4.51,4.51,4.43,4.3,4.17,4.11,4.08,4.13,4.19,4.46,4.36 +2024-11-29,4.76,4.69,4.58,4.52,4.42,4.3,4.13,4.1,4.05,4.1,4.18,4.45,4.36 +2024-11-27,4.76,4.7,4.6,4.54,4.43,4.34,4.19,4.17,4.11,4.17,4.25,4.52,4.44 +2024-11-26,4.74,4.67,4.61,4.52,4.45,4.37,4.21,4.21,4.17,4.24,4.3,4.56,4.48 +2024-11-25,4.74,4.67,4.62,4.52,4.46,4.37,4.21,4.21,4.17,4.21,4.27,4.53,4.45 +2024-11-22,4.72,4.67,4.63,4.53,4.46,4.42,4.37,4.32,4.3,4.35,4.41,4.67,4.6 +2024-11-21,4.72,4.67,4.63,4.52,4.45,4.39,4.34,4.3,4.3,4.36,4.43,4.68,4.61 +2024-11-20,4.68,4.63,4.62,4.52,4.44,4.37,4.31,4.26,4.28,4.34,4.41,4.66,4.59 +2024-11-19,4.67,4.63,4.62,4.51,4.44,4.34,4.27,4.24,4.25,4.32,4.39,4.66,4.57 +2024-11-18,4.7,4.65,4.63,4.52,4.44,4.33,4.29,4.25,4.28,4.35,4.42,4.7,4.61 +2024-11-15,4.7,4.67,4.6,4.52,4.44,4.34,4.31,4.27,4.3,4.36,4.43,4.7,4.6 +2024-11-14,4.7,4.68,4.61,4.53,4.43,4.36,4.34,4.3,4.32,4.38,4.43,4.69,4.58 +2024-11-13,4.69,4.67,4.6,4.5,4.4,4.31,4.27,4.25,4.3,4.38,4.44,4.73,4.63 +2024-11-12,4.7,4.7,4.63,4.54,4.45,4.38,4.34,4.29,4.32,4.38,4.43,4.7,4.58 +2024-11-08,4.7,4.69,4.63,4.53,4.42,4.32,4.26,4.18,4.2,4.25,4.3,4.58,4.47 +2024-11-07,4.69,4.69,4.63,4.52,4.4,4.28,4.21,4.13,4.17,4.25,4.31,4.62,4.52 +2024-11-06,4.68,4.71,4.64,4.54,4.41,4.31,4.27,4.2,4.27,4.37,4.42,4.71,4.6 +2024-11-05,4.72,4.72,4.64,4.49,4.39,4.27,4.19,4.11,4.16,4.22,4.26,4.55,4.44 +2024-11-04,4.75,4.74,4.65,4.51,4.39,4.25,4.17,4.1,4.17,4.24,4.31,4.6,4.5 +2024-11-01,4.75,4.74,4.61,4.53,4.42,4.28,4.21,4.18,4.22,4.3,4.37,4.68,4.57 +2024-10-31,4.76,4.76,4.64,4.56,4.43,4.27,4.16,4.12,4.15,4.21,4.28,4.58,4.47 +2024-10-30,4.87,4.77,4.67,4.57,4.44,4.28,4.15,4.14,4.14,4.2,4.29,4.6,4.49 +2024-10-29,4.88,4.77,4.7,4.64,4.45,4.29,4.11,4.08,4.11,4.18,4.28,4.61,4.52 +2024-10-28,4.88,4.77,4.7,4.65,4.46,4.28,4.12,4.09,4.11,4.19,4.28,4.61,4.53 +2024-10-25,4.89,4.79,4.73,4.68,4.51,4.29,4.11,4.05,4.07,4.15,4.25,4.58,4.51 +2024-10-24,4.87,4.78,4.7,4.65,4.48,4.25,4.07,4.02,4.03,4.12,4.21,4.54,4.47 +2024-10-23,4.88,4.8,4.73,4.66,4.48,4.27,4.07,4.03,4.05,4.14,4.24,4.58,4.51 +2024-10-22,4.89,4.81,4.72,4.66,4.47,4.24,4.03,3.98,4.0,4.1,4.2,4.55,4.49 +2024-10-21,4.92,4.82,4.73,4.66,4.47,4.24,4.02,3.95,3.98,4.07,4.19,4.54,4.49 +2024-10-18,4.92,4.82,4.73,4.65,4.45,4.19,3.95,3.86,3.88,3.97,4.08,4.44,4.38 +2024-10-17,4.93,4.83,4.74,4.65,4.45,4.21,3.96,3.89,3.9,3.99,4.09,4.44,4.39 +2024-10-16,4.91,4.8,4.72,4.63,4.42,4.17,3.93,3.84,3.84,3.92,4.02,4.36,4.3 +2024-10-15,4.93,4.82,4.73,4.65,4.42,4.18,3.95,3.86,3.86,3.93,4.03,4.37,4.32 2024-10-11,4.97,4.82,4.73,4.65,4.44,4.18,3.95,3.85,3.88,3.97,4.08,4.44,4.39 2024-10-10,4.98,4.84,4.75,4.65,4.45,4.22,3.98,3.88,3.91,3.99,4.09,4.44,4.38 2024-10-09,4.93,4.84,4.75,4.65,4.46,4.24,3.99,3.89,3.91,3.97,4.06,4.41,4.34 diff --git a/data/2024-daily-treasury-rates.pkl b/data/2024-daily-treasury-rates.pkl new file mode 100644 index 0000000..9b4d76e Binary files /dev/null and b/data/2024-daily-treasury-rates.pkl differ diff --git a/data/daily-treasury-rates.pkl b/data/daily-treasury-rates.pkl new file mode 100644 index 0000000..1d17a4e Binary files /dev/null and b/data/daily-treasury-rates.pkl differ diff --git a/main.py b/main.py index afc7b0c..c85d864 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,7 @@ import pytz from udt.utils import * import dotenv +import pickle dotenv.load_dotenv() @@ -26,8 +27,32 @@ def read_and_encode_file(file_path, encode=True): return content -if __name__ == '__main__': +def save_and_update_github(file_path, repo, github_path): + """Save file to GitHub and update if it already exists.""" + content = read_and_encode_file(file_path, encode=False) + try: + git_file = repo.get_contents(github_path) + logger.info(f"Found existing file: {git_file.path}") + repo.update_file( + git_file.path, + f"Updated file for {datetime.datetime.now(pytz.timezone('US/Eastern')).date()}", + content, + git_file.sha, + ) + except Exception as e: + if isinstance(e, GithubException) and e.status == 404: # File not found + logger.info(f"File not found. Creating a new file: {github_path}") + repo.create_file( + github_path, + f"Created file for {datetime.datetime.now(pytz.timezone('US/Eastern')).date()}", + content, + ) + else: + logger.error(f"An error occurred: {str(e)}") + raise e + +if __name__ == '__main__': # 1. Fetch and save daily treasury rates data for current year timezone = pytz.timezone('US/Eastern') current_time = datetime.datetime.now(timezone) @@ -38,6 +63,7 @@ def read_and_encode_file(file_path, encode=True): df = dfs[0] work_dir = os.getcwd() csv_path = os.path.join(work_dir, 'data', f'{current_year}-daily-treasury-rates.csv') + pickle_path = os.path.join(work_dir, 'data', f'{current_year}-daily-treasury-rates.pkl') csv_dir = os.path.dirname(csv_path) rel_cols = [x for x in df.columns if 'Yr' in x or 'Mo' in x or 'Date' in x] @@ -45,14 +71,24 @@ def read_and_encode_file(file_path, encode=True): df['Date'] = pd.to_datetime(df['Date'], format='%m/%d/%Y') df.sort_values('Date', inplace=True, ascending=False) + + # Save as CSV df.to_csv(csv_path, index=False) + + # Save as pickle + with open(pickle_path, 'wb') as f: + pickle.dump(df, f) # 2. Merge all the years into one file file_names = sort_file_names(csv_dir) merged = merge_files(file_names, csv_dir) - all_years_path = os.path.join(csv_dir, 'daily-treasury-rates.csv') + all_years_csv_path = os.path.join(csv_dir, 'daily-treasury-rates.csv') + all_years_pickle_path = os.path.join(csv_dir, 'daily-treasury-rates.pkl') - + # Save merged data as CSV and pickle + merged.to_csv(all_years_csv_path, index=False) + with open(all_years_pickle_path, 'wb') as f: + pickle.dump(merged, f) # Prepare github token = os.environ.get("GIT_TOKEN") @@ -61,27 +97,10 @@ def read_and_encode_file(file_path, encode=True): "deerfieldgreen/us-department-treasury" ) # Replace with your repo details - content = read_and_encode_file(all_years_path, encode=False) - try: - git_file = repo.get_contents(f"data/daily-treasury-rates.csv") - logger.info(f"Found existing file: {git_file.path}") - repo.update_file( - git_file.path, - f"Updated file for {current_time.date()}", - content, - git_file.sha, - ) - except Exception as e: - if isinstance(e, GithubException) and e.status == 404: # File not found - logger.info(f"File not found. Creating a new file.") - repo.create_file( - f"data/daily-treasury-rates.csv", - f"Created file for {current_time.date()}", - content, - ) - else: - logger.error(f"An error occurred: {str(e)}") - raise e - - logger.info(f"Pushed to Github") + # Update or create files on GitHub + save_and_update_github(all_years_csv_path, repo, "data/daily-treasury-rates.csv") + save_and_update_github(all_years_pickle_path, repo, "data/daily-treasury-rates.pkl") + save_and_update_github(csv_path, repo, f"data/{current_year}-daily-treasury-rates.csv") + save_and_update_github(pickle_path, repo, f"data/{current_year}-daily-treasury-rates.pkl") + logger.info(f"All files pushed to Github") \ No newline at end of file