data_update_web_hf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2c7a256e-b56d-47ee-bcf5-4348bf3bd624",
   "metadata": {
    "tags": []
   },
   "source": [
    "## data updates with high frequency for the helioforecast.space website\n",
    "\n",
    "Main author: C. Möstl, Austrian Space Weather Office, GeoSphere Austria\n",
    "\n",
    "part of https://github.com/cmoestl/heliocats\n",
    "\n",
    "uses environment 'envs/env_helio4.yml'\n",
    "\n",
    "\n",
    "https://github.com/cmoestl/heliocats  data_update_web_hf.py\n",
    "\n",
    "for updating data with high frequency on the servers\n",
    "\n",
    "Issues:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "757ca544-7eb5-4e31-a500-f93e3914195f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "system is mac\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[NbConvertApp] Converting notebook data_update_web_hf.ipynb to script\n",
      "[NbConvertApp] Writing 9710 bytes to data_update_web_hf.py\n"
     ]
    }
   ],
   "source": [
    "#switches\n",
    "debug_mode=0\n",
    "#always turn off debug mode when deploying!\n",
    "\n",
    "#switches to download and save as pickle\n",
    "get_noaa=1\n",
    "get_stereoa=1\n",
    "\n",
    "\n",
    "import pickle\n",
    "import importlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import matplotlib.dates as mdates\n",
    "import sys\n",
    "import numpy as np\n",
    "import datetime\n",
    "import scipy.signal\n",
    "import urllib\n",
    "import json\n",
    "import os   \n",
    "import time\n",
    "import h5py\n",
    "import pytz\n",
    "import copy\n",
    "import cdflib\n",
    "import sunpy\n",
    "\n",
    "from heliocats import data as hd\n",
    "from heliocats import plot as hp\n",
    "\n",
    "##### check for system type\n",
    "#server\n",
    "if sys.platform == 'linux': \n",
    "    print('system is linux')\n",
    "    matplotlib.use('Agg') \n",
    "#mac\n",
    "if sys.platform =='darwin':  \n",
    "    print('system is mac')\n",
    "    #for testing\n",
    "    #%matplotlib inline     \n",
    "    matplotlib.use('Agg') \n",
    "\n",
    "\n",
    "\n",
    "################################################ CHECK  ##############################################\n",
    "\n",
    "#make sure to convert the current notebook to a script\n",
    "os.system('jupyter nbconvert --to script data_update_web_hf.ipynb')   \n",
    "\n",
    "\n",
    "####################################################################################################################\n",
    "\n",
    "#test execution times\n",
    "t0all = time.time()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1c436e0-430d-4398-b43e-f1796b186dfb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Configure paths depending on server or local machine\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9f5b515c-0aa2-47b1-9b3d-03576f63883d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "------ PATHS \n",
      "/Users/chris/python/data/insitu_python/\n",
      "/Users/chris/python/ops/data/noaa/\n",
      "/Users/chris/python/data/wind/\n",
      "/Users/chris/python/data/stereoa/\n",
      "/Users/chris/python/data/insitu_python/ML_struct_arrays/\n",
      "/Users/chris/python/data/insitu_python/plots/\n",
      "/Users/chris/python/data/insitu_python/plots_positions/\n"
     ]
    }
   ],
   "source": [
    "if sys.platform == 'linux': \n",
    "    \n",
    "    from config_server import data_path\n",
    "    from config_server import noaa_path\n",
    "    from config_server import wind_path\n",
    "    from config_server import stereoa_path\n",
    "    from config_server import data_path_ml\n",
    "    \n",
    "if sys.platform =='darwin':  \n",
    "\n",
    "    from config_local import data_path\n",
    "    from config_local import noaa_path\n",
    "    from config_local import wind_path\n",
    "    from config_local import stereoa_path\n",
    "    from config_local import data_path_ml\n",
    "\n",
    "print(' ')\n",
    "print('------ PATHS ')\n",
    "\n",
    "print(data_path)\n",
    "print(noaa_path)\n",
    "print(wind_path)\n",
    "print(stereoa_path)\n",
    "print(data_path_ml)\n",
    "\n",
    "\n",
    "plot_path=data_path+'plots/'\n",
    "position_path=data_path+'plots_positions/'\n",
    "sun_path=data_path+'plots_sun/'\n",
    "\n",
    "print(plot_path)\n",
    "print(position_path)\n",
    "\n",
    "\n",
    "\n",
    "########### make directories first time\n",
    "if os.path.isdir(plot_path) == False: os.mkdir(plot_path)\n",
    "if os.path.isdir(plot_path+'omni2') == False: os.mkdir(plot_path+'omni2')\n",
    "if os.path.isdir(plot_path+'wind') == False: os.mkdir(plot_path+'wind')\n",
    "if os.path.isdir(plot_path+'stereoa') == False: os.mkdir(plot_path+'stereoa')\n",
    "if os.path.isdir(plot_path+'combined') == False: os.mkdir(plot_path+'combined')\n",
    "\n",
    "\n",
    "if os.path.isdir(position_path) == False: os.mkdir(position_path)\n",
    "if os.path.isdir(sun_path) == False: os.mkdir(sun_path)\n",
    "if os.path.isdir(noaa_path) == False: os.mkdir(noaa_path)\n",
    "if os.path.isdir(data_path_ml) == False: os.mkdir(data_path_ml)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5707160b-74ba-45da-a283-72b08807a167",
   "metadata": {
    "tags": []
   },
   "source": [
    "### positions, Xray and SDO plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5ca18f91-6e84-4d28-8bea-2d50c6c6735a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "------ get Xray data and plot \n",
      "NOAA path /Users/chris/python/ops/data/noaa/     Data path /Users/chris/python/data/insitu_python/\n",
      "download NOAA Xrays\n",
      "2025-01-16 UTC\n",
      "/Users/chris/python/ops/data/noaa/xray/xray-7-day_2025-01-16.json\n",
      "/Users/chris/python/ops/data/noaa/xray/xray2-7-day_2025-01-16.json\n",
      "saved as /Users/chris/python/data/insitu_python/noaa_xray_last_7days_now.p pickle\n",
      "saved as /Users/chris/python/data/insitu_python/noaa_xray2_last_7days_now.p pickle\n",
      "[datetime.datetime(2025, 1, 9, 11, 1) datetime.datetime(2025, 1, 9, 11, 2)\n",
      " datetime.datetime(2025, 1, 9, 11, 3) ...\n",
      " datetime.datetime(2025, 1, 16, 10, 56)\n",
      " datetime.datetime(2025, 1, 16, 10, 57)\n",
      " datetime.datetime(2025, 1, 16, 10, 58)]\n",
      "[datetime.datetime(2025, 1, 9, 11, 1) datetime.datetime(2025, 1, 9, 11, 2)\n",
      " datetime.datetime(2025, 1, 9, 11, 3) ...\n",
      " datetime.datetime(2025, 1, 16, 10, 56)\n",
      " datetime.datetime(2025, 1, 16, 10, 57)\n",
      " datetime.datetime(2025, 1, 16, 10, 58)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chris/python/heliocats/heliocats/plot.py:187: UserWarning:\n",
      "\n",
      "This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved as  /Users/chris/python/data/insitu_python/plots/latest_xray.jpg\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/latest_xray.html\n"
     ]
    }
   ],
   "source": [
    "if debug_mode > 0: \n",
    "    importlib.reload(hd) \n",
    "    importlib.reload(hp) \n",
    "\n",
    "print(' ')\n",
    "print('------ get Xray data and plot ')\n",
    "print('NOAA path',noaa_path,'    Data path',data_path)\n",
    "\n",
    "xraypickle=str('noaa_xray_last_7days_now.p')\n",
    "xraypickle2=str('noaa_xray2_last_7days_now.p')\n",
    "hd.get_noaa_xray(noaa_path,data_path,xraypickle,xraypickle2)\n",
    "hp.plot_noaa_xray(data_path+xraypickle,data_path+xraypickle2,plot_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "92f959e1-8848-42dc-94ea-7c28970e80cd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "------ POSITIONS \n",
      "saved as  /Users/chris/python/data/insitu_python/plots_positions/positions_2025_Jan_16.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots_positions/positions_now.p\n",
      "saved as  /Users/chris/python/data/insitu_python/plots_positions/positions_now.txt\n",
      "saved as  /Users/chris/python/data/insitu_python/plots_positions/positions_now.png\n",
      " \n",
      "------ SDO realtime images \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chris/python/heliocats/heliocats/plot.py:3685: UserWarning:\n",
      "\n",
      "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved  /Users/chris/python/data/insitu_python/plots_sun/latest_1024_0193.jpg\n",
      "2025-01-16 10:47 UT\n",
      "saved  /Users/chris/python/data/insitu_python/plots_sun/latest_1024_HMIB.jpg\n",
      "2025-01-16 10:15 UT\n",
      "saved  /Users/chris/python/data/insitu_python/plots_sun/latest_1024_HMIIC.jpg\n",
      "saved  /Users/chris/python/data/insitu_python/plots_sun/latest_1024_HMIBpfss.jpg\n",
      "\n",
      "Positions and SDO images takes 21.48 seconds\n"
     ]
    }
   ],
   "source": [
    "if debug_mode > 0: \n",
    "    importlib.reload(hd) \n",
    "    importlib.reload(hp) \n",
    "\n",
    "t0 = time.time()\n",
    "\n",
    "print(' ')\n",
    "print('------ POSITIONS ')\n",
    "\n",
    "# spacecraft positions image, uses positions file  /results/positions/positions_psp_solo_sta_bepi_wind_planets_HEEQ_10min_rad.p'\n",
    "hp.plot_positions(datetime.datetime.utcnow(),position_path, 'HEEQ',now=True)\n",
    "\n",
    "\n",
    "#make positions for specific events as pdf\n",
    "#hp.plot_positions(datetime.datetime(2022,9,5,18),position_path, 'HEEQ',pdf=True)\n",
    "\n",
    "\n",
    "print(' ')\n",
    "print('------ SDO realtime images ')\n",
    "# get current SDO images \n",
    "hd.get_sdo_realtime_image(sun_path)\n",
    "\n",
    "t1 = time.time()\n",
    "print()\n",
    "print('Positions and SDO images takes', np.round(t1-t0,2), 'seconds')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ce5f56d-7070-43a5-a667-26670c04e6c1",
   "metadata": {
    "tags": []
   },
   "source": [
    "### NOAA real time solar wind "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61a3f612-5df0-49df-86c0-e75c373be5bd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "------ NOAA real time solar wind data \n",
      "download NOAA real time solar wind plasma and mag and dst\n",
      "2025-01-16 UTC\n",
      "/Users/chris/python/ops/data/noaa/plasma/plasma-7-day_2025-01-16.json\n",
      "/Users/chris/python/ops/data/noaa/mag/mag-7-day_2025-01-16.json\n",
      "/Users/chris/python/ops/data/noaa/dst/dst-7-day_2025-01-16.json\n",
      "NOAA RTSW download complete\n",
      " \n",
      "convert NOAA real time solar wind archive to pickle file\n",
      "directories for the json data\n",
      "/Users/chris/python/ops/data/noaa/mag/\n",
      "Sorted file list to be read with cutoff  35  files. \n",
      "['mag-7-day_2023-09-26.json' 'mag-7-day_2023-09-29.json'\n",
      " 'mag-7-day_2023-10-02.json' 'mag-7-day_2023-11-05.json'\n",
      " 'mag-7-day_2023-11-08.json' 'mag-7-day_2023-11-15.json'\n",
      " 'mag-7-day_2023-11-16.json' 'mag-7-day_2024-02-07.json'\n",
      " 'mag-7-day_2024-03-02.json' 'mag-7-day_2024-03-08.json'\n",
      " 'mag-7-day_2024-03-12.json' 'mag-7-day_2024-03-13.json'\n",
      " 'mag-7-day_2024-03-14.json' 'mag-7-day_2024-03-15.json'\n",
      " 'mag-7-day_2024-03-16.json' 'mag-7-day_2024-03-18.json'\n",
      " 'mag-7-day_2024-03-19.json' 'mag-7-day_2024-04-02.json'\n",
      " 'mag-7-day_2024-04-03.json' 'mag-7-day_2024-05-10.json'\n",
      " 'mag-7-day_2024-05-14.json' 'mag-7-day_2024-05-15.json'\n",
      " 'mag-7-day_2024-05-17.json' 'mag-7-day_2024-08-31.json'\n",
      " 'mag-7-day_2024-09-11.json' 'mag-7-day_2024-09-12.json'\n",
      " 'mag-7-day_2024-09-13.json' 'mag-7-day_2024-09-16.json'\n",
      " 'mag-7-day_2024-09-18.json' 'mag-7-day_2024-09-19.json'\n",
      " 'mag-7-day_2024-09-23.json' 'mag-7-day_2024-09-24.json'\n",
      " 'mag-7-day_2024-12-19.json' 'mag-7-day_2024-12-20.json'\n",
      " 'mag-7-day_2025-01-16.json']\n",
      "/Users/chris/python/ops/data/noaa/plasma/\n",
      "['plasma-7-day_2023-09-26.json' 'plasma-7-day_2023-09-29.json'\n",
      " 'plasma-7-day_2023-10-02.json' 'plasma-7-day_2023-11-05.json'\n",
      " 'plasma-7-day_2023-11-08.json' 'plasma-7-day_2023-11-15.json'\n",
      " 'plasma-7-day_2023-11-16.json' 'plasma-7-day_2024-02-07.json'\n",
      " 'plasma-7-day_2024-03-02.json' 'plasma-7-day_2024-03-08.json'\n",
      " 'plasma-7-day_2024-03-12.json' 'plasma-7-day_2024-03-13.json'\n",
      " 'plasma-7-day_2024-03-14.json' 'plasma-7-day_2024-03-15.json'\n",
      " 'plasma-7-day_2024-03-16.json' 'plasma-7-day_2024-03-18.json'\n",
      " 'plasma-7-day_2024-03-19.json' 'plasma-7-day_2024-04-02.json'\n",
      " 'plasma-7-day_2024-04-03.json' 'plasma-7-day_2024-05-10.json'\n",
      " 'plasma-7-day_2024-05-14.json' 'plasma-7-day_2024-05-15.json'\n",
      " 'plasma-7-day_2024-05-17.json' 'plasma-7-day_2024-08-31.json'\n",
      " 'plasma-7-day_2024-09-11.json' 'plasma-7-day_2024-09-12.json'\n",
      " 'plasma-7-day_2024-09-13.json' 'plasma-7-day_2024-09-16.json'\n",
      " 'plasma-7-day_2024-09-18.json' 'plasma-7-day_2024-09-19.json'\n",
      " 'plasma-7-day_2024-09-23.json' 'plasma-7-day_2024-09-24.json'\n",
      " 'plasma-7-day_2024-12-19.json' 'plasma-7-day_2024-12-20.json'\n",
      " 'plasma-7-day_2025-01-16.json']\n",
      "\n",
      "/Users/chris/python/ops/data/noaa/ephem/\n",
      "['ephemerides_2023-08-07.json' 'ephemerides_2023-08-15.json'\n",
      " 'ephemerides_2023-08-17.json' 'ephemerides_2023-09-22.json'\n",
      " 'ephemerides_2023-09-24.json' 'ephemerides_2024-05-17.json']\n",
      "\n",
      "position start\n",
      "[     0      1      2 ... 180778 180779 180780]\n",
      "conversion HEE to HEEQ\n",
      "HEE to HEEQ done\n",
      "position end \n",
      "file saved  /Users/chris/python/data/insitu_python/noaa_rtsw_last_35files_now.p\n",
      "\n",
      "\n",
      "-------- make Dst file ----\n",
      "/Users/chris/python/ops/data/noaa/dst/\n",
      "['dst-7-day_2023-09-26.json' 'dst-7-day_2023-09-29.json'\n",
      " 'dst-7-day_2023-10-02.json' 'dst-7-day_2023-11-05.json'\n",
      " 'dst-7-day_2023-11-08.json' 'dst-7-day_2023-11-15.json'\n",
      " 'dst-7-day_2023-11-16.json' 'dst-7-day_2024-02-07.json'\n",
      " 'dst-7-day_2024-03-02.json' 'dst-7-day_2024-03-08.json'\n",
      " 'dst-7-day_2024-03-12.json' 'dst-7-day_2024-03-13.json'\n",
      " 'dst-7-day_2024-03-14.json' 'dst-7-day_2024-03-15.json'\n",
      " 'dst-7-day_2024-03-16.json' 'dst-7-day_2024-03-18.json'\n",
      " 'dst-7-day_2024-03-19.json' 'dst-7-day_2024-04-02.json'\n",
      " 'dst-7-day_2024-04-03.json' 'dst-7-day_2024-05-10.json'\n",
      " 'dst-7-day_2024-05-14.json' 'dst-7-day_2024-05-15.json'\n",
      " 'dst-7-day_2024-05-17.json' 'dst-7-day_2024-08-31.json'\n",
      " 'dst-7-day_2024-09-11.json' 'dst-7-day_2024-09-12.json'\n",
      " 'dst-7-day_2024-09-13.json' 'dst-7-day_2024-09-16.json'\n",
      " 'dst-7-day_2024-09-18.json' 'dst-7-day_2024-09-19.json'\n",
      " 'dst-7-day_2024-09-23.json' 'dst-7-day_2024-09-24.json'\n",
      " 'dst-7-day_2024-12-19.json' 'dst-7-day_2024-12-20.json'\n",
      " 'dst-7-day_2025-01-16.json']\n",
      "file saved  /Users/chris/python/data/insitu_python/noaa_dst_last_35files_now.p\n",
      " \n",
      " \n",
      "NOAA RTSW saved as pickle file complete\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/noaa/NOAA_RTSW_2025_Jan_06_2025_Jan_16.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/noaa/NOAA_RTSW_now.png\n",
      "-----done\n",
      "\n",
      "NOAA download latest file, save as pickle last 35 files and plotting takes 69.11 seconds\n"
     ]
    }
   ],
   "source": [
    "if debug_mode > 0: \n",
    "    from heliocats import data as hd\n",
    "    importlib.reload(hd) \n",
    "\n",
    "    from heliocats import plot as hp\n",
    "    importlib.reload(hp) \n",
    "\n",
    "print(' ')\n",
    "print('------ NOAA real time solar wind data ')\n",
    "\n",
    "#test execution times\n",
    "t0 = time.time()\n",
    "\n",
    "#define file name\n",
    "filenoaa='noaa_rtsw_last_35files_now.p'\n",
    "filedst='noaa_dst_last_35files_now.p'\n",
    "\n",
    "\n",
    "if get_noaa > 0:\n",
    "\n",
    "    ######## DOWNLOAD NOAA data \n",
    "    \n",
    "    print('download NOAA real time solar wind plasma and mag and dst')\n",
    "    datestr=str(datetime.datetime.utcnow().strftime(\"%Y-%m-%d\"))\n",
    "    print(datestr+' UTC')\n",
    "\n",
    "    #solar wind\n",
    "    #ephemerides are done with the daily update\n",
    "    #ephem='http://services.swpc.noaa.gov/products/solar-wind/ephemerides.json' \n",
    "    \n",
    "    plasma='http://services.swpc.noaa.gov/products/solar-wind/plasma-7-day.json'\n",
    "    mag='http://services.swpc.noaa.gov/products/solar-wind/mag-7-day.json'\n",
    "    \n",
    "    dst='https://services.swpc.noaa.gov/products/kyoto-dst.json'\n",
    "        \n",
    "\n",
    "    #the ephemerides are done with the daily update\n",
    "    #try: \n",
    "    #    urllib.request.urlretrieve(ephem, noaa_path+'ephem/ephemerides_'+datestr+'.json')\n",
    "    #    print(noaa_path+'ephem/ephemerides_'+datestr+'.json')\n",
    "    #except urllib.error.URLError as e:\n",
    "    #    print(' ', ephem,' ',e.reason)\n",
    "\n",
    "    \n",
    "    try: \n",
    "        urllib.request.urlretrieve(plasma, noaa_path+'plasma/plasma-7-day_'+datestr+'.json')\n",
    "        print(noaa_path+'plasma/plasma-7-day_'+datestr+'.json')\n",
    "    except urllib.error.URLError as e:\n",
    "        print(' ', plasma,' ',e.reason)\n",
    "\n",
    "    try: \n",
    "        urllib.request.urlretrieve(mag, noaa_path+'mag/mag-7-day_'+datestr+'.json')\n",
    "        print(noaa_path+'mag/mag-7-day_'+datestr+'.json')\n",
    "    except urllib.error.URLError as e:\n",
    "        print(' ', mag,' ',e.reason)\n",
    "    \n",
    "    try: \n",
    "        urllib.request.urlretrieve(dst, noaa_path+'dst/dst-7-day_'+datestr+'.json')\n",
    "        print(noaa_path+'dst/dst-7-day_'+datestr+'.json')\n",
    "    except urllib.error.URLError as e:\n",
    "        print(' ', dst,' ',e.reason)\n",
    "        \n",
    "\n",
    "    print('NOAA RTSW download complete')\n",
    "        \n",
    "    ## SAVE NOAA DATA AS PICKLE, dst as extra file\n",
    "    # last parameter gives a cutoff, so only the latest N files are taken for the NOAA data pickle file\n",
    "    \n",
    "    \n",
    "    if debug_mode == 0: hd.save_noaa_rtsw_data(data_path,noaa_path,filenoaa,filedst,35)\n",
    "    if debug_mode > 0: hd.save_noaa_rtsw_data(data_path,noaa_path,filenoaa,filedst,3)\n",
    "\n",
    "    print('NOAA RTSW saved as pickle file complete')\n",
    "    \n",
    "\n",
    "else:\n",
    "    print('NOAA data NOT downloaded and saved as pickle, turn on switch')  \n",
    "\n",
    "\n",
    "#load data file\n",
    "[noaa,hnoaa]=pickle.load(open(data_path+filenoaa, \"rb\" ) ) \n",
    "\n",
    "#load dst file\n",
    "#dst=pickle.load(open(data_path+filedst, \"rb\" ) ) \n",
    "\n",
    " \n",
    "#plot\n",
    "start=datetime.datetime.utcnow() - datetime.timedelta(days=10)\n",
    "end=datetime.datetime.utcnow() \n",
    "hp.plot_insitu_update_noaa_rtsw(noaa, start, end,'NOAA_RTSW',plot_path+'noaa/',now=True)\n",
    "\n",
    "\n",
    "t1 = time.time()\n",
    "\n",
    "print('-----done')\n",
    "\n",
    "print()\n",
    "print('NOAA download latest file, save as pickle last 35 files and plotting takes', np.round(t1-t0,2), 'seconds')\n",
    "\n",
    "#plt.plot(dst.time,dst.dst)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "621fc7c3-027b-47df-b3ea-fd23c8df1f76",
   "metadata": {},
   "source": [
    "### STEREO-A beacon data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "971f0af3-f5bd-4f2f-a2a6-30acc93001cd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "------ STEREO-A beacon data \n",
      " \n",
      "--- download STEREO-A beacon data last 10 days \n",
      "download STEREO-A beacon data to  /Users/chris/python/data/stereoa/\n",
      "STA_LB_IMPACT_20250106_V02.cdf\n",
      "STA_LB_PLASTIC_20250106_V14.cdf\n",
      "STA_LB_IMPACT_20250107_V02.cdf\n",
      "STA_LB_PLASTIC_20250107_V14.cdf\n",
      "STA_LB_IMPACT_20250108_V02.cdf\n",
      "STA_LB_PLASTIC_20250108_V14.cdf\n",
      "STA_LB_IMPACT_20250109_V02.cdf\n",
      "STA_LB_PLASTIC_20250109_V14.cdf\n",
      "STA_LB_IMPACT_20250110_V02.cdf\n",
      "STA_LB_PLASTIC_20250110_V14.cdf\n",
      "STA_LB_IMPACT_20250111_V02.cdf\n",
      "STA_LB_PLASTIC_20250111_V14.cdf\n",
      "STA_LB_IMPACT_20250112_V02.cdf\n",
      "STA_LB_PLASTIC_20250112_V14.cdf\n",
      "STA_LB_IMPACT_20250113_V02.cdf\n",
      "STA_LB_PLASTIC_20250113_V14.cdf\n",
      "STA_LB_IMPACT_20250114_V02.cdf\n",
      "STA_LB_PLASTIC_20250114_V14.cdf\n",
      "STA_LB_IMPACT_20250115_V02.cdf\n",
      "STA_LB_PLASTIC_20250115_V14.cdf\n",
      "STA_LB_IMPACT_20250116_V02.cdf\n",
      "STA_LB_PLASTIC_20250116_V14.cdf\n",
      " \n",
      "--- process STEREO-A beacon data to pickle last 35 days\n",
      "2024-12-12 11:11:00\n",
      "2025-01-16 11:11:00\n",
      "['20241212', '20241213', '20241214', '20241215', '20241216', '20241217', '20241218', '20241219', '20241220', '20241221', '20241222', '20241223', '20241224', '20241225', '20241226', '20241227', '20241228', '20241229', '20241230', '20241231', '20250101', '20250102', '20250103', '20250104', '20250105', '20250106', '20250107', '20250108', '20250109', '20250110', '20250111', '20250112', '20250113', '20250114', '20250115', '20250116']\n",
      "STA_LB_IMPACT_20241212_V02.cdf\n",
      "STA_LB_IMPACT_20241213_V02.cdf\n",
      "STA_LB_IMPACT_20241214_V02.cdf\n",
      "STA_LB_IMPACT_20241215_V02.cdf\n",
      "STA_LB_IMPACT_20241216_V02.cdf\n",
      "STA_LB_IMPACT_20241217_V02.cdf\n",
      "STA_LB_IMPACT_20241218_V02.cdf\n",
      "STA_LB_IMPACT_20241219_V02.cdf\n",
      "STA_LB_IMPACT_20241220_V02.cdf\n",
      "STA_LB_IMPACT_20250106_V02.cdf\n",
      "STA_LB_IMPACT_20250107_V02.cdf\n",
      "STA_LB_IMPACT_20250108_V02.cdf\n",
      "STA_LB_IMPACT_20250109_V02.cdf\n",
      "STA_LB_IMPACT_20250110_V02.cdf\n",
      "STA_LB_IMPACT_20250111_V02.cdf\n",
      "STA_LB_IMPACT_20250112_V02.cdf\n",
      "STA_LB_IMPACT_20250113_V02.cdf\n",
      "STA_LB_IMPACT_20250114_V02.cdf\n",
      "STA_LB_IMPACT_20250115_V02.cdf\n",
      "STA_LB_IMPACT_20250116_V02.cdf\n",
      "done\n",
      "['20241212', '20241213', '20241214', '20241215', '20241216', '20241217', '20241218', '20241219', '20241220', '20241221', '20241222', '20241223', '20241224', '20241225', '20241226', '20241227', '20241228', '20241229', '20241230', '20241231', '20250101', '20250102', '20250103', '20250104', '20250105', '20250106', '20250107', '20250108', '20250109', '20250110', '20250111', '20250112', '20250113', '20250114', '20250115', '20250116']\n",
      "STA_LB_PLASTIC_20241212_V14.cdf\n",
      "STA_LB_PLASTIC_20241213_V14.cdf\n",
      "STA_LB_PLASTIC_20241214_V14.cdf\n",
      "STA_LB_PLASTIC_20241215_V14.cdf\n",
      "STA_LB_PLASTIC_20241216_V14.cdf\n",
      "STA_LB_PLASTIC_20241217_V14.cdf\n",
      "STA_LB_PLASTIC_20241218_V14.cdf\n",
      "STA_LB_PLASTIC_20241219_V14.cdf\n",
      "STA_LB_PLASTIC_20241220_V14.cdf\n",
      "STA_LB_PLASTIC_20241221_V14.cdf\n",
      "STA_LB_PLASTIC_20241222_V14.cdf\n",
      "STA_LB_PLASTIC_20241223_V14.cdf\n",
      "STA_LB_PLASTIC_20241224_V14.cdf\n",
      "STA_LB_PLASTIC_20241225_V14.cdf\n",
      "STA_LB_PLASTIC_20241226_V14.cdf\n",
      "STA_LB_PLASTIC_20241227_V14.cdf\n",
      "STA_LB_PLASTIC_20241228_V14.cdf\n",
      "STA_LB_PLASTIC_20241229_V14.cdf\n",
      "STA_LB_PLASTIC_20241230_V14.cdf\n",
      "STA_LB_PLASTIC_20241231_V14.cdf\n",
      "STA_LB_PLASTIC_20250101_V14.cdf\n",
      "STA_LB_PLASTIC_20250102_V14.cdf\n",
      "STA_LB_PLASTIC_20250103_V14.cdf\n",
      "STA_LB_PLASTIC_20250104_V14.cdf\n",
      "STA_LB_PLASTIC_20250105_V14.cdf\n",
      "STA_LB_PLASTIC_20250106_V14.cdf\n",
      "STA_LB_PLASTIC_20250107_V14.cdf\n",
      "STA_LB_PLASTIC_20250108_V14.cdf\n",
      "STA_LB_PLASTIC_20250109_V14.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chris/python/miniconda3/envs/helio4/lib/python3.10/site-packages/erfa/core.py:154: ErfaWarning:\n",
      "\n",
      "ERFA function \"taiutc\" yielded 1 of \"dubious year (Note 4)\"\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STA_LB_PLASTIC_20250110_V14.cdf\n",
      "STA_LB_PLASTIC_20250111_V14.cdf\n",
      "STA_LB_PLASTIC_20250112_V14.cdf\n",
      "STA_LB_PLASTIC_20250113_V14.cdf\n",
      "STA_LB_PLASTIC_20250114_V14.cdf\n",
      "STA_LB_PLASTIC_20250115_V14.cdf\n",
      "STA_LB_PLASTIC_20250116_V14.cdf\n",
      "PLASTIC done\n",
      "position start with astrospice stereo-A predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Files Downloaded: 100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  1.44file/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "position end \n",
      "conversion RTN to GSE\n",
      "conversion RTN to GSE done\n",
      "conversion GSE to GSM\n",
      "conversion GSE to GSM done\n",
      "save pickle file\n",
      "save pickle file\n",
      "done sta\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/stereoa/STEREO-A_beacon_rtn_2025_Jan_06_2025_Jan_16.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/stereoa/STEREO-A_beacon_rtn_now.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/stereoa/STEREO-A_beacon_gsm_2025_Jan_06_2025_Jan_16.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/stereoa/STEREO-A_beacon_gsm_now.png\n",
      "\n",
      "STEREO-A beacon downloading last 10 days, save as pickle last 35 days and plotting takes 556.93 seconds\n"
     ]
    }
   ],
   "source": [
    "if debug_mode > 0: \n",
    "    importlib.reload(hd) \n",
    "    importlib.reload(hp) \n",
    "\n",
    "#test execution times\n",
    "t0 = time.time()\n",
    "\n",
    "print(' ')\n",
    "print('------ STEREO-A beacon data ')\n",
    "\n",
    "#define filename\n",
    "file_sta_beacon_rtn='stereoa_beacon_rtn_last_35days_now.p'   \n",
    "file_sta_beacon_gsm='stereoa_beacon_gsm_last_35days_now.p'   \n",
    "\n",
    "if get_stereoa > 0:\n",
    "\n",
    "    print(' ')\n",
    "    print('--- download STEREO-A beacon data last 10 days ')\n",
    "    hd.stereoa_download_beacon(start=datetime.datetime.utcnow()-datetime.timedelta(days=10),end=datetime.datetime.utcnow(),stereoa_path=stereoa_path)   \n",
    "    print(' ')\n",
    "\n",
    "    print('--- process STEREO-A beacon data to pickle last 35 days') \n",
    "    #save pickle file\n",
    "    \n",
    "    \n",
    "    hd.save_stereoa_beacon_data(data_path,stereoa_path,file_sta_beacon_rtn,file_sta_beacon_gsm,datetime.datetime.utcnow()-datetime.timedelta(days=35),datetime.datetime.utcnow(),coord='RTN' )   \n",
    "    \n",
    "    \n",
    "#load rtn\n",
    "[sta_rtn,hsta]=pickle.load(open(data_path+file_sta_beacon_rtn, \"rb\" ) )  \n",
    "start=datetime.datetime.utcnow() - datetime.timedelta(days=10)\n",
    "end=datetime.datetime.utcnow() \n",
    "hp.plot_insitu_update_stereoa_beacon(sta_rtn, start, end,'STEREO-A_beacon_rtn',plot_path+'stereoa/',coord='RTN',now=True)\n",
    "\n",
    "\n",
    "#load gsm\n",
    "[sta_gsm,hsta]=pickle.load(open(data_path+file_sta_beacon_gsm, \"rb\" ) )  \n",
    "start=datetime.datetime.utcnow() - datetime.timedelta(days=10)\n",
    "end=datetime.datetime.utcnow() \n",
    "\n",
    "\n",
    "hp.plot_insitu_update_stereoa_beacon(sta_gsm, start, end,'STEREO-A_beacon_gsm',plot_path+'stereoa/',coord='GSM',now=True)\n",
    "\n",
    "\n",
    "    \n",
    "    \n",
    "t1 = time.time()\n",
    "\n",
    "print()\n",
    "print('STEREO-A beacon downloading last 10 days, save as pickle last 35 days and plotting takes', np.round(t1-t0,2), 'seconds')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8330e51f-efbc-4ed6-a9f1-6e31ee0fd92c",
   "metadata": {},
   "source": [
    "## Combined plot STEREO-A NOAA RTSW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0886873a-c8f2-4558-91fe-ca92534a92e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chris/python/heliocats/heliocats/plot.py:550: UserWarning:\n",
      "\n",
      "This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved as  /Users/chris/python/data/insitu_python/plots/combined/NOAA_RTSW_STEREO-A_beacon_2025_Jan_12_2025_Jan_16.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/combined/NOAA_RTSW_STEREO-A_beacon_now.png\n",
      "saved as  /Users/chris/python/data/insitu_python/plots/combined/NOAA_RTSW_STEREO-A_beacon_now.html\n"
     ]
    }
   ],
   "source": [
    "if debug_mode > 0:     \n",
    "    importlib.reload(hd) \n",
    "    importlib.reload(hp) \n",
    "\n",
    "start=datetime.datetime.utcnow() - datetime.timedelta(days=4)\n",
    "end=datetime.datetime.utcnow()\n",
    "hp.plot_insitu_update_stereoa_noaa(noaa, sta_gsm, start, end,'NOAA_RTSW_STEREO-A_beacon',plot_path+'combined/',now=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9555be17-8112-44b7-a8cf-9d755e9b1e48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      " \n",
      " \n",
      "------------------\n",
      "Runtime for full high frequency data update: 11.07 minutes\n",
      "--------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "t1all = time.time()\n",
    "\n",
    "print(' ')\n",
    "print(' ')\n",
    "print(' ')\n",
    "print('------------------')\n",
    "print('Runtime for full high frequency data update:', np.round((t1all-t0all)/60,2), 'minutes')\n",
    "print('--------------------------------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28a9a6a0-c089-40f5-882d-c7acf45029f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00ea861f-1b06-4cbe-b853-bfeba24c2109",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "987daf67-7a6d-48ff-bfbc-bc5e83f48974",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64137dfc-58ec-4d40-ba2b-18c3efc05d0a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "136b5d74-9043-4887-9256-3e84ab980ec8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "038cadb3-7968-46fb-99d0-ea29ebba5f37",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "787cf264-f59c-409f-bd3d-b38c2c5ba2f0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}