diff --git a/CumulusMX/Cumulus.cs b/CumulusMX/Cumulus.cs index f929e505..a42610b3 100644 --- a/CumulusMX/Cumulus.cs +++ b/CumulusMX/Cumulus.cs @@ -34,6 +34,7 @@ using System.Web.Caching; using System.Web.UI.WebControls; using static System.Net.Mime.MediaTypeNames; +using System.Text.RegularExpressions; namespace CumulusMX { @@ -722,7 +723,7 @@ public Cumulus(int HTTPport, bool DebugEnabled, string startParms) LogMessage($"Current culture: {CultureInfo.CurrentCulture.DisplayName} [{CultureInfo.CurrentCulture.Name}]"); - LogMessage($"Running as a {(IntPtr.Size == 4 ? "32" : "64")}bit process"); + LogMessage($"Running as a {(IntPtr.Size == 4 ? "32" : "64")} bit process"); // Messy, but Windows and Linux use different mechanisms for loading DLLs, neither are very pretty // Simplest method is to leave them searching the application directory, and copy the correct file there before it is loaded by the first SQlite connect() @@ -1709,7 +1710,7 @@ internal void SetupPhpUploadClients() internal async void TestPhpUploadCompression() { - LogDebugMessage("Testing PHP upload compression"); + LogDebugMessage($"Testing PHP upload compression: '{FtpOptions.PhpUrl}'"); using (var request = new HttpRequestMessage(HttpMethod.Get, FtpOptions.PhpUrl)) { try @@ -3239,7 +3240,7 @@ private async Task RealtimeUpload(byte cycle) } else { - tasklist.Add(UploadFile(phpRealtimeUploadHttpClient, uploadfile, remotefile, cycle, ExtraFiles[idx].binary)); + tasklist.Add(UploadFile(phpRealtimeUploadHttpClient, uploadfile, remotefile, cycle, ExtraFiles[idx].binary, ExtraFiles[idx].UTF8)); } // no extra files are incremental for now, so no need to update LastDataTime @@ -4039,6 +4040,12 @@ private void ReadIniFile() } ProgramOptions.TimeFormat = ini.GetValue("Program", "TimeFormat", "t"); + if (ProgramOptions.TimeFormat == "t") + ProgramOptions.TimeFormatLong = "T"; + else if (ProgramOptions.TimeFormat == "h:mm tt") + ProgramOptions.TimeFormatLong = "h:mm:ss tt"; + else + ProgramOptions.TimeFormatLong = "HH:mm:ss"; ProgramOptions.WarnMultiple = ini.GetValue("Station", "WarnMultiple", true); ProgramOptions.ListWebTags = ini.GetValue("Station", "ListWebTags", false); @@ -9648,12 +9655,12 @@ public async Task DoIntervalUpload() var uploadfile = ReportPath + NOAAconf.LatestMonthReport; var remotefile = NOAAconf.FtpFolder + '/' + NOAAconf.LatestMonthReport; - tasklist.Add(UploadFile(phpUploadHttpClient, uploadfile, remotefile)); + tasklist.Add(UploadFile(phpUploadHttpClient, uploadfile, remotefile, -1, false, NOAAconf.UseUtf8)); uploadfile = ReportPath + NOAAconf.LatestYearReport; remotefile = NOAAconf.FtpFolder + '/' + NOAAconf.LatestYearReport; - tasklist.Add(UploadFile(phpUploadHttpClient, uploadfile, remotefile)); + tasklist.Add(UploadFile(phpUploadHttpClient, uploadfile, remotefile, -1, false, NOAAconf.UseUtf8)); LogFtpDebugMessage("PHP[Int]: Upload of NOAA reports complete"); NOAAconf.NeedFtp = false; } @@ -9689,20 +9696,20 @@ public async Task DoIntervalUpload() // all checks OK, file needs to be uploaded try { + var idx = i; + if (ExtraFiles[i].process) { - var encoding = UTF8encode ? new UTF8Encoding(false) : Encoding.GetEncoding("iso-8859-1"); - tokenParser.Encoding = encoding; tasklist.Add(Task.Run(async () => { - var data = await ProcessTemplateFile2StringAsync(uploadfile, false); - _ = await UploadString(phpUploadHttpClient, false, string.Empty, data, remotefile); + var data = await ProcessTemplateFile2StringAsync(uploadfile, false, ExtraFiles[idx].UTF8); + _ = await UploadString(phpUploadHttpClient, false, string.Empty, data, remotefile, -1, false, ExtraFiles[idx].UTF8); })); } else { - tasklist.Add(UploadFile(phpUploadHttpClient, uploadfile, remotefile)); + tasklist.Add(UploadFile(phpUploadHttpClient, uploadfile, remotefile, -1, false, ExtraFiles[idx].UTF8)); } } catch (Exception e) @@ -10156,7 +10163,7 @@ private bool UploadStream(SftpClient conn, string remotefile, Stream dataStream, // Return True if the upload worked // Return False if the upload failed - private async Task UploadFile(HttpClient httpclient, string localfile, string remotefile, int cycle = -1, bool binary = false) + private async Task UploadFile(HttpClient httpclient, string localfile, string remotefile, int cycle = -1, bool binary = false, bool utf8=true) { string cycleStr = cycle >= 0 ? cycle.ToString() : "Int"; @@ -10170,7 +10177,7 @@ private async Task UploadFile(HttpClient httpclient, string localfile, str return false; } - var utf8 = new UTF8Encoding(false); + var encoding = utf8 ? new UTF8Encoding(false) : Encoding.GetEncoding("iso-8859-1"); string data; if (binary) @@ -10180,7 +10187,7 @@ private async Task UploadFile(HttpClient httpclient, string localfile, str } else { - data = await Utils.ReadAllTextAsync(localfile, utf8); + data = await Utils.ReadAllTextAsync(localfile, encoding); } return await UploadString(httpclient, false, string.Empty, data, remotefile, cycle, binary); @@ -11077,32 +11084,6 @@ public void DoExtraEndOfDayFiles() { remotefile = GetRemoteFileName(remotefile, logDay); - if (!ExtraFiles[i].FTP) - { - uploadfile += "tmp"; - try - { - if (ExtraFiles[i].process) - { - LogDebugMessage("EOD: Processing extra file " + uploadfile); - // process the file - var output = ProcessTemplateFile2String(uploadfile, tokenParser, false, ExtraFiles[i].UTF8); - File.WriteAllText(remotefile, output); - } - else - { - LogDebugMessage("EOD: Copying extra file " + uploadfile); - File.Copy(uploadfile, remotefile); - } - } - catch (Exception ex) - { - LogDebugMessage("EOD: Error writing file " + uploadfile); - LogDebugMessage(ex.Message); - } - //LogDebugMessage("Finished processing extra file " + uploadfile); - } - if (ExtraFiles[i].FTP) { // FTP the file at the next interval @@ -12055,6 +12036,29 @@ private string GetUploadFilename(string input, DateTime dat) NOAAReports noaa = new NOAAReports(this, station); return noaa.GetLastNoaaMonthReportFilename(dat, true); } + else if (input.StartsWith(""); + + if (match.Success) + { + var idx = int.Parse(match.Groups[1].Value) - 1; // we use a zero relative array + + return GetCustomIntvlLogFileName(idx, dat); + } + else + { + LogMessage("GetUploadFilename: No match found for in " + input); + return input; + } + } + catch (Exception ex) + { + LogMessage($"GetUploadFilename: Error processing , value='{input}', error: {ex.Message}"); + } + } return input; } @@ -12083,6 +12087,29 @@ private string GetRemoteFileName(string input, DateTime dat) NOAAReports noaa = new NOAAReports(this, station); return input.Replace("", Path.GetFileName(noaa.GetLastNoaaMonthReportFilename(dat, false))); } + else if (input.Contains(""); + + if (match.Success) + { + var idx = int.Parse(match.Groups[1].Value) - 1; // we use a zero relative array + + return input.Replace(match.Groups[0].Value, Path.GetFileName(GetCustomIntvlLogFileName(idx, dat))); + } + else + { + LogMessage("GetRemoteFileName: No match found for in " + input); + return input; + } + } + catch (Exception ex) + { + LogMessage($"GetRemoteFileName: Error processing , input='{input}', error: {ex.Message}"); + } + } return input; } @@ -12261,6 +12288,7 @@ public class ProgramOptionsClass public bool DataStoppedExit { get; set; } public int DataStoppedMins { get; set; } public string TimeFormat { get; set; } + public string TimeFormatLong { get; set; } } public class CultureConfig diff --git a/CumulusMX/CumulusMX.csproj b/CumulusMX/CumulusMX.csproj index a2100b21..0641232b 100644 --- a/CumulusMX/CumulusMX.csproj +++ b/CumulusMX/CumulusMX.csproj @@ -57,6 +57,7 @@ 4 ".pdb"="" false + false diff --git a/CumulusMX/DavisStation.cs b/CumulusMX/DavisStation.cs index f70863ad..577c3f3a 100644 --- a/CumulusMX/DavisStation.cs +++ b/CumulusMX/DavisStation.cs @@ -2078,7 +2078,7 @@ private void GetArchiveData() bool midnightraindone = luhour == 0; // work out the next logger interval after the last CMX update - var nextLoggerTime = Utils.RoundTimeUpToInterval(cumulus.LastUpdateTime, TimeSpan.FromMinutes(loggerInterval)); + var nextLoggerTime = Utils.RoundTimeUpToInterval(cumulus.LastUpdateTime.AddMinutes(-1), TimeSpan.FromMinutes(loggerInterval)); // check if the calculated logger time is later than now! if (nextLoggerTime > DateTime.Now) diff --git a/CumulusMX/DavisWllStation.cs b/CumulusMX/DavisWllStation.cs index afe7ead8..32e7b7b4 100644 --- a/CumulusMX/DavisWllStation.cs +++ b/CumulusMX/DavisWllStation.cs @@ -259,7 +259,7 @@ public override void Start() // Create a current conditions thread to poll readings every 10 seconds as temperature updates every 10 seconds GetWllCurrent(null, null); tmrCurrent.Elapsed += GetWllCurrent; - tmrCurrent.Interval = 10 * 1000; // Every 10 seconds + tmrCurrent.Interval = 20 * 1000; // Every 10 seconds tmrCurrent.AutoReset = true; tmrCurrent.Start(); @@ -910,15 +910,20 @@ private void DecodeCurrent(string currentJson) WindRecent[nextwind].Timestamp = dateTime; nextwind = (nextwind + 1) % MaxWindRecent; - RecentMaxGust = gustCal; + if (broadcastStopped) + { + // if we are not receiving live broadcasts, then use current gust value) + RecentMaxGust = gustCal; + } } } } else if (!CalcRecentMaxGust) { // Check for spikes, and set highs - if (CheckHighGust(gustCal, gustDir, dateTime)) + if (CheckHighGust(gustCal, gustDir, dateTime) && broadcastStopped) { + // if we are not receiving live broadcasts, then use current gust value RecentMaxGust = gustCal; } } diff --git a/CumulusMX/ProgramSettings.cs b/CumulusMX/ProgramSettings.cs index 2e06f649..cbf3eca4 100644 --- a/CumulusMX/ProgramSettings.cs +++ b/CumulusMX/ProgramSettings.cs @@ -143,6 +143,13 @@ public string UpdateConfig(IHttpContext context) cumulus.ProgramOptions.TimeFormat = settings.culture.timeFormat; cumulus.ProgramOptions.Culture.RemoveSpaceFromDateSeparator = settings.culture.removespacefromdateseparator; + if (cumulus.ProgramOptions.TimeFormat == "t") + cumulus.ProgramOptions.TimeFormatLong = "T"; + else if (cumulus.ProgramOptions.TimeFormat == "h:mm tt") + cumulus.ProgramOptions.TimeFormatLong = "h:mm:ss tt"; + else + cumulus.ProgramOptions.TimeFormatLong = "HH:mm:ss"; + if (cumulus.ProgramOptions.Culture.RemoveSpaceFromDateSeparator && CultureInfo.CurrentCulture.DateTimeFormat.DateSeparator.Contains(" ")) { diff --git a/CumulusMX/Properties/AssemblyInfo.cs b/CumulusMX/Properties/AssemblyInfo.cs index baec68cb..15d65b86 100644 --- a/CumulusMX/Properties/AssemblyInfo.cs +++ b/CumulusMX/Properties/AssemblyInfo.cs @@ -6,7 +6,7 @@ // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("Cumulus MX")] -[assembly: AssemblyDescription("Version 3.24.1 - Build 3234")] +[assembly: AssemblyDescription("Version 3.24.2 - Build 3235")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] [assembly: AssemblyProduct("Cumulus MX")] @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("3.24.1.3234")] -[assembly: AssemblyFileVersion("3.24.1.3234")] +[assembly: AssemblyVersion("3.24.2.3235")] +[assembly: AssemblyFileVersion("3.24.2.3235")] diff --git a/CumulusMX/StationSettings.cs b/CumulusMX/StationSettings.cs index 1893a00a..8bbdf7dc 100644 --- a/CumulusMX/StationSettings.cs +++ b/CumulusMX/StationSettings.cs @@ -6,6 +6,8 @@ using EmbedIO; using System.Reflection; using static Swan.Terminal; +using Swan.Formatters; +using ServiceStack; namespace CumulusMX { @@ -1249,8 +1251,7 @@ internal string UploadNow(IHttpContext context) var json = WebUtility.UrlDecode(data); var now = DateTime.Now; - // Dead simple (dirty), there is only one setting at present! - var includeGraphs = json.Contains("true"); + var options = json.FromJson(); if (!cumulus.FtpOptions.Enabled && !cumulus.FtpOptions.LocalCopyEnabled) return "Upload/local copy is not enabled!"; @@ -1274,25 +1275,40 @@ internal string UploadNow(IHttpContext context) } // Graph configs may have changed, so force re-create and upload the json files - just flag everything! - cumulus.LogDebugMessage("Upload Now: Flagging the graph data files for full recreation, if required"); + cumulus.LogDebugMessage("Upload Now: Flagging the graph data files for recreation and upload/copy"); + if (options.graphs) + cumulus.LogDebugMessage("Upload Now: Flagging graph data files for full upload rather than incremental"); + for (var i = 0; i < cumulus.GraphDataFiles.Length; i++) { cumulus.GraphDataFiles[i].CreateRequired = true; cumulus.GraphDataFiles[i].FtpRequired = true; cumulus.GraphDataFiles[i].CopyRequired = true; - cumulus.GraphDataFiles[i].Incremental = false; + if (options.graphs) + cumulus.GraphDataFiles[i].Incremental = false; cumulus.GraphDataFiles[i].LastDataTime = DateTime.MinValue; } // (re)generate the daily graph data files, and upload if required - cumulus.LogDebugMessage("Upload Now: Flagging the daily graph data files for full recreation, if required"); - for (var i = 0; i < cumulus.GraphDataEodFiles.Length; i++) + if (options.dailygraphs) { - cumulus.GraphDataEodFiles[i].CreateRequired = true; - cumulus.GraphDataEodFiles[i].FtpRequired = true; - cumulus.GraphDataEodFiles[i].CopyRequired = true; - cumulus.GraphDataEodFiles[i].Incremental = false; - cumulus.GraphDataEodFiles[i].LastDataTime = DateTime.MinValue; + cumulus.LogDebugMessage("Upload Now: Flagging the daily graph data files for recreation and upload/copy"); + for (var i = 0; i < cumulus.GraphDataEodFiles.Length; i++) + { + cumulus.GraphDataEodFiles[i].CreateRequired = true; + cumulus.GraphDataEodFiles[i].FtpRequired = true; + cumulus.GraphDataEodFiles[i].CopyRequired = true; + cumulus.GraphDataEodFiles[i].Incremental = false; + cumulus.GraphDataEodFiles[i].LastDataTime = DateTime.MinValue; + } + } + + // flag the latest NOAA files for upload + if (options.noaa) + { + cumulus.LogDebugMessage("Upload Now: Flagging the latest NOAA report files for upload/copy"); + cumulus.NOAAconf.NeedFtp = true; + cumulus.NOAAconf.NeedCopy = true; } cumulus.WebUpdating = 1; @@ -1308,6 +1324,13 @@ internal string UploadNow(IHttpContext context) } } + private class UploadNowData + { + public bool dailygraphs { get; set; } + public bool noaa { get; set; } + public bool graphs { get; set; } + } + internal string SetSelectaChartOptions(IHttpContext context) { var errorMsg = ""; diff --git a/CumulusMX/WeatherStation.cs b/CumulusMX/WeatherStation.cs index 3bc026bb..6b12e57c 100644 --- a/CumulusMX/WeatherStation.cs +++ b/CumulusMX/WeatherStation.cs @@ -2194,6 +2194,7 @@ public string CreateGraphDataJson(string filename, bool incremental) public void CreateEodGraphDataFiles() { string json = ""; + for (var i = 0; i < cumulus.GraphDataEodFiles.Length; i++) { if (cumulus.GraphDataEodFiles[i].Create) @@ -2203,20 +2204,17 @@ public void CreateEodGraphDataFiles() try { var dest = cumulus.GraphDataEodFiles[i].LocalPath + cumulus.GraphDataEodFiles[i].LocalFileName; - using (var file = new StreamWriter(dest, false)) - { - file.WriteLine(json); - file.Close(); - } - // Now set the flag that upload is required (if enabled) - cumulus.GraphDataEodFiles[i].FtpRequired = true; - cumulus.GraphDataEodFiles[i].CopyRequired = true; + File.WriteAllText(dest, json); } catch (Exception ex) { cumulus.LogMessage($"Error writing {cumulus.GraphDataEodFiles[i].LocalFileName}: {ex}"); } } + + // Now set the flag that upload is required (if enabled) + cumulus.GraphDataEodFiles[i].FtpRequired = true; + cumulus.GraphDataEodFiles[i].CopyRequired = true; } } @@ -13541,7 +13539,7 @@ internal string GetCurrentData() HiLoToday.LowAppTemp, HiLoToday.HighAppTempTime.ToString(cumulus.ProgramOptions.TimeFormat), HiLoToday.LowAppTempTime.ToString(cumulus.ProgramOptions.TimeFormat), (int)Math.Round(CurrentSolarMax), AllTime.HighPress.Val, AllTime.LowPress.Val, SunshineHours, CompassPoint(DominantWindBearing), LastRainTip, HiLoToday.HighHourlyRain, HiLoToday.HighHourlyRainTime.ToString(cumulus.ProgramOptions.TimeFormat), "F" + cumulus.Beaufort(HiLoToday.HighWind), "F" + cumulus.Beaufort(WindAverage), - cumulus.BeaufortDesc(WindAverage), LastDataReadTimestamp.ToString("T"), DataStopped, StormRain, stormRainStart, CloudBase, cumulus.CloudBaseInFeet ? "ft" : "m", RainLast24Hour, + cumulus.BeaufortDesc(WindAverage), LastDataReadTimestamp.ToString(cumulus.ProgramOptions.TimeFormatLong), DataStopped, StormRain, stormRainStart, CloudBase, cumulus.CloudBaseInFeet ? "ft" : "m", RainLast24Hour, cumulus.LowTempAlarm.Triggered, cumulus.HighTempAlarm.Triggered, cumulus.TempChangeAlarm.UpTriggered, cumulus.TempChangeAlarm.DownTriggered, cumulus.HighRainTodayAlarm.Triggered, cumulus.HighRainRateAlarm.Triggered, cumulus.LowPressAlarm.Triggered, cumulus.HighPressAlarm.Triggered, cumulus.PressChangeAlarm.UpTriggered, cumulus.PressChangeAlarm.DownTriggered, cumulus.HighGustAlarm.Triggered, cumulus.HighWindAlarm.Triggered, cumulus.SensorAlarm.Triggered, cumulus.BatteryLowAlarm.Triggered, cumulus.SpikeAlarm.Triggered, cumulus.UpgradeAlarm.Triggered, cumulus.HttpUploadAlarm.Triggered, cumulus.MySqlUploadAlarm.Triggered, cumulus.IsRainingAlarm.Triggered, diff --git a/Updates.txt b/Updates.txt index c08c4179..bbedf772 100644 --- a/Updates.txt +++ b/Updates.txt @@ -1,3 +1,21 @@ +3.24.2 - b3235 +—————————————— +New +- Extra Files now allows you to use the variables thru in the source and destination fields to specify custom log files to be uploaded/copied +- Copy/Upload Now! gets a new option to transfer the latest NOAA monthly/year reports + +Changed +- Copy/Upload Now! the upload of full rather than incremental versions of graph data files is now optional + +Fixed +- Davis VP2 logger suppression of full logger downloads if CMX is stopped and immediately restarted fractionally after a logging interval +- Extra Files with EOD flag having "tmp" appended to all source files +- Not running as a 64 bit application on Windows +- NOAA report PHP uploads were no using the UTF8 flag +- Davis WLL using "stale" current wind gust data +- Daily graphs not being uploaded/copied after day reset + + 3.24.1 - b3234 —————————————— New