From 7733ba82d33e185a1bc5cde81f6c0d2792a53a17 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Fri, 15 Oct 2021 15:38:27 -0400 Subject: [PATCH 01/42] adds translation for selenium code files --- src/Ghosts.Domain/Code/Helpers.cs | 8 ++ .../Handlers/BaseBrowserHandler.cs | 19 +++++ .../Handlers/BrowserFirefox.cs | 4 + .../TimelineManager/Listener.cs | 78 +++++++++++++------ .../TimelineManager/TimelineTranslator.cs | 78 +++++++++++++++++++ 5 files changed, 165 insertions(+), 22 deletions(-) create mode 100644 src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs diff --git a/src/Ghosts.Domain/Code/Helpers.cs b/src/Ghosts.Domain/Code/Helpers.cs index a74931b5..f6b065a0 100755 --- a/src/Ghosts.Domain/Code/Helpers.cs +++ b/src/Ghosts.Domain/Code/Helpers.cs @@ -30,6 +30,14 @@ public static bool IsOlderThanHours(string filename, int hours) public static class StringExtensions { + public static string GetTextBetweenQuotes(this string o) + { + var result = Regex.Match(o, "\"([^\"]*)\"").ToString(); + if(!string.IsNullOrEmpty(result)) + result = result.TrimStart('"').TrimEnd('"'); + return result; + } + public static string ReplaceCaseInsensitive(this string input, string search, string replacement) { var result = Regex.Replace(input, Regex.Escape(search), replacement.Replace("$", "$$"), RegexOptions.IgnoreCase); diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index 7e4697c9..c05be22c 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -15,6 +15,7 @@ public abstract class BaseBrowserHandler : BaseHandler { public static readonly Logger _log = LogManager.GetCurrentClassLogger(); public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } public HandlerType BrowserType { get; set; } private int _stickiness = 0; private int _depthMin = 1; @@ -157,15 +158,33 @@ public void ExecuteEvents(TimelineHandler handler) actions.SendKeys(element, timelineEvent.CommandArgs[1].ToString()).Build().Perform(); break; case "click": + case "click.by.name": element = Driver.FindElement(By.Name(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; case "clickbyid": + case "click.by.id": element = Driver.FindElement(By.Id(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; + case "click.by.linktext": + element = Driver.FindElement(By.LinkText(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "click.by.cssselector": + element = Driver.FindElement(By.CssSelector(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "js.executescript": + JS.ExecuteScript(timelineEvent.CommandArgs[0].ToString()); + break; + case "manage.window.size": + Driver.Manage().Window.Size = new System.Drawing.Size(Convert.ToInt32(timelineEvent.CommandArgs[0]), Convert.ToInt32(timelineEvent.CommandArgs[1])); + break; } if (timelineEvent.DelayAfter > 0) diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index 6d9f9506..ec2805bd 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -15,6 +15,7 @@ public class BrowserFirefox : BaseBrowserHandler private static readonly Logger _log = LogManager.GetCurrentClassLogger(); public IWebDriver Driver { get; private set; } + public IJavaScriptExecutor JS { get; private set; } public BrowserFirefox(TimelineHandler handler) { @@ -89,6 +90,9 @@ private bool FirefoxEx(TimelineHandler handler) Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); Driver = new FirefoxDriver(options); base.Driver = Driver; + + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; //hack: bad urls used in the past... if (handler.Initial.Equals("") || diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index a279804c..b9559880 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -1,8 +1,10 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.IO; using System.Threading; +using ghosts.client.linux.handlers; using Ghosts.Domain; using Ghosts.Domain.Code; using Newtonsoft.Json; @@ -94,7 +96,7 @@ public DirectoryListener() { Path = _in, NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.FileName, - Filter = "*.json" + Filter = "*.*" }; watcher.Changed += OnChanged; watcher.Created += OnChanged; @@ -112,38 +114,70 @@ private void OnChanged(object source, FileSystemEventArgs e) if (!File.Exists(e.FullPath)) return; - - try - { - var raw = File.ReadAllText(e.FullPath); - - var timeline = JsonConvert.DeserializeObject(raw); - foreach (var timelineHandler in timeline.TimeLineHandlers) + if (e.FullPath.EndsWith(".json")) + { + try { - _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); + var raw = File.ReadAllText(e.FullPath); + + var timeline = JsonConvert.DeserializeObject(raw); - foreach (var timelineEvent in timelineHandler.TimeLineEvents) + foreach (var timelineHandler in timeline.TimeLineHandlers) { - if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); + + foreach (var timelineEvent in timelineHandler.TimeLineEvents) { - timelineEvent.TrackableId = Guid.NewGuid().ToString(); + if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + { + timelineEvent.TrackableId = Guid.NewGuid().ToString(); + } } - } - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + var orchestrator = new Orchestrator(); + orchestrator.RunCommand(timelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); } - - var outfile = e.FullPath.Replace(_in, _out); - outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); - - File.Move(e.FullPath, outfile); } - catch (Exception exc) + else if (e.FullPath.EndsWith(".cs")) { - _log.Debug(exc); + try + { + var commands = new List(); + var raw = File.ReadAllText(e.FullPath); + + var lines = raw.Split(Convert.ToChar("\n")); + foreach (var line in lines) + { + var l = line.Trim(); + if ((l.StartsWith("driver.") || l.StartsWith("js.")) && !l.StartsWith("driver.Quit")) + { + commands.Add(l); + } + } + + if (commands.Count > 0) + { + var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); + var orchestrator = new Orchestrator(); + orchestrator.RunCommand(constructedTimelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); + } } + + var outfile = e.FullPath.Replace(_in, _out); + outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); + + File.Move(e.FullPath, outfile); _currentlyProcessing = string.Empty; } diff --git a/src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs b/src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs new file mode 100644 index 00000000..a4621a01 --- /dev/null +++ b/src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using Ghosts.Domain; +using Ghosts.Domain.Code; + +namespace ghosts.client.linux.timelineManager +{ + public static class TimelineTranslator + { + public static TimelineHandler FromBrowserUnitTests(IEnumerable commands) + { + var timelineHandler = new TimelineHandler(); + + timelineHandler.HandlerType = HandlerType.BrowserFirefox; + timelineHandler.Initial = "about:blank"; + timelineHandler.Loop = false; + + foreach (var command in commands) + { + var timelineEvent = new TimelineEvent(); + timelineEvent.DelayBefore = 0; + timelineEvent.DelayAfter = 3000; + + // determine command and commandArgs + if (command.StartsWith("driver.Navigate()", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "browse"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.Manage().Window.Size", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "manage.window.size"; + var s = command.Split("Size(")[1].Replace(");", "").Replace(" ","").Split(","); + var width = Convert.ToInt32(s[0]); + var height = Convert.ToInt32(s[1]); + timelineEvent.CommandArgs.Add(width); + timelineEvent.CommandArgs.Add(height); + } + else if (command.StartsWith("js.", StringComparison.InvariantCultureIgnoreCase)) + { + if (command.StartsWith("js.ExecuteScript(", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "js.executescript"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + } + else if(command.StartsWith("driver.FindElement(", StringComparison.InvariantCultureIgnoreCase) && command.EndsWith(").Click();", StringComparison.InvariantCultureIgnoreCase)) + { + if (command.StartsWith("driver.FindElement(By.LinkText(", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.linktext"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.Id", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.id"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.Name", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.name"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.CssSelector", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.cssselector"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + } + + if(!string.IsNullOrEmpty(timelineEvent.Command) && timelineEvent.CommandArgs.Count > 0) + timelineHandler.TimeLineEvents.Add(timelineEvent); + } + + return timelineHandler; + } + } +} \ No newline at end of file From 756d2bbd6325a597c7791e0322717436d8cd0d78 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Fri, 15 Oct 2021 16:43:39 -0400 Subject: [PATCH 02/42] moves shared code to domain --- src/Ghosts.Domain/Code/Helpers.cs | 5 + src/Ghosts.Domain/Code/TimelineTranslator.cs | 96 +++++++++++++++++++ .../Handlers/BrowserFirefox.cs | 6 +- .../TimelineManager/Listener.cs | 15 ++- .../TimelineManager/TimelineTranslator.cs | 78 --------------- 5 files changed, 115 insertions(+), 85 deletions(-) create mode 100644 src/Ghosts.Domain/Code/TimelineTranslator.cs delete mode 100644 src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs diff --git a/src/Ghosts.Domain/Code/Helpers.cs b/src/Ghosts.Domain/Code/Helpers.cs index f6b065a0..60bb0ce9 100755 --- a/src/Ghosts.Domain/Code/Helpers.cs +++ b/src/Ghosts.Domain/Code/Helpers.cs @@ -30,6 +30,11 @@ public static bool IsOlderThanHours(string filename, int hours) public static class StringExtensions { + public static IEnumerable Split(this string o, string splitString) + { + return o.Split(new [] { splitString }, StringSplitOptions.None); + } + public static string GetTextBetweenQuotes(this string o) { var result = Regex.Match(o, "\"([^\"]*)\"").ToString(); diff --git a/src/Ghosts.Domain/Code/TimelineTranslator.cs b/src/Ghosts.Domain/Code/TimelineTranslator.cs new file mode 100644 index 00000000..54efe111 --- /dev/null +++ b/src/Ghosts.Domain/Code/TimelineTranslator.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NLog; + +namespace Ghosts.Domain.Code +{ + public static class TimelineTranslator + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public static TimelineHandler FromBrowserUnitTests(IEnumerable commands) + { + var timelineHandler = new TimelineHandler(); + + timelineHandler.HandlerType = HandlerType.BrowserFirefox; + timelineHandler.Initial = "about:blank"; + timelineHandler.Loop = false; + + foreach (var command in commands) + { + TimelineEvent timelineEvent = null; + try + { + timelineEvent = GetEvent(command); + } + catch (Exception exc) + { + _log.Error(exc); + } + + if (timelineEvent != null && !string.IsNullOrEmpty(timelineEvent.Command) && timelineEvent.CommandArgs.Count > 0) + timelineHandler.TimeLineEvents.Add(timelineEvent); + } + + return timelineHandler; + } + + private static TimelineEvent GetEvent(string command) + { + var timelineEvent = new TimelineEvent(); + timelineEvent.DelayBefore = 0; + timelineEvent.DelayAfter = 3000; + + // determine command and commandArgs + if (command.StartsWith("driver.Navigate()", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "browse"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.Manage().Window.Size", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "manage.window.size"; + var size = command.Split("Size(").Last().Replace(");", "").Replace(" ", "").Split(Convert.ToChar(",")); + var width = Convert.ToInt32(size[0]); + var height = Convert.ToInt32(size[1]); + timelineEvent.CommandArgs.Add(width); + timelineEvent.CommandArgs.Add(height); + } + else if (command.StartsWith("js.", StringComparison.InvariantCultureIgnoreCase)) + { + if (command.StartsWith("js.ExecuteScript(", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "js.executescript"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + } + else if (command.StartsWith("driver.FindElement(", StringComparison.InvariantCultureIgnoreCase) && + command.EndsWith(").Click();", StringComparison.InvariantCultureIgnoreCase)) + { + if (command.StartsWith("driver.FindElement(By.LinkText(", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.linktext"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.Id", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.id"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.Name", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.name"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.CssSelector", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.cssselector"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + } + + return timelineEvent; + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index ec2805bd..b2d6c266 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -12,10 +12,10 @@ namespace ghosts.client.linux.handlers { public class BrowserFirefox : BaseBrowserHandler { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private new static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public IWebDriver Driver { get; private set; } - public IJavaScriptExecutor JS { get; private set; } + public new IWebDriver Driver { get; private set; } + public new IJavaScriptExecutor JS { get; private set; } public BrowserFirefox(TimelineHandler handler) { diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index b9559880..02ee6c17 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -173,11 +173,18 @@ private void OnChanged(object source, FileSystemEventArgs e) _log.Debug(exc); } } - - var outfile = e.FullPath.Replace(_in, _out); - outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); - File.Move(e.FullPath, outfile); + try + { + var outfile = e.FullPath.Replace(_in, _out); + outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); + + File.Move(e.FullPath, outfile); + } + catch (Exception exception) + { + _log.Debug(exception); + } _currentlyProcessing = string.Empty; } diff --git a/src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs b/src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs deleted file mode 100644 index a4621a01..00000000 --- a/src/ghosts.client.linux/TimelineManager/TimelineTranslator.cs +++ /dev/null @@ -1,78 +0,0 @@ -using System; -using System.Collections.Generic; -using Ghosts.Domain; -using Ghosts.Domain.Code; - -namespace ghosts.client.linux.timelineManager -{ - public static class TimelineTranslator - { - public static TimelineHandler FromBrowserUnitTests(IEnumerable commands) - { - var timelineHandler = new TimelineHandler(); - - timelineHandler.HandlerType = HandlerType.BrowserFirefox; - timelineHandler.Initial = "about:blank"; - timelineHandler.Loop = false; - - foreach (var command in commands) - { - var timelineEvent = new TimelineEvent(); - timelineEvent.DelayBefore = 0; - timelineEvent.DelayAfter = 3000; - - // determine command and commandArgs - if (command.StartsWith("driver.Navigate()", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "browse"; - timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); - } - else if (command.StartsWith("driver.Manage().Window.Size", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "manage.window.size"; - var s = command.Split("Size(")[1].Replace(");", "").Replace(" ","").Split(","); - var width = Convert.ToInt32(s[0]); - var height = Convert.ToInt32(s[1]); - timelineEvent.CommandArgs.Add(width); - timelineEvent.CommandArgs.Add(height); - } - else if (command.StartsWith("js.", StringComparison.InvariantCultureIgnoreCase)) - { - if (command.StartsWith("js.ExecuteScript(", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "js.executescript"; - timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); - } - } - else if(command.StartsWith("driver.FindElement(", StringComparison.InvariantCultureIgnoreCase) && command.EndsWith(").Click();", StringComparison.InvariantCultureIgnoreCase)) - { - if (command.StartsWith("driver.FindElement(By.LinkText(", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "click.by.linktext"; - timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); - } - else if (command.StartsWith("driver.FindElement(By.Id", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "click.by.id"; - timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); - } - else if (command.StartsWith("driver.FindElement(By.Name", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "click.by.name"; - timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); - } - else if (command.StartsWith("driver.FindElement(By.CssSelector", StringComparison.InvariantCultureIgnoreCase)) - { - timelineEvent.Command = "click.by.cssselector"; - timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); - } - } - - if(!string.IsNullOrEmpty(timelineEvent.Command) && timelineEvent.CommandArgs.Count > 0) - timelineHandler.TimeLineEvents.Add(timelineEvent); - } - - return timelineHandler; - } - } -} \ No newline at end of file From 67750271f274ba6ebcfe29bf87ad1fca0f7c27e9 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Fri, 15 Oct 2021 16:53:17 -0400 Subject: [PATCH 03/42] further simplifies browse --- src/Ghosts.Domain/Code/TimelineTranslator.cs | 5 ++++- .../TimelineManager/Listener.cs | 15 ++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/Ghosts.Domain/Code/TimelineTranslator.cs b/src/Ghosts.Domain/Code/TimelineTranslator.cs index 54efe111..9a4f5670 100644 --- a/src/Ghosts.Domain/Code/TimelineTranslator.cs +++ b/src/Ghosts.Domain/Code/TimelineTranslator.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using NLog; +using NPOI.HSSF.Record; namespace Ghosts.Domain.Code { @@ -22,7 +23,7 @@ public static TimelineHandler FromBrowserUnitTests(IEnumerable commands) TimelineEvent timelineEvent = null; try { - timelineEvent = GetEvent(command); + timelineEvent = GetEvent(command.Trim()); } catch (Exception exc) { @@ -42,6 +43,8 @@ private static TimelineEvent GetEvent(string command) timelineEvent.DelayBefore = 0; timelineEvent.DelayAfter = 3000; + if (command.StartsWith("driver.Quit", StringComparison.InvariantCultureIgnoreCase)) return timelineEvent; + // determine command and commandArgs if (command.StartsWith("driver.Navigate()", StringComparison.InvariantCultureIgnoreCase)) { diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index 02ee6c17..81c33321 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using ghosts.client.linux.handlers; using Ghosts.Domain; @@ -148,19 +149,7 @@ private void OnChanged(object source, FileSystemEventArgs e) { try { - var commands = new List(); - var raw = File.ReadAllText(e.FullPath); - - var lines = raw.Split(Convert.ToChar("\n")); - foreach (var line in lines) - { - var l = line.Trim(); - if ((l.StartsWith("driver.") || l.StartsWith("js.")) && !l.StartsWith("driver.Quit")) - { - commands.Add(l); - } - } - + var commands = File.ReadAllText(e.FullPath).Split(Convert.ToChar("\n")).ToList(); if (commands.Count > 0) { var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); From d29d470e18ffeccfb5f5692069c0f2b3d6d6870f Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Fri, 15 Oct 2021 16:59:36 -0400 Subject: [PATCH 04/42] adds browse by script to windows client --- .../Handlers/BaseBrowserHandler.cs | 19 +++ src/Ghosts.Client/Handlers/BrowserChrome.cs | 6 + src/Ghosts.Client/Handlers/BrowserFirefox.cs | 8 +- src/Ghosts.Client/TimelineManager/Listener.cs | 116 +++++++++++------- 4 files changed, 102 insertions(+), 47 deletions(-) diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index 965265f6..be13b2c6 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -15,6 +15,7 @@ public abstract class BaseBrowserHandler : BaseHandler { public static readonly Logger _log = LogManager.GetCurrentClassLogger(); public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } public HandlerType BrowserType { get; set; } private int _stickiness = 0; private int _depthMin = 1; @@ -157,15 +158,33 @@ public void ExecuteEvents(TimelineHandler handler) actions.SendKeys(element, timelineEvent.CommandArgs[1].ToString()).Build().Perform(); break; case "click": + case "click.by.name": element = Driver.FindElement(By.Name(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; case "clickbyid": + case "click.by.id": element = Driver.FindElement(By.Id(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; + case "click.by.linktext": + element = Driver.FindElement(By.LinkText(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "click.by.cssselector": + element = Driver.FindElement(By.CssSelector(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "js.executescript": + JS.ExecuteScript(timelineEvent.CommandArgs[0].ToString()); + break; + case "manage.window.size": + Driver.Manage().Window.Size = new System.Drawing.Size(Convert.ToInt32(timelineEvent.CommandArgs[0]), Convert.ToInt32(timelineEvent.CommandArgs[1])); + break; } if (timelineEvent.DelayAfter > 0) diff --git a/src/Ghosts.Client/Handlers/BrowserChrome.cs b/src/Ghosts.Client/Handlers/BrowserChrome.cs index 6d26af02..9173cdd1 100755 --- a/src/Ghosts.Client/Handlers/BrowserChrome.cs +++ b/src/Ghosts.Client/Handlers/BrowserChrome.cs @@ -5,11 +5,14 @@ using OpenQA.Selenium.Chrome; using System; using System.IO; +using OpenQA.Selenium; namespace Ghosts.Client.Handlers { public class BrowserChrome : BaseBrowserHandler { + public new IJavaScriptExecutor JS { get; private set; } + private string GetInstallLocation() { var path = @"C:\Program Files\Google\Chrome\Application\chrome.exe"; @@ -93,6 +96,9 @@ public BrowserChrome(TimelineHandler handler) Driver = new ChromeDriver(options); base.Driver = Driver; + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + Driver.Navigate().GoToUrl(handler.Initial); if (handler.Loop) diff --git a/src/Ghosts.Client/Handlers/BrowserFirefox.cs b/src/Ghosts.Client/Handlers/BrowserFirefox.cs index b7780a57..d604822d 100755 --- a/src/Ghosts.Client/Handlers/BrowserFirefox.cs +++ b/src/Ghosts.Client/Handlers/BrowserFirefox.cs @@ -13,9 +13,10 @@ namespace Ghosts.Client.Handlers { public class BrowserFirefox : BaseBrowserHandler { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private new static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public IWebDriver Driver { get; private set; } + public new IWebDriver Driver { get; private set; } + public new IJavaScriptExecutor JS { get; private set; } public BrowserFirefox(TimelineHandler handler) { @@ -113,6 +114,9 @@ private bool FirefoxEx(TimelineHandler handler) Driver = new FirefoxDriver(options); base.Driver = Driver; + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + //hack: bad urls used in the past... if (handler.Initial.Equals("") || handler.Initial.Equals("about:internal", StringComparison.InvariantCultureIgnoreCase) || diff --git a/src/Ghosts.Client/TimelineManager/Listener.cs b/src/Ghosts.Client/TimelineManager/Listener.cs index 7c93ee54..ead13a0a 100755 --- a/src/Ghosts.Client/TimelineManager/Listener.cs +++ b/src/Ghosts.Client/TimelineManager/Listener.cs @@ -6,6 +6,7 @@ using SimpleTCP; using System; using System.IO; +using System.Linq; using System.Threading; using Ghosts.Domain.Code; @@ -87,7 +88,7 @@ public DirectoryListener() { Path = _in, NotifyFilter = NotifyFilters.LastWrite, - Filter = "*.json" + Filter = "*.*" }; watcher.Changed += new FileSystemEventHandler(OnChanged); watcher.EnableRaisingEvents = true; @@ -95,50 +96,75 @@ public DirectoryListener() private void OnChanged(object source, FileSystemEventArgs e) { - // filewatcher throws multiple events, we only need 1 - if (!string.IsNullOrEmpty(_currentlyProcessing) && _currentlyProcessing == e.FullPath) return; - if (!File.Exists(e.FullPath)) return; - - _currentlyProcessing = e.FullPath; - - _log.Trace("DirectoryListener found file: " + e.FullPath + " " + e.ChangeType); - - try - { - var raw = File.ReadAllText(e.FullPath); - - var timeline = JsonConvert.DeserializeObject(raw); - - foreach (var timelineHandler in timeline.TimeLineHandlers) - { - _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); - - foreach (var timelineEvent in timelineHandler.TimeLineEvents) - { - if (string.IsNullOrEmpty(timelineEvent.TrackableId)) - { - timelineEvent.TrackableId = Guid.NewGuid().ToString(); - } - } - - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); - } - - var outfile = e.FullPath.Replace(_in, _out); - outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); - - File.Move(e.FullPath, outfile); - } - catch (Exception exc) - { - _log.Debug(exc); - } - finally - { - Thread.Sleep(1000); - _currentlyProcessing = string.Empty; - } + // filewatcher throws multiple events, we only need 1 + if (!string.IsNullOrEmpty(_currentlyProcessing) && _currentlyProcessing == e.FullPath) return; + _currentlyProcessing = e.FullPath; + + _log.Trace("DirectoryListener found file: " + e.FullPath + " " + e.ChangeType); + + if (!File.Exists(e.FullPath)) + return; + + if (e.FullPath.EndsWith(".json")) + { + try + { + var raw = File.ReadAllText(e.FullPath); + + var timeline = JsonConvert.DeserializeObject(raw); + + foreach (var timelineHandler in timeline.TimeLineHandlers) + { + _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); + + foreach (var timelineEvent in timelineHandler.TimeLineEvents) + { + if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + { + timelineEvent.TrackableId = Guid.NewGuid().ToString(); + } + } + + var orchestrator = new Orchestrator(); + orchestrator.RunCommand(timelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); + } + } + else if (e.FullPath.EndsWith(".cs")) + { + try + { + var commands = File.ReadAllText(e.FullPath).Split(Convert.ToChar("\n")).ToList(); + if (commands.Count > 0) + { + var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); + var orchestrator = new Orchestrator(); + orchestrator.RunCommand(constructedTimelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); + } + } + + try + { + var outfile = e.FullPath.Replace(_in, _out); + outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); + + File.Move(e.FullPath, outfile); + } + catch (Exception exception) + { + _log.Debug(exception); + } + + _currentlyProcessing = string.Empty; } } From c7fd3f132cf314682bc0f9c7957187c057dd1e8f Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 18 Oct 2021 16:22:05 -0400 Subject: [PATCH 05/42] Introduces trackable controller for getting trackables and their associated history, timeline with a uuid for controlling actions based on a particular timeline (start/stop) --- .../Controllers/TimelineController.cs | 8 ++ .../Controllers/TrackablesController.cs | 42 ++++++ src/Ghosts.Api/Services/TimelineService.cs | 27 ++++ src/Ghosts.Api/Services/TrackableService.cs | 42 ++++++ src/Ghosts.Api/Startup.cs | 1 + .../Code}/TimelineBuilder.cs | 45 ++++++- src/Ghosts.Domain/Messages/Timeline.cs | 25 ++-- src/ghosts.client.linux/Comms/Updates.cs | 2 +- .../TimelineManager/Listener.cs | 41 +++--- .../TimelineManager/Orchestrator.cs | 124 ++++++++---------- 10 files changed, 254 insertions(+), 103 deletions(-) create mode 100644 src/Ghosts.Api/Controllers/TrackablesController.cs create mode 100755 src/Ghosts.Api/Services/TrackableService.cs rename src/{ghosts.client.linux/TimelineManager => Ghosts.Domain/Code}/TimelineBuilder.cs (60%) diff --git a/src/Ghosts.Api/Controllers/TimelineController.cs b/src/Ghosts.Api/Controllers/TimelineController.cs index 83285c09..ee6e8257 100644 --- a/src/Ghosts.Api/Controllers/TimelineController.cs +++ b/src/Ghosts.Api/Controllers/TimelineController.cs @@ -55,6 +55,14 @@ public async Task Timeline([FromBody] MachineUpdateViewModel mach await _timelineService.UpdateAsync(machineUpdate, ct); return NoContent(); } + + [HttpPost("timeline/{machineId}/stop/{timelineId}")] + [SwaggerOperation(OperationId = "stopTimeline")] + public async Task Timeline([FromRoute] Guid machineId, [FromRoute] Guid timelineId, CancellationToken ct) + { + await _timelineService.StopAsync(machineId, timelineId, ct); + return NoContent(); + } /// /// Send a new timeline to an entire group of machines diff --git a/src/Ghosts.Api/Controllers/TrackablesController.cs b/src/Ghosts.Api/Controllers/TrackablesController.cs new file mode 100644 index 00000000..e3c0e64d --- /dev/null +++ b/src/Ghosts.Api/Controllers/TrackablesController.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Services; +using Microsoft.AspNetCore.Mvc; + +namespace ghosts.api.Controllers +{ + [Produces("application/json")] + [Route("api/[controller]")] + [ResponseCache(Duration = 5)] + public class TrackablesController : Controller + { + private readonly ITrackableService _service; + + public TrackablesController(ITrackableService service) + { + _service = service; + } + + /// + /// Gets all trackables in the system + /// + /// Cancellation Token + /// List of Trackables + [HttpGet] + public async Task GetTrackables(CancellationToken ct) + { + var list = await _service.GetAsync(ct); + if (list == null) return NotFound(); + return Ok(list); + } + + [HttpGet("{id}")] + public async Task GetTrackableHistory([FromRoute] Guid id, CancellationToken ct) + { + var list = await _service.GetActivityByTrackableId(id, ct); + if (list == null) return NotFound(); + return Ok(list); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Services/TimelineService.cs b/src/Ghosts.Api/Services/TimelineService.cs index 4bb6bb8a..b725c946 100644 --- a/src/Ghosts.Api/Services/TimelineService.cs +++ b/src/Ghosts.Api/Services/TimelineService.cs @@ -1,11 +1,15 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; using Ghosts.Api.Infrastructure.Data; +using Ghosts.Api.Models; using Ghosts.Api.ViewModels; +using Ghosts.Domain; using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; namespace Ghosts.Api.Services { @@ -13,6 +17,7 @@ public interface ITimelineService { Task UpdateAsync(MachineUpdateViewModel machineUpdate, CancellationToken ct); Task UpdateGroupAsync(int groupId, MachineUpdateViewModel machineUpdate, CancellationToken ct); + Task StopAsync(Guid machineId, Guid timelineId, CancellationToken ct); } public class TimelineService : ITimelineService @@ -49,5 +54,27 @@ public async Task UpdateGroupAsync(int groupId, MachineUpdateViewModel machineUp await _context.SaveChangesAsync(ct); } + + public async Task StopAsync(Guid machineId, Guid timelineId, CancellationToken ct) + { + var timeline = new Timeline + { + Id = timelineId, + Status = Timeline.TimelineStatus.Stop + }; + + var o = new MachineUpdate + { + Status = StatusType.Active, + Update = JsonConvert.SerializeObject(timeline), + ActiveUtc = DateTime.UtcNow, + CreatedUtc = DateTime.UtcNow, + MachineId = machineId, + Type = UpdateClientConfig.UpdateType.Timeline + }; + + await _context.MachineUpdates.AddAsync(o, ct); + await _context.SaveChangesAsync(ct); + } } } \ No newline at end of file diff --git a/src/Ghosts.Api/Services/TrackableService.cs b/src/Ghosts.Api/Services/TrackableService.cs new file mode 100755 index 00000000..6a3287ae --- /dev/null +++ b/src/Ghosts.Api/Services/TrackableService.cs @@ -0,0 +1,42 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Infrastructure.Data; +using Ghosts.Api.Models; +using Microsoft.EntityFrameworkCore; +using NLog; + +namespace Ghosts.Api.Services +{ + public interface ITrackableService + { + Task> GetAsync(CancellationToken ct); + Task> GetActivityByTrackableId(Guid trackableId, CancellationToken ct); + } + + public class TrackableService : ITrackableService + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private readonly ApplicationDbContext _context; + private readonly int _lookBack = Program.ClientConfig.LookbackRecords; + + public TrackableService(ApplicationDbContext context) + { + _context = context; + } + + public async Task> GetAsync(CancellationToken ct) + { + return await _context.Trackables.ToListAsync(ct); + } + + public async Task> GetActivityByTrackableId(Guid trackableId, CancellationToken ct) + { + return await _context.HistoryTrackables.Where(o => o.TrackableId == trackableId).ToListAsync(ct); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Startup.cs b/src/Ghosts.Api/Startup.cs index 34caeca0..c44b4f3d 100755 --- a/src/Ghosts.Api/Startup.cs +++ b/src/Ghosts.Api/Startup.cs @@ -68,6 +68,7 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ghosts.client.linux/TimelineManager/TimelineBuilder.cs b/src/Ghosts.Domain/Code/TimelineBuilder.cs similarity index 60% rename from src/ghosts.client.linux/TimelineManager/TimelineBuilder.cs rename to src/Ghosts.Domain/Code/TimelineBuilder.cs index f65d21e1..2d77a789 100644 --- a/src/ghosts.client.linux/TimelineManager/TimelineBuilder.cs +++ b/src/Ghosts.Domain/Code/TimelineBuilder.cs @@ -1,12 +1,11 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; using System.IO; -using Ghosts.Domain; -using Ghosts.Domain.Code; using Newtonsoft.Json; using NLog; -namespace ghosts.client.linux.timelineManager +namespace Ghosts.Domain.Code { /// /// Helper class that loads timeline and watches it for future changes @@ -14,7 +13,6 @@ namespace ghosts.client.linux.timelineManager public class TimelineBuilder { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public static string TimelineFile = ApplicationDetails.ConfigurationFiles.Timeline; public static FileInfo TimelineFilePath() @@ -32,12 +30,31 @@ public static Timeline GetLocalTimeline() var raw = File.ReadAllText(TimelineFile); var timeline = JsonConvert.DeserializeObject(raw); + if (timeline.Id == Guid.Empty) + { + timeline.Id = Guid.NewGuid(); + SetLocalTimeline(TimelineFile, timeline); + } - _log.Debug("Timeline config loaded successfully"); + _log.Debug($"Timeline {timeline.Id} loaded successfully"); return timeline; } + public static Timeline GetLocalTimeline(string path) + { + var raw = File.ReadAllText(path); + + var timeline = JsonConvert.DeserializeObject(raw); + if (timeline.Id == Guid.Empty) + { + timeline.Id = Guid.NewGuid(); + SetLocalTimeline(path, timeline); + } + + return timeline; + } + /// /// Save to local disk /// @@ -56,8 +73,22 @@ public static void SetLocalTimeline(Timeline timeline) { using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Timeline)) { - var serializer = new JsonSerializer(); - serializer.Formatting = Formatting.Indented; + var serializer = new JsonSerializer + { + Formatting = Formatting.Indented + }; + serializer.Serialize(file, timeline); + } + } + + public static void SetLocalTimeline(string path, Timeline timeline) + { + using (var file = File.CreateText(path)) + { + var serializer = new JsonSerializer + { + Formatting = Formatting.Indented + }; serializer.Serialize(file, timeline); } } diff --git a/src/Ghosts.Domain/Messages/Timeline.cs b/src/Ghosts.Domain/Messages/Timeline.cs index 7755291b..884f4ae6 100755 --- a/src/Ghosts.Domain/Messages/Timeline.cs +++ b/src/Ghosts.Domain/Messages/Timeline.cs @@ -8,10 +8,15 @@ namespace Ghosts.Domain { /// - /// an array of events that a client should perform in order to best mimic some persona x + /// an array of events that a client should perform in order to best mimic some persona x /// public class Timeline { + /// + /// Useful for tracking where activity on a client originated + /// + public Guid Id { get; set; } + [JsonConverter(typeof(StringEnumConverter))] public enum TimelineStatus { @@ -25,7 +30,7 @@ public Timeline() } /// - /// Run or Stop + /// Run or Stop /// [JsonConverter(typeof(StringEnumConverter))] public TimelineStatus Status { get; set; } @@ -34,7 +39,7 @@ public Timeline() } /// - /// an array of application events that a client will execute - aka "randomly browse 6 different web pages for new shoes at 0900" + /// an array of application events that a client will execute - aka "randomly browse 6 different web pages for new shoes at 0900" /// public class TimelineHandler { @@ -48,7 +53,7 @@ public TimelineHandler() public HandlerType HandlerType { get; set; } /// - /// Used to instantiate browser object + /// Used to instantiate browser object /// public string Initial { get; set; } @@ -64,7 +69,7 @@ public TimelineHandler() } /// - /// handlers map to applications + /// handlers map to applications /// public enum HandlerType { @@ -90,7 +95,7 @@ public enum HandlerType } /// - /// The specific events that a handler will execute + /// The specific events that a handler will execute /// public class TimelineEvent { @@ -100,7 +105,7 @@ public TimelineEvent() } /// - /// AlertIds trace back to an alert that monitors specific activity executed within a timeline + /// AlertIds trace back to an alert that monitors specific activity executed within a timeline /// public string TrackableId { get; set; } @@ -108,18 +113,18 @@ public TimelineEvent() public List CommandArgs { get; set; } /// - /// Milliseconds + /// In milliseconds /// public int DelayAfter { get; set; } /// - /// Milliseconds + /// In milliseconds /// public int DelayBefore { get; set; } } /// - /// Gets passed back to api server 'Chrome, Browse, http://cnn.com' + /// Gets passed back to api server 'Chrome, Browse, https://cmu.edu' /// public class TimeLineRecord { diff --git a/src/ghosts.client.linux/Comms/Updates.cs b/src/ghosts.client.linux/Comms/Updates.cs index 458bdfd3..8cdc0ac9 100644 --- a/src/ghosts.client.linux/Comms/Updates.cs +++ b/src/ghosts.client.linux/Comms/Updates.cs @@ -105,7 +105,7 @@ private static void GetServerUpdates() } var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index 81c33321..9984e93f 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -1,11 +1,9 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; -using ghosts.client.linux.handlers; using Ghosts.Domain; using Ghosts.Domain.Code; using Newtonsoft.Json; @@ -120,10 +118,7 @@ private void OnChanged(object source, FileSystemEventArgs e) { try { - var raw = File.ReadAllText(e.FullPath); - - var timeline = JsonConvert.DeserializeObject(raw); - + var timeline = TimelineBuilder.GetLocalTimeline(e.FullPath); foreach (var timelineHandler in timeline.TimeLineHandlers) { _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); @@ -137,7 +132,7 @@ private void OnChanged(object source, FileSystemEventArgs e) } var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) @@ -154,7 +149,14 @@ private void OnChanged(object source, FileSystemEventArgs e) { var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); var orchestrator = new Orchestrator(); - orchestrator.RunCommand(constructedTimelineHandler); + + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(constructedTimelineHandler); + orchestrator.RunCommand(t, constructedTimelineHandler); } } catch (Exception exc) @@ -199,12 +201,6 @@ public Listener() message.ReplyLine($"{obj}{Environment.NewLine}"); Console.WriteLine(obj); }; - -// while (true) -// { -// Console.WriteLine("..."); -// Thread.Sleep(10000); -// } } catch (Exception e) { @@ -225,14 +221,23 @@ private string Handle(Message message) { var timelineHandler = JsonConvert.DeserializeObject(command); - foreach (var evs in timelineHandler.TimeLineEvents) - if (string.IsNullOrEmpty(evs.TrackableId)) - evs.TrackableId = Guid.NewGuid().ToString(); + foreach (var evs in timelineHandler.TimeLineEvents.Where(evs => string.IsNullOrEmpty(evs.TrackableId))) + { + evs.TrackableId = Guid.NewGuid().ToString(); + } _log.Trace($"Command found: {timelineHandler.HandlerType}"); var o = new Orchestrator(); - o.RunCommand(timelineHandler); + + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(timelineHandler); + + o.RunCommand(t, timelineHandler); var obj = JsonConvert.SerializeObject(timelineHandler); diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index 01a12e6d..02dc6519 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -6,6 +6,7 @@ using System.Threading; using ghosts.client.linux.handlers; using Ghosts.Domain; +using Ghosts.Domain.Code; using NLog; namespace ghosts.client.linux.timelineManager @@ -33,7 +34,7 @@ public void Run() _log.Trace($"watching {timelineWatcher.Path}"); timelineWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size | NotifyFilters.CreationTime | NotifyFilters.LastWrite; timelineWatcher.EnableRaisingEvents = true; - timelineWatcher.Changed += new FileSystemEventHandler(OnChanged); + timelineWatcher.Changed += OnChanged; this._threadJobs = new List(); @@ -75,18 +76,14 @@ private void RunEx(Timeline timeline) foreach (var handler in timeline.TimeLineHandlers) { - ThreadLaunch(handler); + ThreadLaunch(timeline, handler); } - - this.MonitorThread = new Thread(this.ThreadMonitor); - this.MonitorThread.IsBackground = true; - this.MonitorThread.Start(); } - public void RunCommand(TimelineHandler handler) + public void RunCommand(Timeline timeline, TimelineHandler handler) { this.WhatsInstalled(); - ThreadLaunch(handler); + ThreadLaunch(timeline, handler); } private void WhatsInstalled() @@ -94,17 +91,21 @@ private void WhatsInstalled() //TODO: check that used applications exist } - private void ThreadLaunch(TimelineHandler handler) + private void ThreadLaunch(Timeline timeline, TimelineHandler handler) { try { _log.Trace($"Attempting new thread for: {handler.HandlerType}"); Thread t = null; - var threadJob = new ThreadJob(); - threadJob.Id = Guid.NewGuid().ToString(); - threadJob.Handler = handler; + var threadJob = new ThreadJob + { + Id = Guid.NewGuid().ToString(), + Handler = handler, + TimelineId = timeline.Id + }; + object o; switch (handler.HandlerType) { case HandlerType.NpcSystem: @@ -113,51 +114,47 @@ private void ThreadLaunch(TimelineHandler handler) case HandlerType.Command: t = new Thread(() => { - var bash = new Bash(handler); - - }); - t.IsBackground = true; - t.Name = threadJob.Id; + o = new Bash(handler); + }) + { + IsBackground = true, + Name = threadJob.Id + }; t.Start(); - - //threadJob.ProcessName = ProcessManager.ProcessNames.Command; break; case HandlerType.Curl: t = new Thread(() => { - var curl = new Curl(handler); - - }); - t.IsBackground = true; - t.Name = threadJob.Id; + o = new Curl(handler); + }) + { + IsBackground = true, + Name = threadJob.Id + }; t.Start(); - - //threadJob.ProcessName = ProcessManager.ProcessNames.Command; break; case HandlerType.BrowserFirefox: - BrowserFirefox o = new BrowserFirefox(handler); - // t = new Thread(() => - // { - // BrowserFirefox o = new BrowserFirefox(handler); - // }) - // { - // IsBackground = true, - // Name = threadJob.Id - // }; - // t.Start(); - + t = new Thread(() => + { + o = new BrowserFirefox(handler); + }) + { + IsBackground = true, + Name = threadJob.Id + }; + t.Start(); break; } - if (threadJob.ProcessName != null) - { - this._threadJobs.Add(threadJob); - } - if (t != null) { this._threads.Add(t); } + + if (threadJob.ProcessName != null) + { + this._threadJobs.Add(threadJob); + } } catch (Exception e) { @@ -165,42 +162,35 @@ private void ThreadLaunch(TimelineHandler handler) } } - private void ThreadMonitor() - { - //this should be the original list only - var jobs = this._threadJobs.ToArray(); - while (true) - { - Thread.Sleep(30000); - //first, get all jobs and if not running, run a new one - foreach (var job in jobs) - { - //TODO - } - } - } - private void OnChanged(object source, FileSystemEventArgs e) { // filewatcher throws two events, we only need 1 - DateTime lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime > _lastRead.AddSeconds(1)) + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime <= _lastRead.AddSeconds(1)) return; + + _lastRead = lastWriteTime; + _log.Trace("File: " + e.FullPath + " " + e.ChangeType); + + var method = string.Empty; + if (System.Reflection.MethodBase.GetCurrentMethod() != null) { - _lastRead = lastWriteTime; - _log.Trace("File: " + e.FullPath + " " + e.ChangeType); - _log.Trace($"Reloading {System.Reflection.MethodBase.GetCurrentMethod().DeclaringType}"); - - // now terminate existing tasks and rerun - this.Shutdown(); - //StartupTasks.CleanupProcesses(); - this.Run(); + var declaringType = System.Reflection.MethodBase.GetCurrentMethod()?.DeclaringType; + if (declaringType != null) + method = declaringType?.ToString(); } + _log.Trace($"Reloading {method}..."); + + // now terminate existing tasks and rerun + this.Shutdown(); + //StartupTasks.CleanupProcesses(); + this.Run(); } } public class ThreadJob { public string Id { get; set; } + public Guid TimelineId { get; set; } public TimelineHandler Handler { get; set; } public string ProcessName { get; set; } } From 2a3dd0e030c0e660a8484211cfb076ddd0236bab Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Tue, 19 Oct 2021 14:16:02 -0400 Subject: [PATCH 06/42] Enables the stoppage of a single timeline vs the default agent timeline --- src/Ghosts.Domain/Models/ThreadJob.cs | 13 ++ .../Handlers/BrowserFirefox.cs | 2 +- src/ghosts.client.linux/Handlers/NpcSystem.cs | 54 +++++++ src/ghosts.client.linux/Program.cs | 12 +- .../TimelineManager/Listener.cs | 81 ++++++++-- .../TimelineManager/Orchestrator.cs | 146 +++++++----------- 6 files changed, 200 insertions(+), 108 deletions(-) create mode 100644 src/Ghosts.Domain/Models/ThreadJob.cs create mode 100644 src/ghosts.client.linux/Handlers/NpcSystem.cs diff --git a/src/Ghosts.Domain/Models/ThreadJob.cs b/src/Ghosts.Domain/Models/ThreadJob.cs new file mode 100644 index 00000000..18d27eb4 --- /dev/null +++ b/src/Ghosts.Domain/Models/ThreadJob.cs @@ -0,0 +1,13 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Threading; + +namespace Ghosts.Domain.Models +{ + public class ThreadJob + { + public Guid TimelineId { get; set; } + public Thread Thread { get; set; } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index b2d6c266..f333d43d 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -45,7 +45,7 @@ private bool FirefoxEx(TimelineHandler handler) { var path = GetInstallLocation(); - FirefoxOptions options = new FirefoxOptions(); + var options = new FirefoxOptions(); options.AddArguments("--disable-infobars"); options.AddArguments("--disable-extensions"); options.AddArguments("--disable-notifications"); diff --git a/src/ghosts.client.linux/Handlers/NpcSystem.cs b/src/ghosts.client.linux/Handlers/NpcSystem.cs new file mode 100644 index 00000000..75f4aff5 --- /dev/null +++ b/src/ghosts.client.linux/Handlers/NpcSystem.cs @@ -0,0 +1,54 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using ghosts.client.linux.Infrastructure; +using ghosts.client.linux.timelineManager; +using Ghosts.Domain; +using Ghosts.Domain.Code; +using NLog; + +namespace ghosts.client.linux.handlers +{ + public class NpcSystem : BaseHandler + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public NpcSystem(Timeline timeline, TimelineHandler handler) + { + _log.Trace($"Handling NpcSystem call: {handler}"); + + foreach (var timelineEvent in handler.TimeLineEvents) + { + if (string.IsNullOrEmpty(timelineEvent.Command)) + continue; + + Timeline t; + + switch (timelineEvent.Command.ToLower()) + { + case "start": + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Run; + TimelineBuilder.SetLocalTimeline(t); + break; + case "stop": + if (timeline.Id != Guid.Empty) + { + var o = new Orchestrator(); + o.StopTimeline(timeline.Id); + } + else + { + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Stop; + StartupTasks.CleanupProcesses(); + TimelineBuilder.SetLocalTimeline(t); + } + + break; + } + } + } + + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Program.cs b/src/ghosts.client.linux/Program.cs index 2865bc08..b67845b4 100755 --- a/src/ghosts.client.linux/Program.cs +++ b/src/ghosts.client.linux/Program.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.Drawing; using System.IO; using System.Net; @@ -9,21 +10,22 @@ using ghosts.client.linux.Infrastructure; using ghosts.client.linux.timelineManager; using Ghosts.Domain.Code; +using Ghosts.Domain.Models; using NLog; namespace ghosts.client.linux { class Program { - - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); internal static ClientConfiguration Configuration { get; set; } internal static Options OptionFlags; internal static bool IsDebug; - private static ListenerManager _listenerManager { get; set; } + internal static List ThreadJobs { get; set; } + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); static void Main(string[] args) { + ThreadJobs = new List(); ClientConfigurationLoader.UpdateConfigurationWithEnvVars(); try @@ -56,7 +58,7 @@ private static void Run(string[] args) //load configuration try { - Program.Configuration = ClientConfigurationLoader.Config; + Configuration = ClientConfigurationLoader.Config; } catch (Exception e) { @@ -72,7 +74,7 @@ private static void Run(string[] args) StartupTasks.SetStartup(); - _listenerManager = new ListenerManager(); + ListenerManager.Run(); //check id _log.Trace(Comms.CheckId.Id); diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index 9984e93f..2faf9a49 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -9,16 +9,17 @@ using Newtonsoft.Json; using NLog; using SimpleTCP; +// ReSharper disable ObjectCreationAsStatement namespace ghosts.client.linux.timelineManager { - public class ListenerManager + public static class ListenerManager { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private string In = ApplicationDetails.InstanceDirectories.TimelineIn; - private string Out = ApplicationDetails.InstanceDirectories.TimelineOut; + private static readonly string In = ApplicationDetails.InstanceDirectories.TimelineIn; + private static readonly string Out = ApplicationDetails.InstanceDirectories.TimelineOut; - public ListenerManager() + public static void Run() { try { @@ -60,6 +61,13 @@ public ListenerManager() }; t.Start(); + t = new Thread(() => { new InitialDirectoryListener(); }) + { + IsBackground = true, + Name = "ghosts-initialdirectorylistener" + }; + t.Start(); + EnsureWatch(); } else @@ -73,22 +81,71 @@ public ListenerManager() } } - private void EnsureWatch() + private static void EnsureWatch() { File.WriteAllText(In + "init.json", JsonConvert.SerializeObject(new Timeline(), Formatting.None)); } } + public class InitialDirectoryListener + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private static DateTime _lastRead = DateTime.Now; + public InitialDirectoryListener() + { + var directoryName = TimelineBuilder.TimelineFilePath().DirectoryName; + if (directoryName == null) + { + throw new Exception("Timeline builder path cannot be determined"); + } + + var timelineWatcher = new FileSystemWatcher(directoryName); + timelineWatcher.Filter = Path.GetFileName(TimelineBuilder.TimelineFilePath().Name); + _log.Trace($"watching {timelineWatcher.Path}"); + timelineWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size | NotifyFilters.CreationTime | + NotifyFilters.LastWrite; + timelineWatcher.Changed += InitialOnChanged; + timelineWatcher.EnableRaisingEvents = true; + + + new ManualResetEvent(false).WaitOne(); + } + + private void InitialOnChanged(object source, FileSystemEventArgs e) + { + // file watcher throws two events, we only need 1 + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime <= _lastRead.AddSeconds(1)) return; + + _lastRead = lastWriteTime; + _log.Trace("File: " + e.FullPath + " " + e.ChangeType); + + var method = string.Empty; + if (System.Reflection.MethodBase.GetCurrentMethod() != null) + { + var declaringType = System.Reflection.MethodBase.GetCurrentMethod()?.DeclaringType; + if (declaringType != null) + method = declaringType.ToString(); + } + _log.Trace($"Reloading {method}..."); + + // now terminate existing tasks and rerun + var o = new Orchestrator(); + o.Stop(); + o.Run(); + } + } + /// - /// Watches a directory [ghosts install]\instance\timeline for dropped files, and processes them immediately + /// Watches a directory [ghosts install] \ instance \ timeline for dropped files, and processes them immediately /// public class DirectoryListener { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private string _in = ApplicationDetails.InstanceDirectories.TimelineIn; - private string _out = ApplicationDetails.InstanceDirectories.TimelineOut; + private readonly string _in = ApplicationDetails.InstanceDirectories.TimelineIn; + private readonly string _out = ApplicationDetails.InstanceDirectories.TimelineOut; private string _currentlyProcessing = string.Empty; - + public DirectoryListener() { var watcher = new FileSystemWatcher @@ -100,9 +157,9 @@ public DirectoryListener() watcher.Changed += OnChanged; watcher.Created += OnChanged; watcher.EnableRaisingEvents = true; - Console.ReadLine(); - } - + new ManualResetEvent(false).WaitOne(); + } + private void OnChanged(object source, FileSystemEventArgs e) { // filewatcher throws multiple events, we only need 1 diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index 02dc6519..705cb85e 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -1,12 +1,12 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.Collections.Generic; -using System.IO; +using System.Linq; using System.Threading; using ghosts.client.linux.handlers; using Ghosts.Domain; using Ghosts.Domain.Code; +using Ghosts.Domain.Models; using NLog; namespace ghosts.client.linux.timelineManager @@ -18,8 +18,6 @@ public class Orchestrator { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private static DateTime _lastRead = DateTime.MinValue; - private List _threads { get; set; } - private List _threadJobs { get; set; } private Thread MonitorThread { get; set; } public void Run() @@ -28,16 +26,6 @@ public void Run() { var timeline = TimelineBuilder.GetLocalTimeline(); - // now watch that file for changes - var timelineWatcher = new FileSystemWatcher(TimelineBuilder.TimelineFilePath().DirectoryName); - timelineWatcher.Filter = Path.GetFileName(TimelineBuilder.TimelineFilePath().Name); - _log.Trace($"watching {timelineWatcher.Path}"); - timelineWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size | NotifyFilters.CreationTime | NotifyFilters.LastWrite; - timelineWatcher.EnableRaisingEvents = true; - timelineWatcher.Changed += OnChanged; - - this._threadJobs = new List(); - //load into an managing object //which passes the timeline commands to handlers //and creates a thread to execute instructions over that timeline @@ -60,20 +48,56 @@ public void Run() } } - public void Shutdown() + public void StopTimeline(Guid timelineId) { - foreach (var thread in this._threads) + foreach (var threadJob in Program.ThreadJobs.Where(x=>x.TimelineId == timelineId)) { - thread.Abort(null); + try + { + threadJob.Thread.Abort(null); + } + catch (Exception e) + { + _log.Debug(e); + } + + try + { + threadJob.Thread.Join(); + } + catch (Exception e) + { + _log.Debug(e); + } } } - private void RunEx(Timeline timeline) + public void Stop() { - this._threads = new List(); - - this.WhatsInstalled(); + foreach (var threadJob in Program.ThreadJobs) + { + try + { + threadJob.Thread.Abort(null); + } + catch (Exception e) + { + _log.Debug(e); + } + + try + { + threadJob.Thread.Join(); + } + catch (Exception e) + { + _log.Debug(e); + } + } + } + private void RunEx(Timeline timeline) + { foreach (var handler in timeline.TimeLineHandlers) { ThreadLaunch(timeline, handler); @@ -82,15 +106,9 @@ private void RunEx(Timeline timeline) public void RunCommand(Timeline timeline, TimelineHandler handler) { - this.WhatsInstalled(); ThreadLaunch(timeline, handler); } - private void WhatsInstalled() - { - //TODO: check that used applications exist - } - private void ThreadLaunch(Timeline timeline, TimelineHandler handler) { try @@ -100,8 +118,6 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) Thread t = null; var threadJob = new ThreadJob { - Id = Guid.NewGuid().ToString(), - Handler = handler, TimelineId = timeline.Id }; @@ -109,89 +125,39 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) switch (handler.HandlerType) { case HandlerType.NpcSystem: - //var npc = new NpcSystem(handler); - //break; + var npc = new NpcSystem(timeline, handler); + break; case HandlerType.Command: t = new Thread(() => { o = new Bash(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + }); break; case HandlerType.Curl: t = new Thread(() => { o = new Curl(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + }); break; case HandlerType.BrowserFirefox: t = new Thread(() => { o = new BrowserFirefox(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + }); break; } - if (t != null) - { - this._threads.Add(t); - } + if (t == null) return; - if (threadJob.ProcessName != null) - { - this._threadJobs.Add(threadJob); - } + t.IsBackground = true; + t.Start(); + threadJob.Thread = t; + Program.ThreadJobs.Add(threadJob); } catch (Exception e) { _log.Error(e); } } - - private void OnChanged(object source, FileSystemEventArgs e) - { - // filewatcher throws two events, we only need 1 - var lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime <= _lastRead.AddSeconds(1)) return; - - _lastRead = lastWriteTime; - _log.Trace("File: " + e.FullPath + " " + e.ChangeType); - - var method = string.Empty; - if (System.Reflection.MethodBase.GetCurrentMethod() != null) - { - var declaringType = System.Reflection.MethodBase.GetCurrentMethod()?.DeclaringType; - if (declaringType != null) - method = declaringType?.ToString(); - } - _log.Trace($"Reloading {method}..."); - - // now terminate existing tasks and rerun - this.Shutdown(); - //StartupTasks.CleanupProcesses(); - this.Run(); - } - } - - public class ThreadJob - { - public string Id { get; set; } - public Guid TimelineId { get; set; } - public TimelineHandler Handler { get; set; } - public string ProcessName { get; set; } } -} + } From 7bf481f12eb7d2c5d147d4a69a0bc0692f64f1f9 Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Tue, 19 Oct 2021 17:14:01 -0400 Subject: [PATCH 07/42] adds timeline specific shutdown for windows client --- src/Ghosts.Client/Comms/Updates.cs | 2 +- src/Ghosts.Client/Ghosts.Client.csproj | 1 - src/Ghosts.Client/Handlers/NpcSystem.cs | 30 ++- src/Ghosts.Client/Program.cs | 7 +- src/Ghosts.Client/TimelineManager/Listener.cs | 35 ++- .../TimelineManager/Orchestrator.cs | 255 +++++++----------- .../TimelineManager/TimelineBuilder.cs | 77 ------ src/Ghosts.Client/config/application.json | 189 ++++++------- 8 files changed, 235 insertions(+), 361 deletions(-) delete mode 100755 src/Ghosts.Client/TimelineManager/TimelineBuilder.cs diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index 964de6a0..b905ec12 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -112,7 +112,7 @@ private static void GetServerUpdates() } var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) diff --git a/src/Ghosts.Client/Ghosts.Client.csproj b/src/Ghosts.Client/Ghosts.Client.csproj index c66856d5..c51a99a2 100755 --- a/src/Ghosts.Client/Ghosts.Client.csproj +++ b/src/Ghosts.Client/Ghosts.Client.csproj @@ -611,7 +611,6 @@ - diff --git a/src/Ghosts.Client/Handlers/NpcSystem.cs b/src/Ghosts.Client/Handlers/NpcSystem.cs index ce0c7b2b..6f6e05b1 100755 --- a/src/Ghosts.Client/Handlers/NpcSystem.cs +++ b/src/Ghosts.Client/Handlers/NpcSystem.cs @@ -1,8 +1,10 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; using Ghosts.Client.Infrastructure; using Ghosts.Client.TimelineManager; using Ghosts.Domain; +using Ghosts.Domain.Code; using NLog; namespace Ghosts.Client.Handlers @@ -11,7 +13,7 @@ public class NpcSystem : BaseHandler { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public NpcSystem(TimelineHandler handler) + public NpcSystem(Timeline timeline, TimelineHandler handler) { _log.Trace($"Handling NpcSystem call: {handler}"); @@ -20,26 +22,32 @@ public NpcSystem(TimelineHandler handler) if (string.IsNullOrEmpty(timelineEvent.Command)) continue; - Timeline timeline; + Timeline t; switch (timelineEvent.Command.ToLower()) { case "start": - timeline = TimelineBuilder.GetLocalTimeline(); - timeline.Status = Timeline.TimelineStatus.Run; - TimelineBuilder.SetLocalTimeline(timeline); + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Run; + TimelineBuilder.SetLocalTimeline(t); break; case "stop": - timeline = TimelineBuilder.GetLocalTimeline(); - timeline.Status = Timeline.TimelineStatus.Stop; + if (timeline.Id != Guid.Empty) + { + var o = new Orchestrator(); + o.StopTimeline(timeline.Id); + } + else + { + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Stop; + StartupTasks.CleanupProcesses(); + TimelineBuilder.SetLocalTimeline(t); + } - StartupTasks.CleanupProcesses(); - - TimelineBuilder.SetLocalTimeline(timeline); break; } } } - } } \ No newline at end of file diff --git a/src/Ghosts.Client/Program.cs b/src/Ghosts.Client/Program.cs index 97a72110..5e7cf472 100755 --- a/src/Ghosts.Client/Program.cs +++ b/src/Ghosts.Client/Program.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Net; @@ -11,6 +12,7 @@ using Ghosts.Client.Infrastructure; using Ghosts.Client.TimelineManager; using Ghosts.Domain.Code; +using Ghosts.Domain.Models; using NLog; namespace Ghosts.Client @@ -28,6 +30,7 @@ class Program private const int SwHide = 0; private const int SwShow = 5; + internal static List ThreadJobs { get; set; } internal static ClientConfiguration Configuration { get; set; } internal static Options OptionFlags; internal static bool IsDebug; @@ -78,7 +81,7 @@ static void Main(string[] args) { MinimizeFootprint(); minimizeMemory(); - + try { Run(args); @@ -98,6 +101,8 @@ static void Main(string[] args) private static void Run(string[] args) { + ThreadJobs = new List(); + // ignore all certs ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; diff --git a/src/Ghosts.Client/TimelineManager/Listener.cs b/src/Ghosts.Client/TimelineManager/Listener.cs index ead13a0a..26002d83 100755 --- a/src/Ghosts.Client/TimelineManager/Listener.cs +++ b/src/Ghosts.Client/TimelineManager/Listener.cs @@ -126,7 +126,7 @@ private void OnChanged(object source, FileSystemEventArgs e) } var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) @@ -142,8 +142,14 @@ private void OnChanged(object source, FileSystemEventArgs e) if (commands.Count > 0) { var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(constructedTimelineHandler); + var orchestrator = new Orchestrator(); + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(constructedTimelineHandler); + orchestrator.RunCommand(t, constructedTimelineHandler); } } catch (Exception exc) @@ -198,13 +204,13 @@ public PortListener() private string Handle(Message message) { - string tempMsg = + var tempMsg = $"PortListener received raw {message.TcpClient.Client.RemoteEndPoint}: {message.MessageString}"; Console.WriteLine(tempMsg); _log.Trace(tempMsg); - string command = message.MessageString; - int index = command.LastIndexOf("}", StringComparison.InvariantCultureIgnoreCase); + var command = message.MessageString; + var index = command.LastIndexOf("}", StringComparison.InvariantCultureIgnoreCase); if (index > 0) { command = command.Substring(0, index + 1); @@ -214,9 +220,9 @@ private string Handle(Message message) try { - TimelineHandler timelineHandler = JsonConvert.DeserializeObject(command); + var timelineHandler = JsonConvert.DeserializeObject(command); - foreach (TimelineEvent evs in timelineHandler.TimeLineEvents) + foreach (var evs in timelineHandler.TimeLineEvents) { if (string.IsNullOrEmpty(evs.TrackableId)) { @@ -226,10 +232,17 @@ private string Handle(Message message) _log.Trace($"PortListener command found: {timelineHandler.HandlerType}"); - Orchestrator o = new Orchestrator(); - o.RunCommand(timelineHandler); + var o = new Orchestrator(); + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(timelineHandler); + + o.RunCommand(t, timelineHandler); - string obj = JsonConvert.SerializeObject(timelineHandler); + var obj = JsonConvert.SerializeObject(timelineHandler); return obj; } diff --git a/src/Ghosts.Client/TimelineManager/Orchestrator.cs b/src/Ghosts.Client/TimelineManager/Orchestrator.cs index e0e0d95e..61a74424 100755 --- a/src/Ghosts.Client/TimelineManager/Orchestrator.cs +++ b/src/Ghosts.Client/TimelineManager/Orchestrator.cs @@ -6,12 +6,13 @@ using Microsoft.Win32; using NLog; using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Threading; using System.Security.Permissions; +using Ghosts.Domain.Code; +using Ghosts.Domain.Models; namespace Ghosts.Client.TimelineManager { @@ -22,8 +23,6 @@ public class Orchestrator { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private static DateTime _lastRead = DateTime.MinValue; - private List _threads { get; set; } - private List _threadJobs { get; set; } private Thread MonitorThread { get; set; } private Timeline _timeline; private FileSystemWatcher timelineWatcher; @@ -67,9 +66,7 @@ public void Run() timelineWatcher.EnableRaisingEvents = true; timelineWatcher.Changed += OnChanged; } - - _threadJobs = new List(); - + //load into an managing object //which passes the timeline commands to handlers //and creates a thread to execute instructions over that timeline @@ -92,34 +89,68 @@ public void Run() } } - public void Shutdown() + public void StopTimeline(Guid timelineId) { - try + foreach (var threadJob in Program.ThreadJobs.Where(x => x.TimelineId == timelineId)) { - foreach (var thread in _threads) + try + { + threadJob.Thread.Abort(null); + } + catch (Exception e) + { + _log.Debug(e); + } + + try { - thread.Abort(null); + threadJob.Thread.Join(); + } + catch (Exception e) + { + _log.Debug(e); } } - catch { } } - private void RunEx(Timeline timeline) + public void Stop() { - _threads = new List(); + foreach (var threadJob in Program.ThreadJobs) + { + try + { + threadJob.Thread.Abort(null); + } + catch (Exception e) + { + _log.Debug(e); + } + try + { + threadJob.Thread.Join(); + } + catch (Exception e) + { + _log.Debug(e); + } + } + } + + private void RunEx(Timeline timeline) + { WhatsInstalled(); - foreach (TimelineHandler handler in timeline.TimeLineHandlers) + foreach (var handler in timeline.TimeLineHandlers) { ThreadLaunch(timeline, handler); } } - public void RunCommand(TimelineHandler handler) + public void RunCommand(Timeline timeline, TimelineHandler handler) { WhatsInstalled(); - ThreadLaunch(null, handler); + ThreadLaunch(timeline, handler); } ///here lies technical debt @@ -194,7 +225,7 @@ private static void SafetyNet() private void WhatsInstalled() { - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("Outlook.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("Outlook.Application")) { if (regWord != null) { @@ -204,7 +235,7 @@ private void WhatsInstalled() _log.Trace($"Outlook is installed: {_isOutlookInstalled}"); } - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("Word.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("Word.Application")) { if (regWord != null) { @@ -214,7 +245,7 @@ private void WhatsInstalled() _log.Trace($"Word is installed: {_isWordInstalled}"); } - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("Excel.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("Excel.Application")) { if (regWord != null) { @@ -224,7 +255,7 @@ private void WhatsInstalled() _log.Trace($"Excel is installed: {_isExcelInstalled}"); } - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("PowerPoint.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("PowerPoint.Application")) { if (regWord != null) { @@ -237,57 +268,35 @@ private void WhatsInstalled() private void ThreadLaunch(Timeline timeline, TimelineHandler handler) { - try { _log.Trace($"Attempting new thread for: {handler.HandlerType}"); Thread t = null; - ThreadJob threadJob = new ThreadJob - { - Id = Guid.NewGuid().ToString(), - Handler = handler - }; - + object o; switch (handler.HandlerType) { case HandlerType.NpcSystem: - NpcSystem npc = new NpcSystem(handler); + var npc = new NpcSystem(timeline, handler); break; case HandlerType.Command: - t = new Thread(() => + t = new Thread(start: () => { - Cmd o = new Cmd(handler); - - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Command; - + o = new Cmd(handler); + }); break; case HandlerType.Word: _log.Trace("Launching thread for word"); if (_isWordInstalled) { var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.Word).ToList(); - if (pids.Count > timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.Word)) + if (pids.Count > timeline.TimeLineHandlers.Count(x => x.HandlerType == HandlerType.Word)) return; t = new Thread(() => { - WordHandler o = new WordHandler(timeline, handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Word; + o = new WordHandler(timeline, handler); + }); } break; case HandlerType.Excel: @@ -295,176 +304,100 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) if (_isExcelInstalled) { var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.Excel).ToList(); - if (pids.Count > timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.Excel)) + if (pids.Count > timeline.TimeLineHandlers.Count(x => x.HandlerType == HandlerType.Excel)) return; t = new Thread(() => { - ExcelHandler o = new ExcelHandler(timeline, handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Excel; + o = new ExcelHandler(timeline, handler); + }); } break; case HandlerType.Clicks: _log.Trace("Launching thread to handle clicks"); t = new Thread(() => { - Clicks o = new Clicks(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + o = new Clicks(handler); + }); break; case HandlerType.Reboot: _log.Trace("Launching thread to handle reboot"); t = new Thread(() => { - Reboot o = new Reboot(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + o = new Reboot(handler); + }); break; case HandlerType.PowerPoint: _log.Trace("Launching thread for powerpoint"); if (_isPowerPointInstalled) { var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.PowerPoint).ToList(); - if (pids.Count > timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.PowerPoint)) + if (pids.Count > timeline.TimeLineHandlers.Count(x => x.HandlerType == HandlerType.PowerPoint)) return; t = new Thread(() => { - PowerPointHandler o = new PowerPointHandler(timeline, handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.PowerPoint; + o = new PowerPointHandler(timeline, handler); + }); } break; case HandlerType.Outlook: _log.Trace("Launching thread for outlook - note we're not checking if outlook installed, just going for it"); - //if (this.IsOutlookInstalled) - //{ t = new Thread(() => { - Outlook o = new Outlook(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Outlook; - //} - + o = new Outlook(handler); + }); break; case HandlerType.BrowserIE: //IE demands COM apartmentstate be STA so diff thread creation required t = new Thread(() => { - BrowserIE o = new BrowserIE(handler); + o = new BrowserIE(handler); }); t.SetApartmentState(ApartmentState.STA); - t.IsBackground = true; - t.Name = threadJob.Id; - t.Start(); - break; case HandlerType.Notepad: //TODO t = new Thread(() => { - Notepad o = new Notepad(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - + o = new Notepad(handler); + }); break; - case HandlerType.BrowserChrome: t = new Thread(() => { - BrowserChrome o = new BrowserChrome(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Chrome; - + o = new BrowserChrome(handler); + }); break; case HandlerType.BrowserFirefox: t = new Thread(() => { - BrowserFirefox o = new BrowserFirefox(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Firefox; - + o = new BrowserFirefox(handler); + }); break; case HandlerType.Watcher: t = new Thread(() => { - Watcher o = new Watcher(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - //threadJob.ProcessName = ProcessManager.ProcessNames.Watcher; - + o = new Watcher(handler); + }); break; case HandlerType.Print: t = new Thread(() => { - var p = new Print(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - + o = new Print(handler); + }); break; } - if (threadJob.ProcessName != null) - { - _threadJobs.Add(threadJob); - } + if (t == null) return; - if (t != null) + t.IsBackground = true; + t.Start(); + Program.ThreadJobs.Add(new ThreadJob { - _threads.Add(t); - } + TimelineId = timeline.Id, + Thread = t + }); } catch (Exception e) { @@ -490,7 +423,7 @@ private void OnChanged(object source, FileSystemEventArgs e) try { - Shutdown(); + Stop(); } catch (Exception exception) { @@ -522,14 +455,6 @@ private void OnChanged(object source, FileSystemEventArgs e) { _log.Info(exc); } - } } - - public class ThreadJob - { - public string Id { get; set; } - public TimelineHandler Handler { get; set; } - public string ProcessName { get; set; } - } } diff --git a/src/Ghosts.Client/TimelineManager/TimelineBuilder.cs b/src/Ghosts.Client/TimelineManager/TimelineBuilder.cs deleted file mode 100755 index bb8c8251..00000000 --- a/src/Ghosts.Client/TimelineManager/TimelineBuilder.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - -using System; -using System.IO; -using Ghosts.Domain; -using Ghosts.Domain.Code; -using Newtonsoft.Json; -using NLog; - -namespace Ghosts.Client.TimelineManager -{ - /// - /// Helper class that loads timeline and watches it for future changes - /// - public class TimelineBuilder - { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - - public static string TimelineFile = ApplicationDetails.ConfigurationFiles.Timeline; - - public static FileInfo TimelineFilePath() - { - return new FileInfo(TimelineFile); - } - - /// - /// Get from local disk - /// - /// The local timeline to be executed - public static Timeline GetLocalTimeline() - { - Timeline timeline; - _log.Trace($"Loading timeline config {TimelineFile }"); - - try - { - var raw = File.ReadAllText(TimelineFile); - timeline = JsonConvert.DeserializeObject(raw); - } - catch (Exception e) - { - var err = $"ERROR: Could not deserialize timeline json file! {e.Message} {e.StackTrace}"; - Console.WriteLine(err); - _log.Error(err); - throw; - } - - _log.Trace("Timeline config loaded successfully"); - - return timeline; - } - - /// - /// Save to local disk - /// - /// Raw timeline string (to be converted to `Timeline` type) - public static void SetLocalTimeline(string timelineString) - { - var timelineObject = JsonConvert.DeserializeObject(timelineString); - SetLocalTimeline(timelineObject); - } - - /// - /// Save to local disk - /// - /// `Timeline` type - public static void SetLocalTimeline(Timeline timeline) - { - using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Timeline)) - { - var serializer = new JsonSerializer(); - serializer.Formatting = Formatting.Indented; - serializer.Serialize(file, timeline); - } - } - } -} diff --git a/src/Ghosts.Client/config/application.json b/src/Ghosts.Client/config/application.json index c8e107ac..0a619b4f 100755 --- a/src/Ghosts.Client/config/application.json +++ b/src/Ghosts.Client/config/application.json @@ -1,95 +1,96 @@ { - "IdEnabled": true, - "IdUrl": "http://ghosts-c2:52388/api/clientid", - "IdFormat": "guestlocal", - "IdFormatKey": "guestinfo.id", - "IdFormatValue": "$formatkeyvalue$-$machinename$", - "VMWareToolsLocation": "C:\\progra~1\\VMware\\VMware Tools\\vmtoolsd.exe", - "EncodeHeaders": true, - "ClientResults": { - "IsEnabled": true, - "IsSecure": false, - "PostUrl": "http://ghosts-c2:52388/api/clientresults", - "CycleSleep": 9000 - }, - "ClientUpdates": { - "IsEnabled": true, - "PostUrl": "http://ghosts-c2:52388/api/clientupdates", - "CycleSleep": 9000 - }, - "Survey": { - "IsEnabled": true, - "IsSecure": false, - "Frequency": "once", - "CycleSleepMinutes": 5, - "OutputFormat": "indent", - "PostUrl": "http://ghosts-c2:52388/api/clientsurvey" - }, - "Content": { - "EmailContent": "", - "EmailReply": "", - "EmailDomain": "", - "EmailOutside": "", - "FileNames": "", - "Dictionary": "" - }, - "HealthIsEnabled": true, - "HandlersIsEnabled": true, - "ChromeExtensions": "", - "FirefoxInstallLocation": "", - "FirefoxMajorVersionMinimum": 48, - "OfficeDocsMaxAgeInHours": 6, - "Email": { - "RecipientsToMin": 1, - "RecipientsToMax": 3, - "RecipientsCcMin": 0, - "RecipientsCcMax": 2, - "RecipientsBccMin": 2, - "RecipientsBccMax": 2, - "RecipientsOutsideMin": 0, - "RecipientsOutsideMax": 1, - "SetAccountFromConfig": false, - "SetAccountFromLocal": false, - "SetForcedSendReceive": true, - "SaveToOutbox": false, - "EmailDomainSearchString": "Get-ADUser -filter * -searchbase \"CN=USERS,DC=JRSS,DC=GOV\" -properties UserPrincipalName | select -expand UserPrincipalName" - }, - "Listener": { - "Port": -1 - }, - "EmailContent": { - "conflict_1_capital": "", - "conflict_1_name": "", - "conflict_1_peoples": "", - "conflict_1_president": "", - "localized_flashpoint_locale": "", - "friendly_nation_leader_lastname": "", - "friendly_nation_leader_name": "", - "friendly_nation_name": "", - "friendly_nation_peoples": "", - "commander_title": "", - "commander_name": "", - "commander_initials": "", - "commander_lastname": "", - "commander_email": "", - "commander_sub1": "", - "commander_sub2": "", - "us_president": "", - "iraq": "", - "iraqi": "", - "iran": "", - "iranian": "", - "china": "", - "chinese": "", - "russia": "", - "azerbaijan": "", - "turkish": "", - "turkey": "", - "pakistani": "", - "pakistan": "", - "palestinian": "", - "palestine": "", - "gaza": "", - "korea": "" - } -} + "IdEnabled": true, + "IdUrl": "http://ghosts-api:52388/api/clientid", + "IdFormat": "guestlocal", + "IdFormatKey": "guestinfo.id", + "IdFormatValue": "$formatkeyvalue$-$machinename$", + "VMWareToolsLocation": "C:\\progra~1\\VMware\\VMware Tools\\vmtoolsd.exe", + "EncodeHeaders": true, + "ClientResults": { + "IsEnabled": true, + "IsSecure": false, + "PostUrl": "http://ghosts-api:52388/api/clientresults", + "CycleSleep": 9000 + }, + "ClientUpdates": { + "IsEnabled": true, + "PostUrl": "http://ghosts-api:52388/api/clientupdates", + "CycleSleep": 9000 + }, + "Survey": { + "IsEnabled": true, + "IsSecure": false, + "Frequency": "once", + "CycleSleepMinutes": 5, + "OutputFormat": "indent", + "PostUrl": "http://ghosts-api:52388/api/clientsurvey" + }, + "Content": { + "EmailContent": "", + "EmailReply": "", + "EmailDomain": "", + "EmailOutside": "", + "FileNames": "", + "Dictionary": "" + }, + "HealthIsEnabled": true, + "HandlersIsEnabled": true, + "ChromeExtensions": "", + "FirefoxInstallLocation": "", + "FirefoxMajorVersionMinimum": 48, + "OfficeDocsMaxAgeInHours": 6, + "Email": { + "RecipientsToMin": 1, + "RecipientsToMax": 3, + "RecipientsCcMin": 0, + "RecipientsCcMax": 2, + "RecipientsBccMin": 2, + "RecipientsBccMax": 2, + "RecipientsOutsideMin": 0, + "RecipientsOutsideMax": 1, + "SetAccountFromConfig": false, + "SetAccountFromLocal": false, + "SetForcedSendReceive": true, + "SaveToOutbox": false, + "EmailDomainSearchString": + "Get-ADUser -filter * -searchbase \"CN=USERS,DC=JRSS,DC=GOV\" -properties UserPrincipalName | select -expand UserPrincipalName" + }, + "Listener": { + "Port": -1 + }, + "EmailContent": { + "conflict_1_capital": "", + "conflict_1_name": "", + "conflict_1_peoples": "", + "conflict_1_president": "", + "localized_flashpoint_locale": "", + "friendly_nation_leader_lastname": "", + "friendly_nation_leader_name": "", + "friendly_nation_name": "", + "friendly_nation_peoples": "", + "commander_title": "", + "commander_name": "", + "commander_initials": "", + "commander_lastname": "", + "commander_email": "", + "commander_sub1": "", + "commander_sub2": "", + "us_president": "", + "iraq": "", + "iraqi": "", + "iran": "", + "iranian": "", + "china": "", + "chinese": "", + "russia": "", + "azerbaijan": "", + "turkish": "", + "turkey": "", + "pakistani": "", + "pakistan": "", + "palestinian": "", + "palestine": "", + "gaza": "", + "korea": "" + } +} \ No newline at end of file From fe5ab1820afe111e565bf07e18742e59099ea9b3 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Tue, 19 Oct 2021 17:20:52 -0400 Subject: [PATCH 08/42] timeline specific shutdown for linux --- src/Ghosts.Api/Services/TimelineService.cs | 20 +++++++++++++++++-- .../TimelineManager/Orchestrator.cs | 12 +++++------ .../config/application.json | 8 ++++---- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/Ghosts.Api/Services/TimelineService.cs b/src/Ghosts.Api/Services/TimelineService.cs index b725c946..d8389ba8 100644 --- a/src/Ghosts.Api/Services/TimelineService.cs +++ b/src/Ghosts.Api/Services/TimelineService.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -57,10 +58,25 @@ public async Task UpdateGroupAsync(int groupId, MachineUpdateViewModel machineUp public async Task StopAsync(Guid machineId, Guid timelineId, CancellationToken ct) { + var timelineEvent = new TimelineEvent + { + Command = "stop" + }; + + var handler = new TimelineHandler + { + HandlerType = HandlerType.NpcSystem + }; + handler.TimeLineEvents.Add(timelineEvent); + + var handlers = new List(); + handlers.Add(handler); + var timeline = new Timeline { Id = timelineId, - Status = Timeline.TimelineStatus.Stop + Status = Timeline.TimelineStatus.Run, + TimeLineHandlers = handlers }; var o = new MachineUpdate @@ -70,7 +86,7 @@ public async Task StopAsync(Guid machineId, Guid timelineId, CancellationToken c ActiveUtc = DateTime.UtcNow, CreatedUtc = DateTime.UtcNow, MachineId = machineId, - Type = UpdateClientConfig.UpdateType.Timeline + Type = UpdateClientConfig.UpdateType.TimelinePartial }; await _context.MachineUpdates.AddAsync(o, ct); diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index 705cb85e..a5d3ca56 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -116,11 +116,6 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) _log.Trace($"Attempting new thread for: {handler.HandlerType}"); Thread t = null; - var threadJob = new ThreadJob - { - TimelineId = timeline.Id - }; - object o; switch (handler.HandlerType) { @@ -151,8 +146,11 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) t.IsBackground = true; t.Start(); - threadJob.Thread = t; - Program.ThreadJobs.Add(threadJob); + Program.ThreadJobs.Add(new ThreadJob + { + TimelineId = timeline.Id, + Thread = t + }); } catch (Exception e) { diff --git a/src/ghosts.client.linux/config/application.json b/src/ghosts.client.linux/config/application.json index a0aea0fc..a2070aa1 100755 --- a/src/ghosts.client.linux/config/application.json +++ b/src/ghosts.client.linux/config/application.json @@ -1,15 +1,15 @@ { "IdEnabled": true, - "IdUrl": "http://ghosts-api:5000/api/clientid", + "IdUrl": "http://ghosts-api:52388/api/clientid", "ClientResults": { "IsEnabled": true, "IsSecure": false, - "PostUrl": "http://ghosts-api:5000/api/clientresults", + "PostUrl": "http://ghosts-api:52388/api/clientresults", "CycleSleep": 9000 }, "ClientUpdates": { "IsEnabled": true, - "PostUrl": "http://ghosts-api:5000/api/clientupdates", + "PostUrl": "http://ghosts-api:52388/api/clientupdates", "CycleSleep": 9000 }, "Survey": { @@ -18,7 +18,7 @@ "Frequency": "once", "CycleSleepMinutes": 720, "OutputFormat": "indent", - "PostUrl": "http://ghosts-api:5000/api/clientsurvey" + "PostUrl": "http://ghosts-api:52388/api/clientsurvey" }, "HealthIsEnabled": true, "HandlersIsEnabled": true, From bad01008ea3490df1fe723b5d180a6b4867c3e16 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 20 Oct 2021 08:49:32 -0400 Subject: [PATCH 09/42] webhook cleanup --- src/Ghosts.Api/Models/WebHook.cs | 3 +-- src/Ghosts.Api/Services/QueueSyncService.cs | 13 ++++++++++++- src/Ghosts.Api/Startup.cs | 2 +- src/Ghosts.Api/ghosts.api.csproj | 8 ++++---- src/Ghosts.Domain/ghosts.domain.csproj | 4 ++-- src/ghosts.client.linux/Comms/Updates.cs | 6 +++++- src/ghosts.client.linux/ghosts.client.linux.csproj | 8 ++++---- 7 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/Ghosts.Api/Models/WebHook.cs b/src/Ghosts.Api/Models/WebHook.cs index b3d354d1..a1992f59 100755 --- a/src/Ghosts.Api/Models/WebHook.cs +++ b/src/Ghosts.Api/Models/WebHook.cs @@ -27,8 +27,7 @@ public Webhook() public Webhook(WebhookViewModel model) { - var id = Guid.NewGuid(); - if (Guid.TryParse(model.Id, out id)) + if (Guid.TryParse(model.Id, out var id)) Id = id; Status = model.Status; Description = model.Description; diff --git a/src/Ghosts.Api/Services/QueueSyncService.cs b/src/Ghosts.Api/Services/QueueSyncService.cs index 5ec854d1..48027f30 100755 --- a/src/Ghosts.Api/Services/QueueSyncService.cs +++ b/src/Ghosts.Api/Services/QueueSyncService.cs @@ -177,7 +177,18 @@ internal static async void HandleWebhook(Webhook webhook, NotificationQueueEntry using var httpClient = new HttpClient(); // Do the actual request and await the response - var httpResponse = await httpClient.PostAsync(webhook.PostbackUrl, httpContent); + HttpResponseMessage httpResponse; + switch (webhook.PostbackMethod) + { + default: + throw new ArgumentException("webhook configuration encountered unspecified postback method"); + case Webhook.WebhookMethod.POST: + httpResponse = await httpClient.PostAsync(webhook.PostbackUrl, httpContent); + break; + case Webhook.WebhookMethod.GET: + httpResponse = await httpClient.GetAsync($"{webhook.PostbackUrl}?message={formattedResponse}"); + break; + } log.Trace($"Webhook response {webhook.PostbackUrl} {webhook.PostbackMethod} {httpResponse.StatusCode}"); diff --git a/src/Ghosts.Api/Startup.cs b/src/Ghosts.Api/Startup.cs index c44b4f3d..71595e96 100755 --- a/src/Ghosts.Api/Startup.cs +++ b/src/Ghosts.Api/Startup.cs @@ -21,7 +21,7 @@ namespace Ghosts.Api { public class Startup { - public const int apiVersion = 5; + public const int apiVersion = 6; public Startup(IConfiguration configuration) { diff --git a/src/Ghosts.Api/ghosts.api.csproj b/src/Ghosts.Api/ghosts.api.csproj index e602b19f..8d53f45c 100644 --- a/src/Ghosts.Api/ghosts.api.csproj +++ b/src/Ghosts.Api/ghosts.api.csproj @@ -3,10 +3,10 @@ net5.0 ghosts.api - 5.0.0.0 - 5.0.0.0 - 5.0.0.0 - 5.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 false Dustin Updyke for Carnegie Mellon University diff --git a/src/Ghosts.Domain/ghosts.domain.csproj b/src/Ghosts.Domain/ghosts.domain.csproj index df540d24..7bea376d 100755 --- a/src/Ghosts.Domain/ghosts.domain.csproj +++ b/src/Ghosts.Domain/ghosts.domain.csproj @@ -3,8 +3,8 @@ netstandard2.0 Ghosts.Domain - 4.0.0.0 - 4.0.0.0 + 6.0.0.0 + 6.0.0.0 7.3 diff --git a/src/ghosts.client.linux/Comms/Updates.cs b/src/ghosts.client.linux/Comms/Updates.cs index 8cdc0ac9..c6514550 100644 --- a/src/ghosts.client.linux/Comms/Updates.cs +++ b/src/ghosts.client.linux/Comms/Updates.cs @@ -67,7 +67,11 @@ private static void GetServerUpdates() } catch (WebException wex) { - if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + if (wex?.Response == null) + { + _log.Debug($"{DateTime.Now} - API Server appears to be not responding"); + } + else if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) { _log.Debug($"{DateTime.Now} - No new configuration found"); } diff --git a/src/ghosts.client.linux/ghosts.client.linux.csproj b/src/ghosts.client.linux/ghosts.client.linux.csproj index ed61c89a..1f6028cf 100755 --- a/src/ghosts.client.linux/ghosts.client.linux.csproj +++ b/src/ghosts.client.linux/ghosts.client.linux.csproj @@ -5,10 +5,10 @@ netcoreapp3.1 win7-x64;ubuntu.16.10-x64;osx.10.12-x64 - 3.0.0.0 - 3.0.0.0 - 3.0.0.0 - 3.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 false Dustin Updyke for Carnegie Mellon University From 55e878393dd71c4a3ecbc60e1e916c9edea9f6dc Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Wed, 20 Oct 2021 08:52:53 -0400 Subject: [PATCH 10/42] webhook cleanup --- src/Ghosts.Client/Comms/Updates.cs | 6 +++++- src/Ghosts.Client/Properties/AssemblyInfo.cs | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index b905ec12..fcc31d95 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -71,7 +71,11 @@ private static void GetServerUpdates() } catch (WebException wex) { - if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + if (wex?.Response == null) + { + _log.Debug($"{DateTime.Now} - API Server appears to be not responding"); + } + else if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) { _log.Debug($"{DateTime.Now} - No new configuration found"); } diff --git a/src/Ghosts.Client/Properties/AssemblyInfo.cs b/src/Ghosts.Client/Properties/AssemblyInfo.cs index 340bd6c9..7d02c501 100755 --- a/src/Ghosts.Client/Properties/AssemblyInfo.cs +++ b/src/Ghosts.Client/Properties/AssemblyInfo.cs @@ -9,7 +9,7 @@ [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("CMU > SEI > CERT > MSE > Realism Team")] [assembly: AssemblyProduct("GHOSTS NPC Framework Client - please email ddupdyke@sei.cmu.edu with bugs/requests/other")] -[assembly: AssemblyCopyright("Copyright © 2017 - 2020")] +[assembly: AssemblyCopyright("Copyright © 2017 - 2021")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] @@ -31,5 +31,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("4.0.0.0")] -[assembly: AssemblyFileVersion("4.0.0.0")] +[assembly: AssemblyVersion("6.0.0.0")] +[assembly: AssemblyFileVersion("6.0.0.0")] From 1ca641598ef3a38ab1eec3437d85d6d90aa8606f Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 20 Oct 2021 10:32:50 -0400 Subject: [PATCH 11/42] creates webhook callback for successful timeline update deliveries --- .../Controllers/ClientUpdatesController.cs | 15 ++ .../Controllers/TimelineController.cs | 1 - src/Ghosts.Api/Models/QueueEntries.cs | 3 +- src/Ghosts.Api/Services/QueueSyncService.cs | 180 +++++++++--------- 4 files changed, 111 insertions(+), 88 deletions(-) diff --git a/src/Ghosts.Api/Controllers/ClientUpdatesController.cs b/src/Ghosts.Api/Controllers/ClientUpdatesController.cs index aa9a5a42..3d881b33 100755 --- a/src/Ghosts.Api/Controllers/ClientUpdatesController.cs +++ b/src/Ghosts.Api/Controllers/ClientUpdatesController.cs @@ -9,6 +9,7 @@ using Ghosts.Domain; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json.Linq; using NLog; namespace Ghosts.Api.Controllers @@ -86,6 +87,20 @@ public async Task Index(CancellationToken ct) await _updateService.DeleteAsync(u.Id, ct); + // integrators want to know that a timeline was actually delivered + // (the service only guarantees that the update was received) + _queue.Enqueue( + new QueueEntry + { + Payload = + new NotificationQueueEntry() + { + Type = NotificationQueueEntry.NotificationType.TimelineDelivered, + Payload = (JObject) JToken.FromObject(update) + }, + Type = QueueEntry.Types.Notification + }); + return Json(update); } } diff --git a/src/Ghosts.Api/Controllers/TimelineController.cs b/src/Ghosts.Api/Controllers/TimelineController.cs index ee6e8257..cf12a6c6 100644 --- a/src/Ghosts.Api/Controllers/TimelineController.cs +++ b/src/Ghosts.Api/Controllers/TimelineController.cs @@ -8,7 +8,6 @@ using Ghosts.Api.Services; using Ghosts.Api.ViewModels; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; using Swashbuckle.AspNetCore.Annotations; namespace ghosts.api.Controllers diff --git a/src/Ghosts.Api/Models/QueueEntries.cs b/src/Ghosts.Api/Models/QueueEntries.cs index 39a891f0..d7247ce5 100644 --- a/src/Ghosts.Api/Models/QueueEntries.cs +++ b/src/Ghosts.Api/Models/QueueEntries.cs @@ -30,7 +30,8 @@ public class NotificationQueueEntry public enum NotificationType { Timeline = 0, - WebhookCreate = 1 + WebhookCreate = 1, + TimelineDelivered = 10 } public NotificationType Type { get; set; } diff --git a/src/Ghosts.Api/Services/QueueSyncService.cs b/src/Ghosts.Api/Services/QueueSyncService.cs index 48027f30..845e7743 100755 --- a/src/Ghosts.Api/Services/QueueSyncService.cs +++ b/src/Ghosts.Api/Services/QueueSyncService.cs @@ -71,18 +71,18 @@ private async Task Sync() { using var scope = _scopeFactory.CreateScope(); await using var context = scope.ServiceProvider.GetRequiredService(); - + foreach (var item in Queue.GetAll()) switch (item.Type) { case QueueEntry.Types.Machine: - await ProcessMachine(scope, context, (MachineQueueEntry) item.Payload); + await ProcessMachine(scope, context, (MachineQueueEntry)item.Payload); break; case QueueEntry.Types.Notification: - await ProcessNotification(context, (NotificationQueueEntry) item.Payload); + await ProcessNotification(context, (NotificationQueueEntry)item.Payload); break; case QueueEntry.Types.Survey: - await ProcessSurvey(context, (Survey) item.Payload); + await ProcessSurvey(context, (Survey)item.Payload); break; default: throw new ArgumentOutOfRangeException(); @@ -115,7 +115,7 @@ private async Task ProcessNotification(ApplicationDbContext context, Notificatio foreach (var webhook in webhooks) { - var t = new Thread(() => { HandleWebhook(webhook, item); }) {IsBackground = true}; + var t = new Thread(() => { HandleWebhook(webhook, item); }) { IsBackground = true }; t.Start(); } @@ -128,84 +128,6 @@ private async Task ProcessNotification(ApplicationDbContext context, Notificatio } } - internal static async void HandleWebhook(Webhook webhook, NotificationQueueEntry payload) - { - var historyTimeline = JsonConvert.DeserializeObject(payload.Payload.ToString()); - // Serialize our concrete class into a JSON String - - var formattedResponse = webhook.PostbackFormat; - - var isValid = false; - var reg = new Regex(@"\[(.*?)\]"); - foreach (Match match in reg.Matches(formattedResponse)) - switch (match.Value.ToLower()) - { - case "[machinename]": - formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.MachineId.ToString()); - break; - case "[datetime.utcnow]": - //json formatted date! - formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.CreatedUtc.ToString("s")); - break; - case "[messagetype]": - formattedResponse = formattedResponse.Replace(match.Value, "Binary"); - break; - case "[messagepayload]": - if (payload.Payload["Result"] != null && !string.IsNullOrEmpty(payload.Payload["Result"].ToString())) - { - var p = payload.Payload["Result"].ToString().Trim().Trim('"').Trim().Trim('"'); - - p = $"\"{HttpUtility.JavaScriptStringEncode(p)}\""; - - formattedResponse = formattedResponse.Replace(match.Value, p); - isValid = true; - } - - break; - } - - if (!isValid) - { - log.Trace("Webhook has no payload, exiting"); - return; - } - - try - { - // Wrap our JSON inside a StringContent which then can be used by the HttpClient class - var httpContent = new StringContent(formattedResponse, Encoding.UTF8, "application/json"); - - using var httpClient = new HttpClient(); - // Do the actual request and await the response - HttpResponseMessage httpResponse; - switch (webhook.PostbackMethod) - { - default: - throw new ArgumentException("webhook configuration encountered unspecified postback method"); - case Webhook.WebhookMethod.POST: - httpResponse = await httpClient.PostAsync(webhook.PostbackUrl, httpContent); - break; - case Webhook.WebhookMethod.GET: - httpResponse = await httpClient.GetAsync($"{webhook.PostbackUrl}?message={formattedResponse}"); - break; - } - - log.Trace($"Webhook response {webhook.PostbackUrl} {webhook.PostbackMethod} {httpResponse.StatusCode}"); - - // If the response contains content we want to read it! - if (httpResponse.Content != null) - { - var responseContent = await httpResponse.Content.ReadAsStringAsync(); - log.Trace($"Webhook notification sent with {responseContent}"); - // From here on you could deserialize the ResponseContent back again to a concrete C# type using Json.Net - } - } - catch (Exception e) - { - log.Trace($"Webhook failed response {webhook.PostbackUrl} {webhook.PostbackMethod} - {e}"); - } - } - private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext context, MachineQueueEntry item) { var service = scope.ServiceProvider.GetRequiredService(); @@ -272,7 +194,7 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont if (item.LogDump.Log.Length > 0) log.Trace(item.LogDump.Log); - var lines = item.LogDump.Log.Split(new[] {Environment.NewLine}, StringSplitOptions.None); + var lines = item.LogDump.Log.Split(new[] { Environment.NewLine }, StringSplitOptions.None); foreach (var line in lines) try { @@ -318,7 +240,7 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont }; if (data.Tags != null) - timeline.Tags = data.Tags; + timeline.Tags = data.Tags; if (data.Result != null) timeline.Result = data.Result; @@ -334,7 +256,7 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont new NotificationQueueEntry { Type = NotificationQueueEntry.NotificationType.Timeline, - Payload = (JObject) JToken.FromObject(timeline) + Payload = (JObject)JToken.FromObject(timeline) } }); @@ -492,5 +414,91 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont } } } + + internal static async void HandleWebhook(Webhook webhook, NotificationQueueEntry payload) + { + string formattedResponse; + if (payload.Type == NotificationQueueEntry.NotificationType.TimelineDelivered) + { + formattedResponse = payload.Payload.ToString(); + } + else + { + var historyTimeline = JsonConvert.DeserializeObject(payload.Payload.ToString()); + // Serialize our concrete class into a JSON String + + formattedResponse = webhook.PostbackFormat; + + var isValid = false; + var reg = new Regex(@"\[(.*?)\]"); + foreach (Match match in reg.Matches(formattedResponse)) + switch (match.Value.ToLower()) + { + case "[machinename]": + formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.MachineId.ToString()); + break; + case "[datetime.utcnow]": + //json formatted date! + formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.CreatedUtc.ToString("s")); + break; + case "[messagetype]": + formattedResponse = formattedResponse.Replace(match.Value, "Binary"); + break; + case "[messagepayload]": + if (payload.Payload["Result"] != null && !string.IsNullOrEmpty(payload.Payload["Result"].ToString())) + { + var p = payload.Payload["Result"].ToString().Trim().Trim('"').Trim().Trim('"'); + + p = $"\"{HttpUtility.JavaScriptStringEncode(p)}\""; + + formattedResponse = formattedResponse.Replace(match.Value, p); + isValid = true; + } + + break; + } + + if (!isValid) + { + log.Trace("Webhook has no payload, exiting"); + return; + } + } + + try + { + // Wrap our JSON inside a StringContent which then can be used by the HttpClient class + var httpContent = new StringContent(formattedResponse, Encoding.UTF8, "application/json"); + + using var httpClient = new HttpClient(); + // Do the actual request and await the response + HttpResponseMessage httpResponse; + switch (webhook.PostbackMethod) + { + default: + throw new ArgumentException("webhook configuration encountered unspecified postback method"); + case Webhook.WebhookMethod.POST: + httpResponse = await httpClient.PostAsync(webhook.PostbackUrl, httpContent); + break; + case Webhook.WebhookMethod.GET: + httpResponse = await httpClient.GetAsync($"{webhook.PostbackUrl}?message={formattedResponse}"); + break; + } + + log.Trace($"Webhook response {webhook.PostbackUrl} {webhook.PostbackMethod} {httpResponse.StatusCode}"); + + // If the response contains content we want to read it! + if (httpResponse.Content != null) + { + var responseContent = await httpResponse.Content.ReadAsStringAsync(); + log.Trace($"Webhook notification sent with {responseContent}"); + // From here on you could deserialize the ResponseContent back again to a concrete C# type using Json.Net + } + } + catch (Exception e) + { + log.Trace($"Webhook failed response {webhook.PostbackUrl} {webhook.PostbackMethod} - {e}"); + } + } } } \ No newline at end of file From 321ab26a647267d5015ad391c546d28dadffa435 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 20 Oct 2021 11:10:19 -0400 Subject: [PATCH 12/42] Adds chrome as a linux handler --- .../Handlers/BrowserChrome.cs | 134 ++++++++++++++++++ .../Handlers/BrowserFirefox.cs | 22 +-- .../TimelineManager/Orchestrator.cs | 12 +- src/ghosts.client.linux/config/timeline.json | 2 +- 4 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 src/ghosts.client.linux/Handlers/BrowserChrome.cs diff --git a/src/ghosts.client.linux/Handlers/BrowserChrome.cs b/src/ghosts.client.linux/Handlers/BrowserChrome.cs new file mode 100644 index 00000000..8d15591e --- /dev/null +++ b/src/ghosts.client.linux/Handlers/BrowserChrome.cs @@ -0,0 +1,134 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using Ghosts.Domain; +using OpenQA.Selenium.Chrome; +using System; +using System.IO; +using OpenQA.Selenium; + +namespace ghosts.client.linux.handlers +{ + public class BrowserChrome : BaseBrowserHandler + { + public new IJavaScriptExecutor JS { get; private set; } + + private string GetInstallLocation() + { + var path = "/bin/google chrome"; + if (File.Exists(path)) + { + return path; + } + + path = "/usr/bin/google chrome"; + return path; + } + + public BrowserChrome(TimelineHandler handler) + { + BrowserType = HandlerType.BrowserChrome; + var hasRunSuccessfully = false; + while (!hasRunSuccessfully) + { + hasRunSuccessfully = ChromeEx(handler); + } + } + + private bool ChromeEx(TimelineHandler handler) + { + BrowserType = HandlerType.BrowserChrome; + try + { + var options = new ChromeOptions(); + options.AddArguments("disable-infobars"); + options.AddArguments("disable-logging"); + options.AddArguments("--disable-logging"); + options.AddArgument("--log-level=3"); + options.AddArgument("--silent"); + + options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); + options.AddUserProfilePreference("disable-popup-blocking", "true"); + //options.BinaryLocation = GetInstallLocation(); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKey("executable-location") && + !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) + { + options.BinaryLocation = handler.HandlerArgs["executable-location"]; + } + + if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") + { + options.AddArguments("headless"); + } + + if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") + { + options.AddArguments("--incognito"); + } + + if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") + { + options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); + } + + if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") + { + options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); + } + + if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") + { + // ? + } + + if (handler.HandlerArgs.ContainsKey("blockscripts") && + handler.HandlerArgs["blockscripts"] == "true") + { + options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); + } + } + + options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + + if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + { + options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + } + + Driver = new ChromeDriver(options); + base.Driver = Driver; + + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + + Driver.Navigate().GoToUrl(handler.Initial); + + if (handler.Loop) + { + while (true) + { + ExecuteEvents(handler); + } + } + else + { + ExecuteEvents(handler); + } + } + catch (Exception e) + { + _log.Error(e); + return false; + } + + return true; + } + } +} diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index f333d43d..3f92e2ca 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -20,31 +20,17 @@ public class BrowserFirefox : BaseBrowserHandler public BrowserFirefox(TimelineHandler handler) { BrowserType = HandlerType.BrowserFirefox; - bool hasRunSuccessfully = false; + var hasRunSuccessfully = false; while (!hasRunSuccessfully) { hasRunSuccessfully = FirefoxEx(handler); } } - private string GetInstallLocation() - { - var path = "/bin/firefox"; - if (File.Exists(path)) - { - return path; - } - - path = "/usr/bin/firefox"; - return File.Exists(path) ? path : Program.Configuration.FirefoxInstallLocation; - } - private bool FirefoxEx(TimelineHandler handler) { try { - var path = GetInstallLocation(); - var options = new FirefoxOptions(); options.AddArguments("--disable-infobars"); options.AddArguments("--disable-extensions"); @@ -126,11 +112,7 @@ private bool FirefoxEx(TimelineHandler handler) _log.Debug(e); return false; } - finally - { - - } - + return true; } } diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index a5d3ca56..78091663 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -134,6 +134,12 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) o = new Curl(handler); }); break; + case HandlerType.BrowserChrome: + t = new Thread(() => + { + o = new BrowserChrome(handler); + }); + break; case HandlerType.BrowserFirefox: t = new Thread(() => { @@ -142,7 +148,11 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) break; } - if (t == null) return; + if (t == null) + { + _log.Debug($"HandlerType {handler.HandlerType} not supported on this platform"); + return; + } t.IsBackground = true; t.Start(); diff --git a/src/ghosts.client.linux/config/timeline.json b/src/ghosts.client.linux/config/timeline.json index 01a64cc3..75a5d824 100755 --- a/src/ghosts.client.linux/config/timeline.json +++ b/src/ghosts.client.linux/config/timeline.json @@ -2,7 +2,7 @@ "Status": "Run", "TimeLineHandlers": [ { - "HandlerType": "BrowserFirefox", + "HandlerType": "BrowserChrome", "HandlerArgs": { "isheadless": "false", "blockimages": "true", From 800424636c15629557ddf2d9de6b8358fe6863f9 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 20 Oct 2021 11:33:10 -0400 Subject: [PATCH 13/42] Browser cleanup --- src/ghosts.client.linux/Handlers/BrowserChrome.cs | 14 +------------- src/ghosts.client.linux/Handlers/BrowserFirefox.cs | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/ghosts.client.linux/Handlers/BrowserChrome.cs b/src/ghosts.client.linux/Handlers/BrowserChrome.cs index 8d15591e..894ec6e4 100644 --- a/src/ghosts.client.linux/Handlers/BrowserChrome.cs +++ b/src/ghosts.client.linux/Handlers/BrowserChrome.cs @@ -3,27 +3,15 @@ using Ghosts.Domain; using OpenQA.Selenium.Chrome; using System; -using System.IO; using OpenQA.Selenium; namespace ghosts.client.linux.handlers { public class BrowserChrome : BaseBrowserHandler { + public new IWebDriver Driver { get; private set; } public new IJavaScriptExecutor JS { get; private set; } - private string GetInstallLocation() - { - var path = "/bin/google chrome"; - if (File.Exists(path)) - { - return path; - } - - path = "/usr/bin/google chrome"; - return path; - } - public BrowserChrome(TimelineHandler handler) { BrowserType = HandlerType.BrowserChrome; diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index 3f92e2ca..7143ce73 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -3,7 +3,6 @@ using Ghosts.Domain; using NLog; using System; -using System.IO; using System.Text; using OpenQA.Selenium; using OpenQA.Selenium.Firefox; @@ -74,6 +73,7 @@ private bool FirefoxEx(TimelineHandler handler) CodePagesEncodingProvider.Instance.GetEncoding(437); Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + Driver = new FirefoxDriver(options); base.Driver = Driver; From 9ad4630f48d649d0f0fa93f4a36a710709708426 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 20 Oct 2021 11:42:04 -0400 Subject: [PATCH 14/42] prioritize relative links over http|s links for browsing --- src/Ghosts.Domain/Code/LinkManager.cs | 7 ++++--- src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs | 4 ++-- src/ghosts.client.linux/Handlers/Curl.cs | 4 ++-- src/ghosts.client.linux/config/timeline.json | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Ghosts.Domain/Code/LinkManager.cs b/src/Ghosts.Domain/Code/LinkManager.cs index ca52d8f2..9a189d3a 100644 --- a/src/Ghosts.Domain/Code/LinkManager.cs +++ b/src/Ghosts.Domain/Code/LinkManager.cs @@ -31,14 +31,15 @@ public LinkManager(string baseUrl) public List Links { private set; get; } /// - /// Adds proper links — invalid links get quickly discarded + /// Adds proper links — invalid links get quickly discarded /// /// http|s://some.link/path/etc - public void AddLink(string url) + /// priority for choosing next - indeterminate as highest to 0 as lowest priority + public void AddLink(string url, int priority) { try { - Links.Add(new Link {Url = new Uri(url)}); + Links.Add(new Link {Url = new Uri(url), Priority = priority}); } catch { diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index c05be22c..de6791fd 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -97,12 +97,12 @@ public void ExecuteEvents(TimelineHandler handler) // http|s links else if (node.ToLower().StartsWith("http")) { - linkManager.AddLink(node.ToLower()); + linkManager.AddLink(node.ToLower(), 1); } // relative links - prefix the scheme and host else { - linkManager.AddLink($"{config.GetHost()}{node.ToLower()}"); + linkManager.AddLink($"{config.GetHost()}{node.ToLower()}", 2); } } diff --git a/src/ghosts.client.linux/Handlers/Curl.cs b/src/ghosts.client.linux/Handlers/Curl.cs index e7aa3cb2..90032ca2 100755 --- a/src/ghosts.client.linux/Handlers/Curl.cs +++ b/src/ghosts.client.linux/Handlers/Curl.cs @@ -189,12 +189,12 @@ private void DeepBrowse() // http|s links else if (node.Attributes["href"].Value.ToLower().StartsWith("http")) { - linkManager.AddLink(node.Attributes["href"].Value.ToLower()); + linkManager.AddLink(node.Attributes["href"].Value.ToLower(), 1); } // relative links - prefix the scheme and host else { - linkManager.AddLink($"{this._currentHost}{node.Attributes["href"].Value.ToLower()}"); + linkManager.AddLink($"{this._currentHost}{node.Attributes["href"].Value.ToLower()}", 2); } } diff --git a/src/ghosts.client.linux/config/timeline.json b/src/ghosts.client.linux/config/timeline.json index 75a5d824..84b06366 100755 --- a/src/ghosts.client.linux/config/timeline.json +++ b/src/ghosts.client.linux/config/timeline.json @@ -9,7 +9,7 @@ "blockstyles": "true", "blockflash": "true", "blockscripts": "true", - "stickiness": 75, + "stickiness": 100, "stickiness-depth-min": 5, "stickiness-depth-max": 10000, "incognito": "true" From ad634d4083cfeaef3afb6a96f427711ebc2c0629 Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Wed, 20 Oct 2021 11:43:22 -0400 Subject: [PATCH 15/42] prioritize relative links for browsing stickiness --- src/Ghosts.Client/Handlers/BaseBrowserHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index be13b2c6..c15c3ac8 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -97,12 +97,12 @@ public void ExecuteEvents(TimelineHandler handler) // http|s links else if (node.ToLower().StartsWith("http")) { - linkManager.AddLink(node.ToLower()); + linkManager.AddLink(node.ToLower(), 1); } // relative links - prefix the scheme and host else { - linkManager.AddLink($"{config.GetHost()}{node.ToLower()}"); + linkManager.AddLink($"{config.GetHost()}{node.ToLower()}", 2); } } From 7e2946e0f797ef86f23e67bed1f5d22535d1d133 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 20 Oct 2021 13:23:40 -0400 Subject: [PATCH 16/42] better linkmanager choosing of next link to browse --- src/Ghosts.Domain/Code/LinkManager.cs | 65 ++++++++++++++++++--------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/src/Ghosts.Domain/Code/LinkManager.cs b/src/Ghosts.Domain/Code/LinkManager.cs index 9a189d3a..ed3d39ba 100644 --- a/src/Ghosts.Domain/Code/LinkManager.cs +++ b/src/Ghosts.Domain/Code/LinkManager.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; namespace Ghosts.Domain.Code { @@ -19,6 +20,8 @@ public Link() public class LinkManager { + public List Links { private set; get; } + private readonly string _baseUrl; private readonly Random _random = new Random(); @@ -27,57 +30,75 @@ public LinkManager(string baseUrl) Links = new List(); _baseUrl = baseUrl; } - - public List Links { private set; get; } - - /// - /// Adds proper links — invalid links get quickly discarded - /// - /// http|s://some.link/path/etc - /// priority for choosing next - indeterminate as highest to 0 as lowest priority + public void AddLink(string url, int priority) { try { Links.Add(new Link {Url = new Uri(url), Priority = priority}); } - catch + catch(Exception e) { + Console.WriteLine($"{url} {e}"); } } public Link Choose() { + var pickList = new List(); var baseUri = new Uri(_baseUrl); + var schemesToIgnore = new [] {"mailto", "skype", "tel"}; + foreach (var link in Links) try { - if (!link.Url.Host.Replace("www.", "").Contains(baseUri.Host.Replace("www.", ""))) - link.Priority += 10; + if(schemesToIgnore.Any(s => s.StartsWith(link.Url.ToString()))) + continue; + + // give relative links priority + if((link.Url.Scheme + link.Url.Host).Replace("www.", "").Equals((baseUri.Scheme + baseUri.Host).Replace("www.", ""), StringComparison.InvariantCultureIgnoreCase)) + link.Priority += 1; + else if(link.Url.Scheme.Equals("file", StringComparison.InvariantCultureIgnoreCase)) + link.Priority += 1; + + pickList.Add(link); } catch (Exception e) { Console.WriteLine($"{link.Url} : {e}"); } - Links = Links.OrderByDescending(o => o.Priority).ToList(); + Links = pickList.OrderByDescending(o => o.Priority).ToList(); if (Links.Count < 1) return null; - var totalWeight = Convert.ToInt32(Links.Sum(o => o.Priority)); + var priority = Links.First().Priority; + var chosen = Links.Where(x => x.Priority == priority).PickRandom(); - // totalWeight is the sum of all weights - var r = _random.Next(0, totalWeight); - - foreach (var link in Links) + if (chosen.Url.Scheme.ToLower().StartsWith("file")) { - if (r < link.Priority) return link; - - r -= link.Priority; + try + { + var bUrl = baseUri.ToString(); + if (bUrl.EndsWith("/")) + bUrl = bUrl.Substring(0, bUrl.Length - 1); + + var thisUrl = chosen.Url.ToString().Replace("file://",""); + + thisUrl = Regex.Replace(thisUrl, "////", "//"); + if (thisUrl.StartsWith("/")) + thisUrl = thisUrl.Substring(1, thisUrl.Length - 1); + + chosen.Url = new Uri($"{bUrl}/{thisUrl}"); + } + catch (Exception e) + { + Console.WriteLine($"{chosen.Url} : {e}"); + } } - return Links.PickRandom(); + return chosen; } - } + } } \ No newline at end of file From 116e123e26a6e6bec1a536f6efd6bf299b282919 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Fri, 22 Oct 2021 08:36:11 -0400 Subject: [PATCH 17/42] linux code cleanup --- .../{Comms => Communications}/CheckId.cs | 24 ++- .../{Comms => Communications}/Updates.cs | 109 ++++---------- .../Handlers/BaseBrowserHandler.cs | 30 +--- .../Handlers/BaseHandler.cs | 17 +-- src/ghosts.client.linux/Handlers/Bash.cs | 18 ++- .../Handlers/BrowserChrome.cs | 11 +- .../Handlers/BrowserFirefox.cs | 11 +- src/ghosts.client.linux/Handlers/Curl.cs | 137 ++++++++---------- src/ghosts.client.linux/Handlers/NpcSystem.cs | 9 +- src/ghosts.client.linux/Health/Check.cs | 51 +++---- .../Health/HealthRecord.cs | 19 ++- .../Health/MachineHealth.cs | 23 ++- .../Browser/ExtendedConfiguration.cs | 10 +- .../Browser/RequestConfiguration.cs | 5 +- .../ClientConfigurationResolver.cs | 14 -- .../Infrastructure/CommandLineFlagManager.cs | 9 +- .../Infrastructure/StartupTasks.cs | 39 +---- .../Infrastructure/WebClientHeaders.cs | 2 +- src/ghosts.client.linux/Program.cs | 16 +- .../TimelineManager/Listener.cs | 27 ++-- .../TimelineManager/Orchestrator.cs | 28 ++-- 21 files changed, 238 insertions(+), 371 deletions(-) rename src/ghosts.client.linux/{Comms => Communications}/CheckId.cs (78%) rename src/ghosts.client.linux/{Comms => Communications}/Updates.cs (68%) delete mode 100644 src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs diff --git a/src/ghosts.client.linux/Comms/CheckId.cs b/src/ghosts.client.linux/Communications/CheckId.cs similarity index 78% rename from src/ghosts.client.linux/Comms/CheckId.cs rename to src/ghosts.client.linux/Communications/CheckId.cs index 2bd52214..ae053c3f 100644 --- a/src/ghosts.client.linux/Comms/CheckId.cs +++ b/src/ghosts.client.linux/Communications/CheckId.cs @@ -8,10 +8,10 @@ using Ghosts.Domain.Code; using NLog; -namespace ghosts.client.linux.Comms +namespace ghosts.client.linux.Communications { /// - /// The client ID is used in the header to save having to send hostname/user/fqdn/etc. inforamtion with every request + /// The client ID is used in the header to save having to send hostname/user/fqdn/etc. information with every request /// public static class CheckId { @@ -20,7 +20,7 @@ public static class CheckId /// /// The actual path to the client id file, specified in application config /// - public static string ConfigFile = ApplicationDetails.InstanceFiles.Id; + private static readonly string ConfigFile = ApplicationDetails.InstanceFiles.Id; /// /// Gets the agent's current id from local instance, and if it does not exist, gets an id from the server and saves it locally @@ -31,11 +31,7 @@ public static string Id { try { - if (!File.Exists(ConfigFile)) - { - return Run(); - } - return File.ReadAllText(ConfigFile); + return !File.Exists(ConfigFile) ? Run() : File.ReadAllText(ConfigFile); } catch { @@ -63,18 +59,16 @@ private static string Run() { try { - using (var reader = - new StreamReader(client.OpenRead(Program.Configuration.IdUrl))) - { - s = reader.ReadToEnd(); - _log.Debug($"{DateTime.Now} - Received client ID"); - } + using var reader = + new StreamReader(client.OpenRead(Program.Configuration.IdUrl) ?? throw new Exception("Application has invalid ID url")); + s = reader.ReadToEnd(); + _log.Debug($"{DateTime.Now} - Received client ID"); } catch (WebException wex) { if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) { - _log.Debug("No ID returned!", wex); + _log.Debug($"No ID returned from API! {wex}"); } } catch (Exception e) diff --git a/src/ghosts.client.linux/Comms/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs similarity index 68% rename from src/ghosts.client.linux/Comms/Updates.cs rename to src/ghosts.client.linux/Communications/Updates.cs index c6514550..b30b796d 100644 --- a/src/ghosts.client.linux/Comms/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -2,6 +2,7 @@ using System; using System.IO; +using System.Linq; using System.Net; using System.Text; using System.Threading; @@ -13,7 +14,7 @@ using Newtonsoft.Json; using NLog; -namespace ghosts.client.linux.Comms +namespace ghosts.client.linux.Communications { /// /// Get updates from the C2 server - could be timeline, health, etc. @@ -53,17 +54,15 @@ private static void GetServerUpdates() { try { - string s = string.Empty; + var s = string.Empty; using (var client = WebClientBuilder.Build(machine)) { try { - using (var reader = - new StreamReader(client.OpenRead(Program.Configuration.ClientUpdates.PostUrl))) - { - s = reader.ReadToEnd(); - _log.Debug($"{DateTime.Now} - Received new configuration"); - } + using var reader = + new StreamReader(client.OpenRead(Program.Configuration.ClientUpdates.PostUrl)); + s = reader.ReadToEnd(); + _log.Debug($"{DateTime.Now} - Received new configuration"); } catch (WebException wex) { @@ -86,6 +85,7 @@ private static void GetServerUpdates() { var update = JsonConvert.DeserializeObject(s); + // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault switch (update.Type) { case UpdateClientConfig.UpdateType.Timeline: @@ -100,16 +100,12 @@ private static void GetServerUpdates() { _log.Trace($"PartialTimeline found: {timelineHandler.HandlerType}"); - foreach (var timelineEvent in timelineHandler.TimeLineEvents) + foreach (var timelineEvent in timelineHandler.TimeLineEvents.Where(timelineEvent => string.IsNullOrEmpty(timelineEvent.TrackableId))) { - if (string.IsNullOrEmpty(timelineEvent.TrackableId)) - { - timelineEvent.TrackableId = Guid.NewGuid().ToString(); - } + timelineEvent.TrackableId = Guid.NewGuid().ToString(); } - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timeline, timelineHandler); + Orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) @@ -120,14 +116,14 @@ private static void GetServerUpdates() break; case UpdateClientConfig.UpdateType.Health: { - var newTimeline = JsonConvert.DeserializeObject(update.Update.ToString()); + var newTimeline = JsonConvert.DeserializeObject(update.Update.ToString()); //save to local disk - using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Health)) + using var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Health); + var serializer = new JsonSerializer { - var serializer = new JsonSerializer(); - serializer.Formatting = Formatting.Indented; - serializer.Serialize(file, newTimeline); - } + Formatting = Formatting.Indented + }; + serializer.Serialize(file, newTimeline); break; } @@ -155,12 +151,12 @@ private static void PostClientResults() return; var fileName = ApplicationDetails.LogFiles.ClientUpdates; - var cyclesleep = Program.Configuration.ClientResults.CycleSleep; - var posturl = Program.Configuration.ClientResults.PostUrl; + var cycleSleep = Program.Configuration.ClientResults.CycleSleep; + var postUrl = Program.Configuration.ClientResults.PostUrl; var machine = new ResultMachine(); - Thread.Sleep(cyclesleep); + Thread.Sleep(cycleSleep); while (true) { @@ -168,7 +164,7 @@ private static void PostClientResults() { if(File.Exists(fileName)) { - PostResults(fileName, machine, posturl); + PostResults(fileName, machine, postUrl); } else { @@ -187,16 +183,16 @@ private static void PostClientResults() { if (!file.EndsWith("app.log") && file != fileName) { - PostResults(file, machine, posturl, true); + PostResults(file, machine, postUrl, true); } } } catch (Exception e) { - _log.Debug($"Problem posting overflow logs from {fileName} to server {posturl}: {e}"); + _log.Debug($"Problem posting overflow logs from {fileName} to server {postUrl}: {e}"); } - Thread.Sleep(cyclesleep); + Thread.Sleep(cycleSleep); } } @@ -209,8 +205,10 @@ private static void PostResults(string fileName, ResultMachine machine, string p sb.AppendLine(d); } - var r = new TransferLogDump(); - r.Log = sb.ToString(); + var r = new TransferLogDump + { + Log = sb.ToString() + }; var payload = JsonConvert.SerializeObject(r); @@ -219,8 +217,10 @@ private static void PostResults(string fileName, ResultMachine machine, string p payload = Crypto.EncryptStringAes(payload, machine.Name); payload = Base64Encoder.Base64Encode(payload); - var p = new EncryptedPayload(); - p.Payload = payload; + var p = new EncryptedPayload + { + Payload = payload + }; payload = JsonConvert.SerializeObject(p); } @@ -242,50 +242,5 @@ private static void PostResults(string fileName, ResultMachine machine, string p _log.Trace($"{DateTime.Now} - {fileName} posted to server successfully"); } - - internal static void PostSurvey() - { - try - { - _log.Trace("posting survey"); - - var posturl = Program.Configuration.Survey.PostUrl; - - if (!File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) - return; - - var survey = JsonConvert.DeserializeObject(File.ReadAllText(ApplicationDetails.InstanceFiles.SurveyResults)); - - var payload = JsonConvert.SerializeObject(survey); - - var machine = new ResultMachine(); - - if (Program.Configuration.Survey.IsSecure) - { - payload = Crypto.EncryptStringAes(payload, machine.Name); - payload = Base64Encoder.Base64Encode(payload); - - var p = new EncryptedPayload(); - p.Payload = payload; - - payload = JsonConvert.SerializeObject(p); - } - - using (var client = WebClientBuilder.Build(machine)) - { - client.Headers[HttpRequestHeader.ContentType] = "application/json"; - client.UploadString(posturl, payload); - } - - _log.Trace($"{DateTime.Now} - survey posted to server successfully"); - - File.Delete(ApplicationDetails.InstanceFiles.SurveyResults); - } - catch (Exception e) - { - _log.Trace("Problem posting logs to server"); - _log.Error(e); - } - } } } diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index de6791fd..12ab096e 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -8,24 +8,25 @@ using NLog; using OpenQA.Selenium; using OpenQA.Selenium.Interactions; +// ReSharper disable StringLiteralTypo namespace ghosts.client.linux.handlers { public abstract class BaseBrowserHandler : BaseHandler { - public static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public IWebDriver Driver { get; set; } - public IJavaScriptExecutor JS { get; set; } - public HandlerType BrowserType { get; set; } - private int _stickiness = 0; + protected static readonly Logger _log = LogManager.GetCurrentClassLogger(); + protected IWebDriver Driver { get; set; } + protected IJavaScriptExecutor JS { get; set; } + protected HandlerType BrowserType { get; set; } + private int _stickiness; private int _depthMin = 1; private int _depthMax = 10; - public void ExecuteEvents(TimelineHandler handler) + protected void ExecuteEvents(TimelineHandler handler) { try { - foreach (TimelineEvent timelineEvent in handler.TimeLineEvents) + foreach (var timelineEvent in handler.TimeLineEvents) { WorkingHours.Is(handler); @@ -232,20 +233,5 @@ private void MakeRequest(RequestConfiguration config) _log.Trace(e.Message); } } - - /// - /// Close browser - /// - public void Close() - { - Report(BrowserType.ToString(), "Close", string.Empty); - Driver.Close(); - } - - public void Stop() - { - Report(BrowserType.ToString(), "Stop", string.Empty); - Close(); - } } } \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BaseHandler.cs b/src/ghosts.client.linux/Handlers/BaseHandler.cs index 3ac68538..311dde60 100644 --- a/src/ghosts.client.linux/Handlers/BaseHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseHandler.cs @@ -11,17 +11,14 @@ public abstract class BaseHandler { private static readonly Logger _timelineLog = LogManager.GetLogger("TIMELINE"); - public void Report(string handler, string command, string arg) + protected static void Report(string handler, string command, string arg, string trackable = null) { - Report(handler, command, arg, null); - } - - public void Report(string handler, string command, string arg, string trackable) - { - var result = new TimeLineRecord(); - result.Handler = handler; - result.Command = command; - result.CommandArg = arg; + var result = new TimeLineRecord + { + Handler = handler, + Command = command, + CommandArg = arg + }; if (!string.IsNullOrEmpty(trackable)) { diff --git a/src/ghosts.client.linux/Handlers/Bash.cs b/src/ghosts.client.linux/Handlers/Bash.cs index 8449a9de..58e459c8 100644 --- a/src/ghosts.client.linux/Handlers/Bash.cs +++ b/src/ghosts.client.linux/Handlers/Bash.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; +using System.Linq; using System.Threading; using Ghosts.Domain; using Ghosts.Domain.Code; @@ -12,7 +13,7 @@ namespace ghosts.client.linux.handlers public class Bash : BaseHandler { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public string Result { get; private set; } + private string Result { get; set; } public Bash(TimelineHandler handler) { @@ -36,7 +37,7 @@ public Bash(TimelineHandler handler) } } - public void Ex(TimelineHandler handler) + private void Ex(TimelineHandler handler) { foreach (var timelineEvent in handler.TimeLineEvents) { @@ -53,9 +54,10 @@ public void Ex(TimelineHandler handler) this.Command(handler.Initial, timelineEvent.Command); - foreach (var cmd in timelineEvent.CommandArgs) - if (!string.IsNullOrEmpty(cmd.ToString())) - this.Command(handler.Initial, cmd.ToString()); + foreach (var cmd in timelineEvent.CommandArgs.Where(cmd => !string.IsNullOrEmpty(cmd.ToString()))) + { + this.Command(handler.Initial, cmd.ToString()); + } break; } @@ -88,15 +90,15 @@ private void Command(string initial, string command) p.WaitForExit(); - this.Report(HandlerType.Command.ToString(), escapedArgs, this.Result); + Report(HandlerType.Command.ToString(), escapedArgs, this.Result); } - void OutputHandler(object sendingProcess, DataReceivedEventArgs outLine) + private void OutputHandler(object sendingProcess, DataReceivedEventArgs outLine) { this.Result += outLine.Data; } - void ErrorHandler(object sendingProcess, DataReceivedEventArgs outLine) + private static void ErrorHandler(object sendingProcess, DataReceivedEventArgs outLine) { //* Do your stuff with the output (write to console/log/StringBuilder) Console.WriteLine(outLine.Data); diff --git a/src/ghosts.client.linux/Handlers/BrowserChrome.cs b/src/ghosts.client.linux/Handlers/BrowserChrome.cs index 894ec6e4..40cf60e3 100644 --- a/src/ghosts.client.linux/Handlers/BrowserChrome.cs +++ b/src/ghosts.client.linux/Handlers/BrowserChrome.cs @@ -4,13 +4,14 @@ using OpenQA.Selenium.Chrome; using System; using OpenQA.Selenium; +// ReSharper disable StringLiteralTypo namespace ghosts.client.linux.handlers { public class BrowserChrome : BaseBrowserHandler { - public new IWebDriver Driver { get; private set; } - public new IJavaScriptExecutor JS { get; private set; } + private new IWebDriver Driver { get; set; } + private new IJavaScriptExecutor JS { get; set; } public BrowserChrome(TimelineHandler handler) { @@ -105,10 +106,8 @@ private bool ChromeEx(TimelineHandler handler) ExecuteEvents(handler); } } - else - { - ExecuteEvents(handler); - } + + ExecuteEvents(handler); } catch (Exception e) { diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index 7143ce73..79fec24a 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -6,6 +6,7 @@ using System.Text; using OpenQA.Selenium; using OpenQA.Selenium.Firefox; +// ReSharper disable StringLiteralTypo namespace ghosts.client.linux.handlers { @@ -13,8 +14,8 @@ public class BrowserFirefox : BaseBrowserHandler { private new static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public new IWebDriver Driver { get; private set; } - public new IJavaScriptExecutor JS { get; private set; } + private new IWebDriver Driver { get; set; } + private new IJavaScriptExecutor JS { get; set; } public BrowserFirefox(TimelineHandler handler) { @@ -102,10 +103,8 @@ private bool FirefoxEx(TimelineHandler handler) ExecuteEvents(handler); } } - else - { - ExecuteEvents(handler); - } + + ExecuteEvents(handler); } catch (Exception e) { diff --git a/src/ghosts.client.linux/Handlers/Curl.cs b/src/ghosts.client.linux/Handlers/Curl.cs index 90032ca2..2bd77d3b 100755 --- a/src/ghosts.client.linux/Handlers/Curl.cs +++ b/src/ghosts.client.linux/Handlers/Curl.cs @@ -14,9 +14,9 @@ namespace ghosts.client.linux.handlers public class Curl : BaseHandler { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public string Result { get; private set; } + private string Result { get; set; } private readonly TimelineHandler _handler; - private readonly int _stickiness = 0; + private readonly int _stickiness; private readonly int _depthMin = 1; private readonly int _depthMax = 10; private int _wait = 500; @@ -55,10 +55,8 @@ public Curl(TimelineHandler handler) Ex(); } } - else - { - Ex(); - } + + Ex(); } catch (Exception e) { @@ -80,22 +78,18 @@ private void Ex() default: this.Command(timelineEvent.Command); - foreach (var cmd in timelineEvent.CommandArgs) + foreach (var cmd in timelineEvent.CommandArgs.Where(cmd => !string.IsNullOrEmpty(cmd.ToString()))) { - if (!string.IsNullOrEmpty(cmd.ToString())) - { - this.Command(cmd.ToString()); - } + this.Command(cmd.ToString()); } break; } - if (timelineEvent.DelayAfter > 0) - { - _wait = timelineEvent.DelayAfter; - Thread.Sleep(timelineEvent.DelayAfter); - } + if (timelineEvent.DelayAfter <= 0) continue; + + _wait = timelineEvent.DelayAfter; + Thread.Sleep(timelineEvent.DelayAfter); } } @@ -141,7 +135,7 @@ private void Command(string command) this.Result += p.StandardOutput.ReadToEnd(); } - this.Report(HandlerType.Curl.ToString(), escapedArgs, this.Result); + Report(HandlerType.Curl.ToString(), escapedArgs, this.Result); this.DeepBrowse(); } catch (Exception exc) @@ -155,72 +149,69 @@ private void Command(string command) /// private void DeepBrowse() { - if (_stickiness > 0) + if (_stickiness <= 0) return; + var random = new Random(); + if (random.Next(100) >= _stickiness) return; + + // some percentage of the time, we should stay on this site + var loops = random.Next(_depthMin, _depthMax); + for (var loopNumber = 0; loopNumber < loops; loopNumber++) { - var random = new Random(); - // some percentage of the time, we should stay on this site - if (random.Next(100) < _stickiness) + try { - var loops = random.Next(_depthMin, _depthMax); - for (var loopNumber = 0; loopNumber < loops; loopNumber++) + //get all links from results + var doc = new HtmlDocument(); + doc.LoadHtml(this.Result.ToLower()); + + var nodes = doc.DocumentNode.SelectNodes("//a"); + if (nodes == null || !nodes.Any()) { - try + return; + } + + var linkManager = new LinkManager(this._currentHost); + foreach (var node in nodes) + { + if (!node.HasAttributes + || node.Attributes["href"] == null + || string.IsNullOrEmpty(node.Attributes["href"].Value) + || node.Attributes["href"].Value.ToLower().StartsWith("//")) { - //get all links from results - var doc = new HtmlDocument(); - doc.LoadHtml(this.Result.ToLower()); - - var nodes = doc.DocumentNode.SelectNodes("//a"); - if (nodes == null || !nodes.Any()) - { - return; - } - - var linkManager = new LinkManager(this._currentHost); - foreach (var node in nodes) - { - if (!node.HasAttributes - || node.Attributes["href"] == null - || string.IsNullOrEmpty(node.Attributes["href"].Value) - || node.Attributes["href"].Value.ToLower().StartsWith("//")) - { - //skip, these seem ugly - } - // http|s links - else if (node.Attributes["href"].Value.ToLower().StartsWith("http")) - { - linkManager.AddLink(node.Attributes["href"].Value.ToLower(), 1); - } - // relative links - prefix the scheme and host - else - { - linkManager.AddLink($"{this._currentHost}{node.Attributes["href"].Value.ToLower()}", 2); - } - } - - var link = linkManager.Choose(); - if (link == null) - { - return; - } - var href = link.Url.ToString(); - - if (!string.IsNullOrEmpty(href)) - { - this.Result = ""; - Command(href); - } + //skip, these seem ugly } - catch (Exception e) + // http|s links + else if (node.Attributes["href"].Value.ToLower().StartsWith("http")) { - _log.Error(e); + linkManager.AddLink(node.Attributes["href"].Value.ToLower(), 1); } - - if (_wait > 0) + // relative links - prefix the scheme and host + else { - Thread.Sleep(_wait); + linkManager.AddLink($"{this._currentHost}{node.Attributes["href"].Value.ToLower()}", 2); } } + + var link = linkManager.Choose(); + if (link == null) + { + return; + } + var href = link.Url.ToString(); + + if (!string.IsNullOrEmpty(href)) + { + this.Result = ""; + Command(href); + } + } + catch (Exception e) + { + _log.Error(e); + } + + if (_wait > 0) + { + Thread.Sleep(_wait); } } } diff --git a/src/ghosts.client.linux/Handlers/NpcSystem.cs b/src/ghosts.client.linux/Handlers/NpcSystem.cs index 75f4aff5..3b8b47d6 100644 --- a/src/ghosts.client.linux/Handlers/NpcSystem.cs +++ b/src/ghosts.client.linux/Handlers/NpcSystem.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Linq; using ghosts.client.linux.Infrastructure; using ghosts.client.linux.timelineManager; using Ghosts.Domain; @@ -17,11 +18,8 @@ public NpcSystem(Timeline timeline, TimelineHandler handler) { _log.Trace($"Handling NpcSystem call: {handler}"); - foreach (var timelineEvent in handler.TimeLineEvents) + foreach (var timelineEvent in handler.TimeLineEvents.Where(timelineEvent => !string.IsNullOrEmpty(timelineEvent.Command))) { - if (string.IsNullOrEmpty(timelineEvent.Command)) - continue; - Timeline t; switch (timelineEvent.Command.ToLower()) @@ -34,8 +32,7 @@ public NpcSystem(Timeline timeline, TimelineHandler handler) case "stop": if (timeline.Id != Guid.Empty) { - var o = new Orchestrator(); - o.StopTimeline(timeline.Id); + Orchestrator.StopTimeline(timeline.Id); } else { diff --git a/src/ghosts.client.linux/Health/Check.cs b/src/ghosts.client.linux/Health/Check.cs index f354d462..070ec65b 100644 --- a/src/ghosts.client.linux/Health/Check.cs +++ b/src/ghosts.client.linux/Health/Check.cs @@ -18,7 +18,7 @@ public class Check private static readonly Logger _healthLog = LogManager.GetLogger("HEALTH"); private static DateTime _lastRead = DateTime.MinValue; - private List _threads { get; set; } + private List _threads { get; } public Check() { @@ -35,7 +35,7 @@ public void Run() _log.Trace($"watching {watcher.Path}"); watcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size; watcher.EnableRaisingEvents = true; - watcher.Changed += new FileSystemEventHandler(OnChanged); + watcher.Changed += OnChanged; Thread t = null; new Thread(() => @@ -43,15 +43,13 @@ public void Run() Thread.CurrentThread.IsBackground = true; t = Thread.CurrentThread; t.Name = Guid.NewGuid().ToString(); - this.RunEx(); + RunEx(); }).Start(); - if (t != null) - { - _log.Trace($"HEALTH THREAD: {t.Name}"); - this._threads.Add(t); - } + if (t == null) return; + _log.Trace($"HEALTH THREAD: {t.Name}"); + this._threads.Add(t); } catch (Exception exc) { @@ -59,18 +57,16 @@ public void Run() } } - public void Shutdown() + private void Shutdown() { - if (this._threads != null) + if (this._threads == null) return; + foreach (var thread in this._threads) { - foreach (var thread in this._threads) - { - thread.Abort(null); - } + thread.Abort(null); } } - private void RunEx() + private static void RunEx() { var c = new ConfigHealth(ApplicationDetails.ConfigurationFiles.Health); var config = c.Load(); @@ -97,23 +93,22 @@ private void RunEx() _log.Debug(e); } } + // ReSharper disable once FunctionNeverReturns } private void OnChanged(object source, FileSystemEventArgs e) { - // filewatcher throws two events, we only need 1 - DateTime lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime > _lastRead.AddSeconds(1)) - { - _lastRead = lastWriteTime; - _log.Trace("File: " + e.FullPath + " " + e.ChangeType); - _log.Trace($"Reloading {System.Reflection.MethodBase.GetCurrentMethod().DeclaringType}"); - - // now terminate existing tasks and rerun - this.Shutdown(); - StartupTasks.CleanupProcesses(); - this.RunEx(); - } + // file watcher throws two events, we only need 1 + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime <= _lastRead.AddSeconds(1)) return; + _lastRead = lastWriteTime; + _log.Trace("File: " + e.FullPath + " " + e.ChangeType); + _log.Trace($"Reloading {System.Reflection.MethodBase.GetCurrentMethod()?.DeclaringType}"); + + // now terminate existing tasks and rerun + this.Shutdown(); + StartupTasks.CleanupProcesses(); + RunEx(); } } } diff --git a/src/ghosts.client.linux/Health/HealthRecord.cs b/src/ghosts.client.linux/Health/HealthRecord.cs index d92a316c..3613ef3c 100644 --- a/src/ghosts.client.linux/Health/HealthRecord.cs +++ b/src/ghosts.client.linux/Health/HealthRecord.cs @@ -1,7 +1,6 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.IO; using System.Net; using Ghosts.Domain; @@ -26,15 +25,15 @@ public static ResultHealth Check(ConfigHealth config) try { - using (var response = (HttpWebResponse)request.GetResponse()) + using var response = (HttpWebResponse)request.GetResponse(); + using var stream = response.GetResponseStream(); + if (stream != null) { - using (var stream = response.GetResponseStream()) - { - using (var reader = new StreamReader(stream)) - { - //html = reader.ReadToEnd(); //if we wanted to read html - } - } + // ignore + // using (var reader = new StreamReader(stream)) + // { + // //html = reader.ReadToEnd(); //if we wanted to read html + // } } } catch (WebException e) @@ -49,7 +48,7 @@ public static ResultHealth Check(ConfigHealth config) watch.Stop(); r.ExecutionTime = watch.ElapsedMilliseconds; - r.Internet = (r.Errors.Count == 0); + r.Internet = r.Errors.Count == 0; r.Stats = MachineHealth.Run(); } diff --git a/src/ghosts.client.linux/Health/MachineHealth.cs b/src/ghosts.client.linux/Health/MachineHealth.cs index 89dde2f8..849ae355 100644 --- a/src/ghosts.client.linux/Health/MachineHealth.cs +++ b/src/ghosts.client.linux/Health/MachineHealth.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; using System.IO; +using System.Linq; using Ghosts.Domain; using Ghosts.Domain.Code; using NLog; @@ -31,30 +32,24 @@ public static ResultHealth.MachineStats Run() return stats; } - public static float GetMemory() + private static float GetMemory() { - float totalMemory = 0; - foreach (var process in Process.GetProcesses()) - { - totalMemory += process.PrivateMemorySize64; - } - - return totalMemory; + return Process.GetProcesses().Aggregate(0, (current, process) => current + process.PrivateMemorySize64); } - public static float GetCpu() + private static float GetCpu() { var proc = Process.GetCurrentProcess(); var cpu = proc.TotalProcessorTime; - foreach (var process in Process.GetProcesses()) - { - //Console.WriteLine("Proc {0,30} CPU {1,-20:n} msec", process.ProcessName, cpu.TotalMilliseconds); - } + // foreach (var process in Process.GetProcesses()) + // { + // //Console.WriteLine("Proc {0,30} CPU {1,-20:n} m sec", process.ProcessName, cpu.TotalMilliseconds); + // } return cpu.Ticks; } - public static float GetDiskSpace() + private static float GetDiskSpace() { foreach (var drive in DriveInfo.GetDrives()) { diff --git a/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs b/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs index a4cdcc11..8685f1fa 100755 --- a/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs +++ b/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using Newtonsoft.Json; +// ReSharper disable UnusedMember.Global namespace ghosts.client.linux.Infrastructure.Browser { @@ -20,13 +21,10 @@ public static ExtendedConfiguration Load(object o) { var commandArg = o.ToString(); var result = new ExtendedConfiguration(); - if (commandArg.StartsWith("{")) - { - result = JsonConvert.DeserializeObject(commandArg); - return result; - } - + if (commandArg == null || !commandArg.StartsWith("{")) return result; + result = JsonConvert.DeserializeObject(commandArg); return result; + } } } diff --git a/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs b/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs index 6a0f7d36..ec33256a 100755 --- a/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs +++ b/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs @@ -20,13 +20,16 @@ public string GetHost() /// /// For categorizing browsing, e.g. I want to simulate someone shopping for shoes on x, y and z sites /// + // ReSharper disable once UnusedMember.Global public string Category { get; set; } /// /// GET, POST, PUT, DELETE /// public string Method { get; set; } + // ReSharper disable once UnusedMember.Global public IDictionary Headers { get; set; } public IDictionary FormValues { get; set; } + // ReSharper disable once UnusedMember.Global public string Body { get; set; } public override string ToString() @@ -38,7 +41,7 @@ public static RequestConfiguration Load(object o) { var commandArg = o.ToString(); var result = new RequestConfiguration(); - if (commandArg.StartsWith("{")) + if (commandArg != null && commandArg.StartsWith("{")) { result = JsonConvert.DeserializeObject(commandArg); return result; diff --git a/src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs b/src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs deleted file mode 100644 index d82ed28f..00000000 --- a/src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Ghosts.Domain.Code; - -namespace ghosts.client.linux.Infrastructure -{ - public static class ClientConfigurationResolver - { - public static string Dictionary => ApplicationDetails.ConfigurationFiles.Dictionary(Program.Configuration.Content.Dictionary); - public static string EmailContent => ApplicationDetails.ConfigurationFiles.EmailContent(Program.Configuration.Content.EmailContent); - public static string EmailReply => ApplicationDetails.ConfigurationFiles.EmailReply(Program.Configuration.Content.EmailReply); - public static string EmailDomain => ApplicationDetails.ConfigurationFiles.EmailDomain(Program.Configuration.Content.EmailDomain); - public static string EmailOutside => ApplicationDetails.ConfigurationFiles.EmailOutside(Program.Configuration.Content.EmailOutside); - public static string FileNames => ApplicationDetails.ConfigurationFiles.FileNames(Program.Configuration.Content.FileNames); - } -} \ No newline at end of file diff --git a/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs b/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs index 2cde613e..e5484c4c 100644 --- a/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs +++ b/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs @@ -1,7 +1,8 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + using System; -using System.Drawing; +using System.Collections.Generic; using System.IO; -using System.Linq; using System.Reflection; using System.Runtime.Versioning; using CommandLine; @@ -14,7 +15,7 @@ namespace ghosts.client.linux.Infrastructure { internal static class CommandLineFlagManager { - internal static bool Parse(string[] args) + internal static bool Parse(IEnumerable args) { Console.WriteLine(ApplicationDetails.Header); @@ -32,8 +33,6 @@ internal static bool Parse(string[] args) .ParseArguments(args) .WithParsed(o => options = o); - Program.OptionFlags = options; - // start handling flags that result in program exit if (options.Help) { diff --git a/src/ghosts.client.linux/Infrastructure/StartupTasks.cs b/src/ghosts.client.linux/Infrastructure/StartupTasks.cs index d1cc234c..4c77c627 100644 --- a/src/ghosts.client.linux/Infrastructure/StartupTasks.cs +++ b/src/ghosts.client.linux/Infrastructure/StartupTasks.cs @@ -41,13 +41,14 @@ public static void CleanupProcesses() Thread.CurrentThread.IsBackground = true; foreach (var process in Process.GetProcessesByName(cleanupItem)) { - if (process.Id != ghosts.Id) //don't kill thyself + if (process.Id == ghosts.Id) continue; + try { - try - { - process.Kill(); - } - catch { } + process.Kill(); + } + catch + { + // ignored } } }).Start(); @@ -70,31 +71,7 @@ public static void CleanupProcesses() /// public static void SetStartup() { - return; - /* - try - { - throw new NotImplementedException(); - - [Unit] - Description=GHOSTS NPC Orchestration - After=multi-user.target - - [Service] - Type=simple - ExecStart=/usr/bin/ghosts - - [Install] - WantedBy=multi-user.target - - - //_log.Trace("Startup set successfully"); - } - catch (Exception e) - { - //_log.Debug($"Set startup: {e}"); - } - */ + // ignored } } } diff --git a/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs b/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs index 2337e0a0..9ecffef7 100644 --- a/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs +++ b/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs @@ -1,7 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System.Net; -using ghosts.client.linux.Comms; +using ghosts.client.linux.Communications; using Ghosts.Domain; using Ghosts.Domain.Code; diff --git a/src/ghosts.client.linux/Program.cs b/src/ghosts.client.linux/Program.cs index b67845b4..f8508313 100755 --- a/src/ghosts.client.linux/Program.cs +++ b/src/ghosts.client.linux/Program.cs @@ -7,6 +7,7 @@ using System.Net; using System.Reflection; using System.Threading; +using ghosts.client.linux.Communications; using ghosts.client.linux.Infrastructure; using ghosts.client.linux.timelineManager; using Ghosts.Domain.Code; @@ -15,15 +16,14 @@ namespace ghosts.client.linux { - class Program + internal static class Program { - internal static ClientConfiguration Configuration { get; set; } - internal static Options OptionFlags; + internal static ClientConfiguration Configuration { get; private set; } internal static bool IsDebug; - internal static List ThreadJobs { get; set; } + internal static List ThreadJobs { get; private set; } private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - static void Main(string[] args) + private static void Main(string[] args) { ThreadJobs = new List(); ClientConfigurationLoader.UpdateConfigurationWithEnvVars(); @@ -40,7 +40,7 @@ static void Main(string[] args) } } - private static void Run(string[] args) + private static void Run(IEnumerable args) { // ignore all certs ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; @@ -77,10 +77,10 @@ private static void Run(string[] args) ListenerManager.Run(); //check id - _log.Trace(Comms.CheckId.Id); + _log.Trace(CheckId.Id); //connect to command server for updates and sending logs - Comms.Updates.Run(); + Updates.Run(); //linux clients do not perform local survey diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index 2faf9a49..2129bb12 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -111,7 +111,7 @@ public InitialDirectoryListener() new ManualResetEvent(false).WaitOne(); } - private void InitialOnChanged(object source, FileSystemEventArgs e) + private static void InitialOnChanged(object source, FileSystemEventArgs e) { // file watcher throws two events, we only need 1 var lastWriteTime = File.GetLastWriteTime(e.FullPath); @@ -131,7 +131,7 @@ private void InitialOnChanged(object source, FileSystemEventArgs e) // now terminate existing tasks and rerun var o = new Orchestrator(); - o.Stop(); + Orchestrator.Stop(); o.Run(); } } @@ -162,7 +162,7 @@ public DirectoryListener() private void OnChanged(object source, FileSystemEventArgs e) { - // filewatcher throws multiple events, we only need 1 + // file watcher throws multiple events, we only need 1 if (!string.IsNullOrEmpty(_currentlyProcessing) && _currentlyProcessing == e.FullPath) return; _currentlyProcessing = e.FullPath; @@ -180,16 +180,12 @@ private void OnChanged(object source, FileSystemEventArgs e) { _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); - foreach (var timelineEvent in timelineHandler.TimeLineEvents) + foreach (var timelineEvent in timelineHandler.TimeLineEvents.Where(timelineEvent => string.IsNullOrEmpty(timelineEvent.TrackableId))) { - if (string.IsNullOrEmpty(timelineEvent.TrackableId)) - { - timelineEvent.TrackableId = Guid.NewGuid().ToString(); - } + timelineEvent.TrackableId = Guid.NewGuid().ToString(); } - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timeline, timelineHandler); + Orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) @@ -205,7 +201,6 @@ private void OnChanged(object source, FileSystemEventArgs e) if (commands.Count > 0) { var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); - var orchestrator = new Orchestrator(); var t = new Timeline { @@ -213,7 +208,7 @@ private void OnChanged(object source, FileSystemEventArgs e) Status = Timeline.TimelineStatus.Run }; t.TimeLineHandlers.Add(constructedTimelineHandler); - orchestrator.RunCommand(t, constructedTimelineHandler); + Orchestrator.RunCommand(t, constructedTimelineHandler); } } catch (Exception exc) @@ -265,12 +260,12 @@ public Listener() } } - private string Handle(Message message) + private static string Handle(Message message) { var command = message.MessageString; var index = command.LastIndexOf("}", StringComparison.InvariantCultureIgnoreCase); if (index > 0) - command = command.Substring(0, index + 1); + command = command[..(index + 1)]; _log.Trace($"Received from {message.TcpClient.Client.RemoteEndPoint}: {command}"); @@ -285,8 +280,6 @@ private string Handle(Message message) _log.Trace($"Command found: {timelineHandler.HandlerType}"); - var o = new Orchestrator(); - var t = new Timeline { Id = Guid.NewGuid(), @@ -294,7 +287,7 @@ private string Handle(Message message) }; t.TimeLineHandlers.Add(timelineHandler); - o.RunCommand(t, timelineHandler); + Orchestrator.RunCommand(t, timelineHandler); var obj = JsonConvert.SerializeObject(timelineHandler); diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index 78091663..8873a704 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -17,7 +17,6 @@ namespace ghosts.client.linux.timelineManager public class Orchestrator { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private static DateTime _lastRead = DateTime.MinValue; private Thread MonitorThread { get; set; } public void Run() @@ -31,15 +30,13 @@ public void Run() //and creates a thread to execute instructions over that timeline if (timeline.Status == Timeline.TimelineStatus.Run) { - this.RunEx(timeline); + RunEx(timeline); } else { - if (this.MonitorThread != null) - { - this.MonitorThread.Abort(); - this.MonitorThread = null; - } + if (this.MonitorThread == null) return; + this.MonitorThread.Abort(); + this.MonitorThread = null; } } catch (Exception exc) @@ -48,7 +45,7 @@ public void Run() } } - public void StopTimeline(Guid timelineId) + public static void StopTimeline(Guid timelineId) { foreach (var threadJob in Program.ThreadJobs.Where(x=>x.TimelineId == timelineId)) { @@ -72,7 +69,7 @@ public void StopTimeline(Guid timelineId) } } - public void Stop() + public static void Stop() { foreach (var threadJob in Program.ThreadJobs) { @@ -96,7 +93,7 @@ public void Stop() } } - private void RunEx(Timeline timeline) + private static void RunEx(Timeline timeline) { foreach (var handler in timeline.TimeLineHandlers) { @@ -104,23 +101,26 @@ private void RunEx(Timeline timeline) } } - public void RunCommand(Timeline timeline, TimelineHandler handler) + public static void RunCommand(Timeline timeline, TimelineHandler handler) { ThreadLaunch(timeline, handler); } - private void ThreadLaunch(Timeline timeline, TimelineHandler handler) + private static void ThreadLaunch(Timeline timeline, TimelineHandler handler) { try { _log.Trace($"Attempting new thread for: {handler.HandlerType}"); Thread t = null; + // ReSharper disable once NotAccessedVariable object o; + // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault switch (handler.HandlerType) { case HandlerType.NpcSystem: - var npc = new NpcSystem(timeline, handler); + // ReSharper disable once RedundantAssignment + o = new NpcSystem(timeline, handler); break; case HandlerType.Command: t = new Thread(() => @@ -146,6 +146,8 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) o = new BrowserFirefox(handler); }); break; + default: + throw new ArgumentOutOfRangeException(); } if (t == null) From fd7e56428c6b79ad39f2d7e494c9d12c7c8d65e4 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Thu, 28 Oct 2021 10:26:15 -0400 Subject: [PATCH 18/42] adds requestfortimeline functionality to linux client --- .../Communications/Updates.cs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/ghosts.client.linux/Communications/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs index b30b796d..857a3b60 100644 --- a/src/ghosts.client.linux/Communications/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -88,6 +88,9 @@ private static void GetServerUpdates() // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault switch (update.Type) { + case UpdateClientConfig.UpdateType.RequestForTimeline: + PostCurrentTimeline(); + break; case UpdateClientConfig.UpdateType.Timeline: TimelineBuilder.SetLocalTimeline(update.Update.ToString()); break; @@ -144,6 +147,45 @@ private static void GetServerUpdates() Thread.Sleep(Program.Configuration.ClientUpdates.CycleSleep); } } + + private static void PostCurrentTimeline() + { + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + + var posturl = string.Empty; + + try + { + posturl = Program.Configuration.IdUrl.Replace("clientid", "clienttimeline"); + } + catch (Exception exc) + { + _log.Error("Can't get timeline posturl!"); + return; + } + + try + { + _log.Trace("posting timeline"); + + var payload = File.ReadAllText(ApplicationDetails.ConfigurationFiles.Timeline); + var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); // TODO? + + using (var client = WebClientBuilder.Build(machine)) + { + client.Headers[HttpRequestHeader.ContentType] = "application/json"; + client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + } + + _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); + } + catch (Exception e) + { + _log.Debug($"Problem posting timeline to server from { ApplicationDetails.ConfigurationFiles.Timeline } to { posturl }"); + _log.Error(e); + } + } private static void PostClientResults() { From 3c6d0721edf85fc88b9aba3f725c7a877ad1ba45 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Fri, 29 Oct 2021 10:33:29 -0400 Subject: [PATCH 19/42] adds support for multiple timelines being sent up the c2 --- src/Ghosts.Domain/Code/TimelineBuilder.cs | 36 +++++++-- src/Ghosts.Domain/Code/TimelineManager.cs | 73 +++++++++++++++++++ .../Communications/Updates.cs | 63 +++++++++++----- 3 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 src/Ghosts.Domain/Code/TimelineManager.cs diff --git a/src/Ghosts.Domain/Code/TimelineBuilder.cs b/src/Ghosts.Domain/Code/TimelineBuilder.cs index 2d77a789..0735a160 100644 --- a/src/Ghosts.Domain/Code/TimelineBuilder.cs +++ b/src/Ghosts.Domain/Code/TimelineBuilder.cs @@ -43,18 +43,38 @@ public static Timeline GetLocalTimeline() public static Timeline GetLocalTimeline(string path) { - var raw = File.ReadAllText(path); + try + { + var raw = File.ReadAllText(path); - var timeline = JsonConvert.DeserializeObject(raw); - if (timeline.Id == Guid.Empty) + var timeline = JsonConvert.DeserializeObject(raw); + if (timeline.Id == Guid.Empty) + { + timeline.Id = Guid.NewGuid(); + SetLocalTimeline(path, timeline); + } + + return timeline; + } + catch { - timeline.Id = Guid.NewGuid(); - SetLocalTimeline(path, timeline); + return null; } - - return timeline; } + public static string TimelineToString(Timeline timeline) + { + try + { + return JsonConvert.SerializeObject(timeline); + } + catch + { + // not a timeline? + return null; + } + } + /// /// Save to local disk /// @@ -64,7 +84,7 @@ public static void SetLocalTimeline(string timelineString) var timelineObject = JsonConvert.DeserializeObject(timelineString); SetLocalTimeline(timelineObject); } - + /// /// Save to local disk /// diff --git a/src/Ghosts.Domain/Code/TimelineManager.cs b/src/Ghosts.Domain/Code/TimelineManager.cs new file mode 100644 index 00000000..6ed54fb3 --- /dev/null +++ b/src/Ghosts.Domain/Code/TimelineManager.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Newtonsoft.Json.Linq; + +namespace Ghosts.Domain.Code +{ + public static class TimelineManager + { + public static IEnumerable GetLocalTimelines() + { + var timelines = new List + { + // get default timeline + TimelineBuilder.GetLocalTimeline() + }; + + var placesToLook = new List + { + // look for instance timelines + ApplicationDetails.InstanceDirectories.TimelineIn, + ApplicationDetails.InstanceDirectories.TimelineOut + }; + + foreach (var placeToLook in placesToLook) + { + var d = new DirectoryInfo(placeToLook); + + foreach (var file in d.GetFiles("*.*")) + { + // is this a timeline file? + try + { + var t = TimelineBuilder.GetLocalTimeline(file.FullName); + if (t != null) + { + timelines.Add(t); + } + } + catch + { + // ignored: not a timeline + } + } + } + + return timelines; + } + } + + public static class TimelineUpdateClientConfigManager + { + public static Guid GetConfigUpdateTimelineId(UpdateClientConfig config) + { + var result = new Guid(); + + try + { + if (config.Update != null) + { + var id = JObject.Parse(config.Update.ToString().ToLower())["id"].ToString(); + Guid.TryParse(id, out result); + } + } + catch + { + // ignored: not valid update config + } + + return result; + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Communications/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs index 857a3b60..e681b097 100644 --- a/src/ghosts.client.linux/Communications/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -89,7 +90,7 @@ private static void GetServerUpdates() switch (update.Type) { case UpdateClientConfig.UpdateType.RequestForTimeline: - PostCurrentTimeline(); + PostCurrentTimeline(update); break; case UpdateClientConfig.UpdateType.Timeline: TimelineBuilder.SetLocalTimeline(update.Update.ToString()); @@ -148,8 +149,31 @@ private static void GetServerUpdates() } } - private static void PostCurrentTimeline() + private static void PostCurrentTimeline(UpdateClientConfig update) { + // is the config for a specific timeline id? + var timelineId = TimelineUpdateClientConfigManager.GetConfigUpdateTimelineId(update); + + // get all timelines + var localTimelines = TimelineManager.GetLocalTimelines(); + + var timelines = localTimelines as Timeline[] ?? localTimelines.ToArray(); + if (timelineId != Guid.Empty) + { + foreach (var timeline in timelines) + { + if (timeline.Id == timelineId) + { + timelines = new List() + { + timeline + }.ToArray(); + break; + } + } + } + + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; var posturl = string.Empty; @@ -158,32 +182,35 @@ private static void PostCurrentTimeline() { posturl = Program.Configuration.IdUrl.Replace("clientid", "clienttimeline"); } - catch (Exception exc) + catch { _log.Error("Can't get timeline posturl!"); return; } - try + foreach (var timeline in timelines) { - _log.Trace("posting timeline"); + try + { + _log.Trace("posting timeline"); + + var payload = TimelineBuilder.TimelineToString(timeline); + var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); // TODO? - var payload = File.ReadAllText(ApplicationDetails.ConfigurationFiles.Timeline); - var machine = new ResultMachine(); - // GuestInfoVars.Load(machine); // TODO? + using (var client = WebClientBuilder.Build(machine)) + { + client.Headers[HttpRequestHeader.ContentType] = "application/json"; + client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + } - using (var client = WebClientBuilder.Build(machine)) + _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); + } + catch (Exception e) { - client.Headers[HttpRequestHeader.ContentType] = "application/json"; - client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + _log.Debug($"Problem posting timeline to server from {ApplicationDetails.ConfigurationFiles.Timeline} to {posturl}"); + _log.Error(e); } - - _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); - } - catch (Exception e) - { - _log.Debug($"Problem posting timeline to server from { ApplicationDetails.ConfigurationFiles.Timeline } to { posturl }"); - _log.Error(e); } } From 95d612bcd915be0d54152292b51ffbd78de2e7e9 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Fri, 29 Oct 2021 10:59:17 -0400 Subject: [PATCH 20/42] changes id to timelineid for clarity --- src/Ghosts.Domain/Code/TimelineManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Ghosts.Domain/Code/TimelineManager.cs b/src/Ghosts.Domain/Code/TimelineManager.cs index 6ed54fb3..0924f65c 100644 --- a/src/Ghosts.Domain/Code/TimelineManager.cs +++ b/src/Ghosts.Domain/Code/TimelineManager.cs @@ -58,7 +58,7 @@ public static Guid GetConfigUpdateTimelineId(UpdateClientConfig config) { if (config.Update != null) { - var id = JObject.Parse(config.Update.ToString().ToLower())["id"].ToString(); + var id = JObject.Parse(config.Update.ToString().ToLower())["timelineid"].ToString(); Guid.TryParse(id, out result); } } From 19fcf76b9cf4a447bdbc011c7afdd1f9e095c97b Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Fri, 29 Oct 2021 11:13:58 -0400 Subject: [PATCH 21/42] adds multiple timeline send to c2 for windows windows --- src/Ghosts.Client/Comms/Updates.cs | 64 +++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index fcc31d95..c542a58c 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -1,7 +1,9 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Text; using System.Threading; @@ -93,7 +95,7 @@ private static void GetServerUpdates() switch (update.Type) { case UpdateClientConfig.UpdateType.RequestForTimeline: - PostCurrentTimeline(); + PostCurrentTimeline(update); break; case UpdateClientConfig.UpdateType.Timeline: TimelineBuilder.SetLocalTimeline(update.Update.ToString()); @@ -154,8 +156,30 @@ private static void GetServerUpdates() } } - private static void PostCurrentTimeline() + private static void PostCurrentTimeline(UpdateClientConfig update) { + // is the config for a specific timeline id? + var timelineId = TimelineUpdateClientConfigManager.GetConfigUpdateTimelineId(update); + + // get all timelines + var localTimelines = Domain.Code.TimelineManager.GetLocalTimelines(); + + var timelines = localTimelines as Timeline[] ?? localTimelines.ToArray(); + if (timelineId != Guid.Empty) + { + foreach (var timeline in timelines) + { + if (timeline.Id == timelineId) + { + timelines = new List() + { + timeline + }.ToArray(); + break; + } + } + } + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; var posturl = string.Empty; @@ -164,32 +188,36 @@ private static void PostCurrentTimeline() { posturl = Program.Configuration.IdUrl.Replace("clientid", "clienttimeline"); } - catch (Exception exc) + catch { _log.Error("Can't get timeline posturl!"); return; } - try + foreach (var timeline in timelines) { - _log.Trace("posting timeline"); + try + { + _log.Trace("posting timeline"); - var payload = File.ReadAllText(ApplicationDetails.ConfigurationFiles.Timeline); - var machine = new ResultMachine(); - GuestInfoVars.Load(machine); + var payload = TimelineBuilder.TimelineToString(timeline); + var machine = new ResultMachine(); + GuestInfoVars.Load(machine); - using (var client = WebClientBuilder.Build(machine)) + using (var client = WebClientBuilder.Build(machine)) + { + client.Headers[HttpRequestHeader.ContentType] = "application/json"; + client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + } + + _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); + } + catch (Exception e) { - client.Headers[HttpRequestHeader.ContentType] = "application/json"; - client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + _log.Debug( + $"Problem posting timeline to server from {ApplicationDetails.ConfigurationFiles.Timeline} to {posturl}"); + _log.Error(e); } - - _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); - } - catch (Exception e) - { - _log.Debug($"Problem posting timeline to server from { ApplicationDetails.ConfigurationFiles.Timeline } to { posturl }"); - _log.Error(e); } } From 40ce7b9e039102ce10aaf1bb0ab7926e3fe69a2c Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Fri, 29 Oct 2021 12:07:16 -0400 Subject: [PATCH 22/42] adds x64 build profile --- src/Ghosts.Client/Ghosts.Client.csproj | 20 ++++++++++++++++++++ src/Ghosts.Domain/ghosts.domain.csproj | 1 + src/ghosts.windows.sln | 10 ++++++++++ 3 files changed, 31 insertions(+) diff --git a/src/Ghosts.Client/Ghosts.Client.csproj b/src/Ghosts.Client/Ghosts.Client.csproj index c51a99a2..b10cf282 100755 --- a/src/Ghosts.Client/Ghosts.Client.csproj +++ b/src/Ghosts.Client/Ghosts.Client.csproj @@ -68,6 +68,26 @@ false + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + ..\packages\CommandLineParser.2.8.0\lib\net461\CommandLine.dll diff --git a/src/Ghosts.Domain/ghosts.domain.csproj b/src/Ghosts.Domain/ghosts.domain.csproj index 7bea376d..43781883 100755 --- a/src/Ghosts.Domain/ghosts.domain.csproj +++ b/src/Ghosts.Domain/ghosts.domain.csproj @@ -6,6 +6,7 @@ 6.0.0.0 6.0.0.0 7.3 + AnyCPU;x64 diff --git a/src/ghosts.windows.sln b/src/ghosts.windows.sln index 96b49562..8b91216e 100755 --- a/src/ghosts.windows.sln +++ b/src/ghosts.windows.sln @@ -9,17 +9,27 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|x64.ActiveCfg = Debug|x64 + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|x64.Build.0 = Debug|x64 {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|Any CPU.Build.0 = Release|Any CPU + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|x64.ActiveCfg = Release|x64 + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|x64.Build.0 = Release|x64 {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|x64.ActiveCfg = Debug|x64 + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|x64.Build.0 = Debug|x64 {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|Any CPU.ActiveCfg = Release|Any CPU {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|Any CPU.Build.0 = Release|Any CPU + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|x64.ActiveCfg = Release|x64 + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From fb8124cafd41d346bab382f65624d3f2a74d9ea2 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 1 Nov 2021 10:00:36 -0400 Subject: [PATCH 23/42] updates timeline controller to timelines and adds ability to pull all timelines or just one by id --- src/Ghosts.Api/Controllers/ClientTimeline.cs | 4 +- ...neController.cs => TimelinesController.cs} | 38 +++++++++++++------ .../Services/MachineTimelineService.cs | 30 ++++++++++++--- src/Ghosts.Api/Startup.cs | 2 +- src/Ghosts.Domain/Code/TimelineBuilder.cs | 14 +++++++ 5 files changed, 67 insertions(+), 21 deletions(-) rename src/Ghosts.Api/Controllers/{TimelineController.cs => TimelinesController.cs} (65%) diff --git a/src/Ghosts.Api/Controllers/ClientTimeline.cs b/src/Ghosts.Api/Controllers/ClientTimeline.cs index 6961dae0..93570de2 100644 --- a/src/Ghosts.Api/Controllers/ClientTimeline.cs +++ b/src/Ghosts.Api/Controllers/ClientTimeline.cs @@ -22,10 +22,10 @@ namespace ghosts.api.Controllers public class ClientTimelineController : Controller { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private readonly IMachineTimelineService _service; + private readonly IMachineTimelinesService _service; private readonly IMachineService _machineService; - public ClientTimelineController(IMachineTimelineService service, IMachineService machineService) + public ClientTimelineController(IMachineTimelinesService service, IMachineService machineService) { _service = service; _machineService = machineService; diff --git a/src/Ghosts.Api/Controllers/TimelineController.cs b/src/Ghosts.Api/Controllers/TimelinesController.cs similarity index 65% rename from src/Ghosts.Api/Controllers/TimelineController.cs rename to src/Ghosts.Api/Controllers/TimelinesController.cs index cf12a6c6..284dd3b5 100644 --- a/src/Ghosts.Api/Controllers/TimelineController.cs +++ b/src/Ghosts.Api/Controllers/TimelinesController.cs @@ -1,7 +1,6 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.Net; using System.Threading; using System.Threading.Tasks; using Ghosts.Api.Models; @@ -15,29 +14,44 @@ namespace ghosts.api.Controllers /// /// Get or update a machine timeline via the API /// - public class TimelineController : Controller + public class TimelinesController : Controller { private readonly ITimelineService _timelineService; - private readonly IMachineTimelineService _machineTimelineService; + private readonly IMachineTimelinesService _machineTimelinesService; - public TimelineController(ITimelineService timelineService, IMachineTimelineService machineTimelineService) + public TimelinesController(ITimelineService timelineService, IMachineTimelinesService machineTimelinesService) { _timelineService = timelineService; - _machineTimelineService = machineTimelineService; + _machineTimelinesService = machineTimelinesService; } /// - /// This returns the timeline for a requested machine. If the timeline is not available, + /// This returns all timelines for a requested machine. If all or a specific timeline is not available, + /// a MachineUpdate request can be made to retrieve the machine timelines + /// + /// Machine Guid + /// Cancellation token + /// MachineTimelines + [ProducesResponseType(typeof(MachineTimeline), 200)] + [HttpGet("timelines/{machineId}")] + public async Task Timeline([FromRoute] Guid machineId, CancellationToken ct) + { + return Ok(await _machineTimelinesService.GetByMachineIdAsync(machineId, ct)); + } + + /// + /// This returns a specific timeline for a requested machine. If the timeline is not available, /// a MachineUpdate request can be made to retrieve the machine timeline /// /// Machine Guid + /// /// Timeline Id Guid /// Cancellation token /// MachineTimeline [ProducesResponseType(typeof(MachineTimeline), 200)] - [HttpGet("timeline/{machineId}")] - public async Task Timeline([FromRoute] Guid machineId, CancellationToken ct) + [HttpGet("timelines/{machineId}/{timelineId}")] + public async Task TimelineById([FromRoute] Guid machineId, [FromRoute] Guid timelineId, CancellationToken ct) { - return Ok(await _machineTimelineService.GetByMachineIdAsync(machineId, ct)); + return Ok(await _machineTimelinesService.GetByMachineIdAndTimelineIdAsync(machineId, timelineId, ct)); } /// @@ -46,7 +60,7 @@ public async Task Timeline([FromRoute] Guid machineId, Cancellati /// The update to send /// Cancellation token /// 204 No content - [HttpPost("timeline")] + [HttpPost("timelines")] // [ProducesResponseType(typeof(Task), (int) HttpStatusCode.NoContent)] Swagger hates this https://stackoverflow.com/questions/35605427/swagger-ui-freezes-after-api-fetch-and-browser-crashes [SwaggerOperation(OperationId = "createTimeline")] public async Task Timeline([FromBody] MachineUpdateViewModel machineUpdate, CancellationToken ct) @@ -55,7 +69,7 @@ public async Task Timeline([FromBody] MachineUpdateViewModel mach return NoContent(); } - [HttpPost("timeline/{machineId}/stop/{timelineId}")] + [HttpPost("timelines/{machineId}/{timelineId}/stop")] [SwaggerOperation(OperationId = "stopTimeline")] public async Task Timeline([FromRoute] Guid machineId, [FromRoute] Guid timelineId, CancellationToken ct) { @@ -70,7 +84,7 @@ public async Task Timeline([FromRoute] Guid machineId, [FromRoute /// The update to send /// Cancellation token /// 204 No content - [HttpPost("timeline/bygroup/{groupId}")] + [HttpPost("timelines/bygroup/{groupId}")] // [ProducesResponseType(typeof(Task), (int) HttpStatusCode.NoContent)] Swagger hates this [SwaggerOperation(OperationId = "createTimelineForGroup")] public async Task GroupTimeline([FromRoute] int groupId, [FromBody] MachineUpdateViewModel machineUpdate, CancellationToken ct) diff --git a/src/Ghosts.Api/Services/MachineTimelineService.cs b/src/Ghosts.Api/Services/MachineTimelineService.cs index f8d758e1..9c93493e 100755 --- a/src/Ghosts.Api/Services/MachineTimelineService.cs +++ b/src/Ghosts.Api/Services/MachineTimelineService.cs @@ -1,37 +1,55 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Ghosts.Api.Infrastructure.Data; using Ghosts.Api.Models; using Ghosts.Domain; +using Ghosts.Domain.Code; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using NLog; namespace Ghosts.Api.Services { - public interface IMachineTimelineService + public interface IMachineTimelinesService { - Task GetByMachineIdAsync(Guid id, CancellationToken ct); + Task> GetByMachineIdAsync(Guid id, CancellationToken ct); + Task GetByMachineIdAndTimelineIdAsync(Guid id, Guid timelineId, CancellationToken ct); Task CreateAsync(Machine model, Timeline timeline, CancellationToken ct); Task DeleteByMachineIdAsync(Guid model, CancellationToken ct); } - public class MachineTimelineService : IMachineTimelineService + public class MachineTimelinesService : IMachineTimelinesService { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private readonly ApplicationDbContext _context; - public MachineTimelineService(ApplicationDbContext context) + public MachineTimelinesService(ApplicationDbContext context) { _context = context; } - public async Task GetByMachineIdAsync(Guid id, CancellationToken ct) + public async Task> GetByMachineIdAsync(Guid id, CancellationToken ct) { - return await _context.MachineTimelines.FirstOrDefaultAsync(x => x.MachineId == id, cancellationToken: ct); + return _context.MachineTimelines.Where(x => x.MachineId == id); + } + + public async Task GetByMachineIdAndTimelineIdAsync(Guid id, Guid timelineId, CancellationToken ct) + { + var timelines = _context.MachineTimelines.Where(x => x.MachineId == id); + foreach (var timeline in timelines) + { + var t = TimelineBuilder.StringToTimeline(timeline.Timeline); + if (t.Id == timelineId) + return timeline; + } + + return null; } public async Task CreateAsync(Machine model, Timeline timeline, CancellationToken ct) diff --git a/src/Ghosts.Api/Startup.cs b/src/Ghosts.Api/Startup.cs index 71595e96..711b8448 100755 --- a/src/Ghosts.Api/Startup.cs +++ b/src/Ghosts.Api/Startup.cs @@ -67,7 +67,7 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddSingleton(); diff --git a/src/Ghosts.Domain/Code/TimelineBuilder.cs b/src/Ghosts.Domain/Code/TimelineBuilder.cs index 0735a160..668ab118 100644 --- a/src/Ghosts.Domain/Code/TimelineBuilder.cs +++ b/src/Ghosts.Domain/Code/TimelineBuilder.cs @@ -62,6 +62,20 @@ public static Timeline GetLocalTimeline(string path) } } + public static Timeline StringToTimeline(string raw) + { + try + { + var timeline = JsonConvert.DeserializeObject(raw); + return timeline; + } + catch + { + _log.Debug($"String is not a timeline: {raw}"); + return null; + } + } + public static string TimelineToString(Timeline timeline) { try From 373abd4432c1cd2d8e4bb13ef391778a1f1772da Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 1 Nov 2021 11:08:31 -0400 Subject: [PATCH 24/42] updates async methods --- src/Ghosts.Api/Services/MachineTimelineService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ghosts.Api/Services/MachineTimelineService.cs b/src/Ghosts.Api/Services/MachineTimelineService.cs index 9c93493e..588eddfd 100755 --- a/src/Ghosts.Api/Services/MachineTimelineService.cs +++ b/src/Ghosts.Api/Services/MachineTimelineService.cs @@ -36,12 +36,12 @@ public MachineTimelinesService(ApplicationDbContext context) public async Task> GetByMachineIdAsync(Guid id, CancellationToken ct) { - return _context.MachineTimelines.Where(x => x.MachineId == id); + return await _context.MachineTimelines.Where(x => x.MachineId == id).ToListAsync(ct); } public async Task GetByMachineIdAndTimelineIdAsync(Guid id, Guid timelineId, CancellationToken ct) { - var timelines = _context.MachineTimelines.Where(x => x.MachineId == id); + var timelines = await _context.MachineTimelines.Where(x => x.MachineId == id).ToListAsync(ct); foreach (var timeline in timelines) { var t = TimelineBuilder.StringToTimeline(timeline.Timeline); From dcebb2b05557fd2344c5578e94c91e392bced237 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Tue, 2 Nov 2021 10:05:24 -0400 Subject: [PATCH 25/42] adds local survey for linux clients (limited) and api endpoint for viewing --- .../Controllers/SurveysController.cs | 34 ++ src/Ghosts.Api/Services/SurveyService.cs | 57 +++ src/Ghosts.Api/Startup.cs | 1 + src/Ghosts.Domain/Code/Jitter.cs | 11 + .../Communications/Updates.cs | 61 +++ src/ghosts.client.linux/Program.cs | 13 +- .../Survey/SurveyManager.cs | 391 ++++++++++++++++++ .../config/application.json | 2 +- 8 files changed, 568 insertions(+), 2 deletions(-) create mode 100644 src/Ghosts.Api/Controllers/SurveysController.cs create mode 100644 src/Ghosts.Api/Services/SurveyService.cs create mode 100644 src/ghosts.client.linux/Survey/SurveyManager.cs diff --git a/src/Ghosts.Api/Controllers/SurveysController.cs b/src/Ghosts.Api/Controllers/SurveysController.cs new file mode 100644 index 00000000..14388f10 --- /dev/null +++ b/src/Ghosts.Api/Controllers/SurveysController.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Services; +using Ghosts.Domain.Messages.MesssagesForServer; +using Microsoft.AspNetCore.Mvc; + +namespace ghosts.api.Controllers +{ + public class SurveysController : Controller + { + private readonly ISurveyService _surveyService; + + public SurveysController(ISurveyService surveyService) + { + _surveyService = surveyService; + } + + [ProducesResponseType(typeof(Survey), 200)] + [HttpGet("surveys/{machineId}")] + public async Task Survey([FromRoute] Guid machineId, CancellationToken ct) + { + return Ok(await _surveyService.GetLatestAsync(machineId, ct)); + } + + [ProducesResponseType(typeof(IEnumerable), 200)] + [HttpGet("surveys/{machineId}/all")] + public async Task SurveyAll([FromRoute] Guid machineId, CancellationToken ct) + { + return Ok(await _surveyService.GetAllAsync(machineId, ct)); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Services/SurveyService.cs b/src/Ghosts.Api/Services/SurveyService.cs new file mode 100644 index 00000000..5f7aff73 --- /dev/null +++ b/src/Ghosts.Api/Services/SurveyService.cs @@ -0,0 +1,57 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Infrastructure.Data; +using Ghosts.Domain.Messages.MesssagesForServer; +using Microsoft.EntityFrameworkCore; + +namespace Ghosts.Api.Services +{ + public interface ISurveyService + { + Task GetLatestAsync(Guid machineId, CancellationToken ct); + Task> GetAllAsync(Guid machineId, CancellationToken ct); + } + + public class SurveyService : ISurveyService + { + private readonly ApplicationDbContext _context; + + public SurveyService(ApplicationDbContext context) + { + _context = context; + } + + public async Task GetLatestAsync(Guid machineId, CancellationToken ct) + { + return await _context.Surveys + .Include(x=>x.Drives) + .Include("Interfaces.Bindings") + .Include(x=>x.Ports) + .Include(x=>x.Processes) + .Include(x=>x.EventLogs) + .Include(x=>x.LocalUsers) + .Where(x=>x.MachineId == machineId) + .OrderByDescending(x => x.Created) + .FirstOrDefaultAsync(ct); + } + + public async Task> GetAllAsync(Guid machineId, CancellationToken ct) + { + return await _context.Surveys + .Include(x=>x.Drives) + .Include("Interfaces.Bindings") + .Include(x=>x.Ports) + .Include(x=>x.Processes) + .Include(x=>x.EventLogs) + .Include(x=>x.LocalUsers) + .Where(x=>x.MachineId == machineId) + .OrderByDescending(x => x.Created) + .ToArrayAsync(ct); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Startup.cs b/src/Ghosts.Api/Startup.cs index 711b8448..57c579ae 100755 --- a/src/Ghosts.Api/Startup.cs +++ b/src/Ghosts.Api/Startup.cs @@ -69,6 +69,7 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Ghosts.Domain/Code/Jitter.cs b/src/Ghosts.Domain/Code/Jitter.cs index bb38e4d1..0a2dc035 100755 --- a/src/Ghosts.Domain/Code/Jitter.cs +++ b/src/Ghosts.Domain/Code/Jitter.cs @@ -41,5 +41,16 @@ public static int Randomize(int baseSleepValue, int lowJitter, int highJitter) return newSleepValue; } + + public static int Basic(int baseSleep) + { + //sleep with jitter + var sleep = baseSleep; + var r = new Random().Next(-999, 1999); + sleep += r; + if (sleep < 0) + sleep = 1; + return sleep; + } } } \ No newline at end of file diff --git a/src/ghosts.client.linux/Communications/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs index e681b097..ab7d6a03 100644 --- a/src/ghosts.client.linux/Communications/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -311,5 +311,66 @@ private static void PostResults(string fileName, ResultMachine machine, string p _log.Trace($"{DateTime.Now} - {fileName} posted to server successfully"); } + + internal static void PostSurvey() + { + // ignore all certs + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + + var posturl = string.Empty; + + try + { + posturl = Program.Configuration.Survey.PostUrl; + } + catch (Exception exc) + { + _log.Error("Can't get survey posturl!"); + return; + } + + try + { + _log.Trace("posting survey"); + + Thread.Sleep(Jitter.Basic(100)); + + if (!File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) + return; + + var survey = JsonConvert.DeserializeObject(File.ReadAllText(ApplicationDetails.InstanceFiles.SurveyResults)); + + var payload = JsonConvert.SerializeObject(survey); + + var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); + + if (Program.Configuration.Survey.IsSecure) + { + payload = Crypto.EncryptStringAes(payload, machine.Name); + payload = Base64Encoder.Base64Encode(payload); + + var p = new EncryptedPayload(); + p.Payload = payload; + + payload = JsonConvert.SerializeObject(p); + } + + using (var client = WebClientBuilder.Build(machine)) + { + client.Headers[HttpRequestHeader.ContentType] = "application/json"; + client.UploadString(posturl, payload); + } + + _log.Trace($"{DateTime.Now} - survey posted to server successfully"); + + File.Delete(ApplicationDetails.InstanceFiles.SurveyResults); + } + catch (Exception e) + { + _log.Debug($"Problem posting logs to server from { ApplicationDetails.InstanceFiles.SurveyResults } to { Program.Configuration.Survey.PostUrl }"); + _log.Error(e); + } + } } } diff --git a/src/ghosts.client.linux/Program.cs b/src/ghosts.client.linux/Program.cs index f8508313..bb5d8c34 100755 --- a/src/ghosts.client.linux/Program.cs +++ b/src/ghosts.client.linux/Program.cs @@ -82,7 +82,18 @@ private static void Run(IEnumerable args) //connect to command server for updates and sending logs Updates.Run(); - //linux clients do not perform local survey + //local survey gathers information such as drives, accounts, logs, etc. + if (Configuration.Survey.IsEnabled) + { + try + { + Survey.SurveyManager.Run(); + } + catch (Exception exc) + { + _log.Error(exc); + } + } if (Configuration.HealthIsEnabled) { diff --git a/src/ghosts.client.linux/Survey/SurveyManager.cs b/src/ghosts.client.linux/Survey/SurveyManager.cs new file mode 100644 index 00000000..f139e239 --- /dev/null +++ b/src/ghosts.client.linux/Survey/SurveyManager.cs @@ -0,0 +1,391 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net.NetworkInformation; +using System.Text.RegularExpressions; +using System.Threading; +using ghosts.client.linux.Communications; +using Ghosts.Domain.Code; +using Newtonsoft.Json; +using NLog; + +namespace ghosts.client.linux.Survey +{ + public static class SurveyManager + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public static void Run() + { + try + { + new Thread(() => + { + Thread.CurrentThread.IsBackground = true; + RunEx(); + + }).Start(); + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private static void RunEx() + { + while (true) + { + //has file already been generated + if (Program.Configuration.Survey.Frequency.Equals("once", + StringComparison.InvariantCultureIgnoreCase) && + File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) + break; + + + _log.Trace("Running new survey..."); + try + { + var s = new SurveyResult + { + Survey = + { + Created = DateTime.UtcNow + } + }; + + if (Guid.TryParse(CheckId.Id, out var g)) + s.Survey.MachineId = g; + + s.LoadAll(); + + var f = new FileInfo(ApplicationDetails.InstanceFiles.SurveyResults); + if (f.Directory is { Exists: false }) + { + Directory.CreateDirectory(f.DirectoryName); + } + + try + { + if (File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) + File.Delete(ApplicationDetails.InstanceFiles.SurveyResults); + } + catch (Exception e) + { + _log.Trace(e); + } + + var formatting = Formatting.Indented; + if (Program.Configuration.Survey.OutputFormat.Equals("none", + StringComparison.InvariantCultureIgnoreCase)) + formatting = Formatting.None; + + using (var file = File.CreateText(ApplicationDetails.InstanceFiles.SurveyResults)) + { + var serializer = new JsonSerializer + { + Formatting = formatting, + NullValueHandling = NullValueHandling.Ignore + }; + serializer.Serialize(file, s.Survey); + } + Updates.PostSurvey(); + } + catch (Exception e) + { + _log.Trace(e); + } + + Thread.Sleep(Program.Configuration.Survey.CycleSleepMinutes * 60000); + } + } + } + + public class SurveyResult + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public Ghosts.Domain.Messages.MesssagesForServer.Survey Survey { get; set; } + + public SurveyResult() + { + this.Survey = new Ghosts.Domain.Messages.MesssagesForServer.Survey + { + Uptime = GetUptime() + }; + } + + public void LoadAll() + { + var random = new Random(); + + this.Survey.Ports = GetNetStatPorts(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.Interfaces = GetInterfaces(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.LocalUsers = GetLocalAccounts(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.Drives = this.GetDriveInfo(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.Processes = this.GetProcesses(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.EventLogs = GetEventLogs(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + foreach (var item in this.Survey.EventLogs) + { + item.Entries = new List(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 5000)); + foreach (var o in this.GetEventLogEntries(item.Name)) + item.Entries.Add(o); + } + } + + public static TimeSpan GetUptime() + { + try + { + return TimeSpan.FromMilliseconds(Stopwatch.GetTimestamp()); + } + catch (Exception e) + { + _log.Trace(e); + } + return new TimeSpan(); + } + + public static List GetNetStatPorts() + { + var ports = new List(); + + try + { + using var p = new Process(); + var ps = new ProcessStartInfo + { + Arguments = "-a -n -o", + FileName = "netstat.exe", + UseShellExecute = false, + WindowStyle = ProcessWindowStyle.Hidden, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + p.StartInfo = ps; + p.Start(); + + var stdOutput = p.StandardOutput; + var stdError = p.StandardError; + + var content = stdOutput.ReadToEnd() + stdError.ReadToEnd(); + var exitStatus = p.ExitCode.ToString(); + + if (exitStatus != "0") + { + // Command Errored. Handle Here If Need Be + } + + //Get The Rows + var rows = Regex.Split(content, "\r\n"); + foreach (var row in rows) + { + //Split it + var tokens = Regex.Split(row, "\\s+"); + if (tokens.Length > 4 && (tokens[1].Equals("UDP") || tokens[1].Equals("TCP"))) + { + var localAddress = Regex.Replace(tokens[2], @"\[(.*?)\]", "1.1.1.1"); + var foreignAddress = Regex.Replace(tokens[3], @"\[(.*?)\]", "1.1.1.1"); + ports.Add(new Ghosts.Domain.Messages.MesssagesForServer.Survey.Port + { + LocalAddress = localAddress.Split(':')[0], + LocalPort = localAddress.Split(':')[1], + ForeignAddress = foreignAddress.Split(':')[0], + ForeignPort = foreignAddress.Split(':')[1], + State = tokens[1] == "UDP" ? null : tokens[4], + PID = tokens[1] == "UDP" ? Convert.ToInt16(tokens[4]) : Convert.ToInt16(tokens[5]), + Protocol = localAddress.Contains("1.1.1.1") ? String.Format("{0}v6", tokens[1]) : String.Format("{0}v4", tokens[1]), + Process = tokens[1] == "UDP" ? LookupProcess(Convert.ToInt16(tokens[4])) : LookupProcess(Convert.ToInt16(tokens[5])) + }); + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + + return ports; + } + + public static string LookupProcess(int pid) + { + string procName; + try { procName = Process.GetProcessById(pid).ProcessName; } + catch (Exception) { procName = "-"; } + return procName; + } + + public List GetInterfaces() + { + var results = new List(); + + try + { + var adapters = NetworkInterface.GetAllNetworkInterfaces(); + var adapterCount = 0; + foreach (var adapter in adapters) + { + adapterCount += 1; + var iFace = new Ghosts.Domain.Messages.MesssagesForServer.Survey.Interface(); + iFace.Name = adapter.Name; + iFace.Id = adapterCount; + + var physicalAddress = adapter.GetPhysicalAddress(); + var ipProperties = adapter.GetIPProperties(); + if (ipProperties.UnicastAddresses != null) + { + foreach (var address in ipProperties.UnicastAddresses) + { + var bind = new Ghosts.Domain.Messages.MesssagesForServer.Survey.Interface.InterfaceBinding(); + bind.Type = adapter.NetworkInterfaceType.ToString(); + bind.InternetAddress = address.Address.ToString(); + bind.PhysicalAddress = physicalAddress.ToString(); + iFace.Bindings.Add(bind); + } + } + + results.Add(iFace); + } + } + catch (Exception e) + { + _log.Trace(e); + } + + return results; + } + + public List GetLocalAccounts() + { + var users = new List(); + try + { + + } + catch (Exception e) + { + _log.Trace(e); + } + return users; + } + + public List GetDriveInfo() + { + var results = new List(); + try + { + var allDrives = System.IO.DriveInfo.GetDrives(); + foreach (var drive in allDrives) + { + var result = new Ghosts.Domain.Messages.MesssagesForServer.Survey.DriveInfo(); + result.AvailableFreeSpace = drive.AvailableFreeSpace; + result.DriveFormat = drive.DriveFormat; + result.DriveType = drive.DriveType.ToString(); + result.IsReady = drive.IsReady; + result.Name = drive.Name; + result.RootDirectory = drive.RootDirectory.ToString(); + result.TotalFreeSpace = drive.TotalFreeSpace; + result.TotalSize = drive.TotalSize; + result.VolumeLabel = drive.VolumeLabel; + results.Add(result); + } + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + + public List GetProcesses() + { + var results = new List(); + try + { + foreach (var item in Process.GetProcesses()) + { + var result = new Ghosts.Domain.Messages.MesssagesForServer.Survey.LocalProcess(); + + if (!string.IsNullOrEmpty(item.MainWindowTitle)) + result.MainWindowTitle = item.MainWindowTitle; + if (!string.IsNullOrEmpty(item.ProcessName)) + result.ProcessName = item.ProcessName; + try { result.StartTime = item.StartTime; } + catch + { + // ignore + } + if (!string.IsNullOrEmpty(item.StartInfo.FileName)) + result.FileName = item.StartInfo.FileName; + if (!string.IsNullOrEmpty(item.StartInfo.UserName)) + result.Owner = item.StartInfo.UserName; + if (string.IsNullOrEmpty(result.Owner)) + { + // how to get owners on linux? + } + if (!results.Exists(x => x.ProcessName == item.ProcessName)) + results.Add(result); + } + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + + public List GetEventLogs() + { + var results = new List(); + try + { + + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + + public List GetEventLogEntries(string logName) + { + var results = new List(); + try + { + + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/config/application.json b/src/ghosts.client.linux/config/application.json index a2070aa1..69d942c7 100755 --- a/src/ghosts.client.linux/config/application.json +++ b/src/ghosts.client.linux/config/application.json @@ -16,7 +16,7 @@ "IsEnabled": true, "IsSecure": false, "Frequency": "once", - "CycleSleepMinutes": 720, + "CycleSleepMinutes": 5, "OutputFormat": "indent", "PostUrl": "http://ghosts-api:52388/api/clientsurvey" }, From 854f28aadcf3240caf01f89a2ff7b0a0dfb39510 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 8 Nov 2021 18:32:38 -0500 Subject: [PATCH 26/42] updates projects to dotnet 6.0 LTS, except for windows --- src/Dockerfile-api | 8 ++--- src/Ghosts.Api/docker-compose.yml | 2 +- src/Ghosts.Api/ghosts.api.csproj | 2 +- src/Ghosts.Domain/ghosts.domain.csproj | 4 +-- .../Communications/Updates.cs | 2 +- src/ghosts.client.linux/Health/Check.cs | 2 +- .../TimelineManager/Orchestrator.cs | 10 +++--- src/ghosts.client.linux/dockerfile-alpine | 14 ++------ src/ghosts.client.linux/dockerfile-kali | 35 ------------------- .../ghosts.client.linux.csproj | 2 +- 10 files changed, 18 insertions(+), 63 deletions(-) delete mode 100644 src/ghosts.client.linux/dockerfile-kali diff --git a/src/Dockerfile-api b/src/Dockerfile-api index 2bcdfffb..fe885e48 100644 --- a/src/Dockerfile-api +++ b/src/Dockerfile-api @@ -5,7 +5,7 @@ # #multi-stage target: dev # -FROM mcr.microsoft.com/dotnet/sdk:5.0-alpine AS dev +FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS dev ENV ASPNETCORE_URLS=http://*:5000 \ ASPNETCORE_ENVIRONMENT=DEVELOPMENT @@ -20,14 +20,14 @@ CMD ["dotnet", "run"] # #multi-stage target: prod # -FROM mcr.microsoft.com/dotnet/aspnet:5.0-alpine AS prod +FROM mcr.microsoft.com/dotnet/aspnet:6.0-alpine AS prod ARG commit ENV COMMIT=$commit COPY --from=dev /app/dist /app WORKDIR /app -ENV GHOSTS_VERSION=5.0.0.0 -ENV GHOSTS_API_VERSION=v5 +ENV GHOSTS_VERSION=6.0.0.0 +ENV GHOSTS_API_VERSION=v6 ENV ASPNETCORE_URLS=http://*:5000 EXPOSE 5000 diff --git a/src/Ghosts.Api/docker-compose.yml b/src/Ghosts.Api/docker-compose.yml index 25e309b6..318fe1f9 100644 --- a/src/Ghosts.Api/docker-compose.yml +++ b/src/Ghosts.Api/docker-compose.yml @@ -38,7 +38,7 @@ services: depends_on: - postgres ports: - - '5000:5000' + - '52388:5000' networks: - ghosts-network restart: always diff --git a/src/Ghosts.Api/ghosts.api.csproj b/src/Ghosts.Api/ghosts.api.csproj index 8d53f45c..21c338f4 100644 --- a/src/Ghosts.Api/ghosts.api.csproj +++ b/src/Ghosts.Api/ghosts.api.csproj @@ -1,7 +1,7 @@  - net5.0 + net6.0 ghosts.api 6.0.0.0 6.0.0.0 diff --git a/src/Ghosts.Domain/ghosts.domain.csproj b/src/Ghosts.Domain/ghosts.domain.csproj index 43781883..def1dde1 100755 --- a/src/Ghosts.Domain/ghosts.domain.csproj +++ b/src/Ghosts.Domain/ghosts.domain.csproj @@ -10,9 +10,7 @@ - - Always - + diff --git a/src/ghosts.client.linux/Communications/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs index ab7d6a03..15950206 100644 --- a/src/ghosts.client.linux/Communications/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -323,7 +323,7 @@ internal static void PostSurvey() { posturl = Program.Configuration.Survey.PostUrl; } - catch (Exception exc) + catch { _log.Error("Can't get survey posturl!"); return; diff --git a/src/ghosts.client.linux/Health/Check.cs b/src/ghosts.client.linux/Health/Check.cs index 070ec65b..3b6f029b 100644 --- a/src/ghosts.client.linux/Health/Check.cs +++ b/src/ghosts.client.linux/Health/Check.cs @@ -62,7 +62,7 @@ private void Shutdown() if (this._threads == null) return; foreach (var thread in this._threads) { - thread.Abort(null); + thread.Interrupt(); } } diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index 8873a704..c027d348 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -35,7 +35,7 @@ public void Run() else { if (this.MonitorThread == null) return; - this.MonitorThread.Abort(); + this.MonitorThread.Interrupt(); this.MonitorThread = null; } } @@ -51,7 +51,7 @@ public static void StopTimeline(Guid timelineId) { try { - threadJob.Thread.Abort(null); + threadJob.Thread.Interrupt(); } catch (Exception e) { @@ -60,7 +60,7 @@ public static void StopTimeline(Guid timelineId) try { - threadJob.Thread.Join(); + threadJob.Thread.Interrupt(); } catch (Exception e) { @@ -75,7 +75,7 @@ public static void Stop() { try { - threadJob.Thread.Abort(null); + threadJob.Thread.Interrupt(); } catch (Exception e) { @@ -84,7 +84,7 @@ public static void Stop() try { - threadJob.Thread.Join(); + threadJob.Thread.Interrupt(); } catch (Exception e) { diff --git a/src/ghosts.client.linux/dockerfile-alpine b/src/ghosts.client.linux/dockerfile-alpine index 1b01eea6..8223b552 100755 --- a/src/ghosts.client.linux/dockerfile-alpine +++ b/src/ghosts.client.linux/dockerfile-alpine @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:latest AS build +FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS dev # copy csproj and restore as distinct layers COPY ghosts.linux.sln ./app/ @@ -15,15 +15,7 @@ COPY Ghosts.Domain/ ./Ghosts.Domain/ WORKDIR /app/ghosts.client.linux/ RUN dotnet publish -c Release -o out -FROM mcr.microsoft.com/dotnet/core/runtime:3.1-alpine - -# Install ASP.NET Core -RUN aspnetcore_version=3.1.4 \ - && wget -O aspnetcore.tar.gz https://dotnetcli.azureedge.net/dotnet/aspnetcore/Runtime/$aspnetcore_version/aspnetcore-runtime-$aspnetcore_version-linux-musl-x64.tar.gz \ - && aspnetcore_sha512='f60e9226a5b399470479fd6fdebd03442b0440128be1090adcbe473dba46a3e7a57a9e59b4abff96214e0dd0b1123c67fe764b74c61de1cb35c8b8ac45767eb9' \ - && echo "$aspnetcore_sha512 aspnetcore.tar.gz" | sha512sum -c - \ - && tar -ozxf aspnetcore.tar.gz -C /usr/share/dotnet ./shared/Microsoft.AspNetCore.App \ - && rm aspnetcore.tar.gz +FROM mcr.microsoft.com/dotnet/aspnet:6.0-alpine AS prod RUN apk add curl && \ apk add nmap && \ @@ -38,7 +30,7 @@ RUN apk add curl && \ cd sqlmap/ # python sqlmap.py --version -COPY --from=build /app/ghosts.client.linux/out ./app +COPY --from=dev /app/ghosts.client.linux/out ./app WORKDIR /app/ ENV ASPNETCORE_URLS=http://+:5000 diff --git a/src/ghosts.client.linux/dockerfile-kali b/src/ghosts.client.linux/dockerfile-kali deleted file mode 100644 index 1ef930c2..00000000 --- a/src/ghosts.client.linux/dockerfile-kali +++ /dev/null @@ -1,35 +0,0 @@ -FROM booyaabes/kali-linux-full:latest - -# install dotnetcore -RUN sudo apt-get --assume-yes install -y gpg && \ - wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg && \ - sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ && \ - wget -q https://packages.microsoft.com/config/ubuntu/19.04/prod.list && \ - sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list && \ - sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg && \ - sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list && \ - apt-get update && \ - sudo apt-get --assume-yes install -y apt-transport-https && \ - sudo apt-get update && \ - sudo apt-get --assume-yes install aspnetcore-runtime-3.1 - -WORKDIR /opt/ghosts -COPY bin/publish . -ENV ASPNETCORE_URLS=http://+:5000 -EXPOSE 5000/tcp - -# if we wanted to do other commands at startup in the future -# CMD [ "/usr/local/bin/start.sh" ] - -CMD ["dotnet", "./ghosts.client.linux.dll"] - - - -# dotnet publish -c Release -o bin/publish -# docker build . -f dockerfile-kali -t ghosts/staypuft - -# docker swarm init -# docker stack deploy ghosts-staypuft --compose-file docker-compose.yml -# Docker swarm leave --force - -# docker run -d -p 7000:5000 --name ghosts-staypuft ghosts/staypuft diff --git a/src/ghosts.client.linux/ghosts.client.linux.csproj b/src/ghosts.client.linux/ghosts.client.linux.csproj index 1f6028cf..d9baf23c 100755 --- a/src/ghosts.client.linux/ghosts.client.linux.csproj +++ b/src/ghosts.client.linux/ghosts.client.linux.csproj @@ -2,7 +2,6 @@ Exe - netcoreapp3.1 win7-x64;ubuntu.16.10-x64;osx.10.12-x64 6.0.0.0 @@ -17,6 +16,7 @@ GHOSTS NPC Orchestration Platform - please email ddupdyke@sei.cmu.edu with bugs/requests/other Carnegie Mellon University 2017 NU1701 + net6.0;netcoreapp3.1 From f58832b69f52e2561e62373944d401d4aebaefcb Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 8 Nov 2021 18:35:58 -0500 Subject: [PATCH 27/42] updates projects to dotnet 6.0 LTS, except for windows --- src/ghosts.client.linux/TimelineManager/Orchestrator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index c027d348..8634aa37 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -60,7 +60,7 @@ public static void StopTimeline(Guid timelineId) try { - threadJob.Thread.Interrupt(); + threadJob.Thread.Join(); } catch (Exception e) { From c006ce479d49be4d4f7dfec69ac1f7e27d71d504 Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Mon, 22 Nov 2021 17:59:06 -0500 Subject: [PATCH 28/42] fixes various windows client bugs, intros pdf files --- src/Ghosts.Client/Comms/CheckId.cs | 45 ++--- src/Ghosts.Client/Comms/Updates.cs | 2 +- .../Handlers/BaseBrowserHandler.cs | 1 + src/Ghosts.Client/Handlers/BrowserChrome.cs | 14 +- src/Ghosts.Client/Handlers/BrowserFirefox.cs | 13 +- src/Ghosts.Client/Handlers/Excel.cs | 77 ++++++--- src/Ghosts.Client/Handlers/PowerPoint.cs | 18 +- src/Ghosts.Client/Handlers/Word.cs | 40 ++++- .../Email/EmailConfiguration.cs | 2 +- .../Infrastructure/Email/EmailContent.cs | 1 + .../Infrastructure/FileListing.cs | 7 +- .../Infrastructure/RandomFilename.cs | 1 + .../Infrastructure/RandomText.cs | 23 +++ src/Ghosts.Client/Properties/AssemblyInfo.cs | 4 +- src/Ghosts.Client/Survey/SurveyManager.cs | 2 +- .../TimelineManager/Orchestrator.cs | 19 +-- src/Ghosts.Client/config/application.json | 2 +- src/Ghosts.Client/config/timeline.json | 6 +- .../Code/Helpers/DateTimeExtensions.cs | 16 ++ .../Code/Helpers/DictionaryExtensions.cs | 14 ++ .../Code/Helpers/EnumerableExtensions.cs | 46 ++++++ .../StringExtensions.cs} | 154 ++++++------------ .../Code/Helpers/StylingExtensions.cs | 16 ++ src/Ghosts.Domain/Code/LinkManager.cs | 1 + src/Ghosts.Domain/Code/TimelineBuilder.cs | 1 + src/Ghosts.Domain/Code/TimelineTranslator.cs | 4 +- src/Ghosts.Domain/Code/UserAgentManager.cs | 1 + 27 files changed, 342 insertions(+), 188 deletions(-) create mode 100644 src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs create mode 100644 src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs create mode 100644 src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs rename src/Ghosts.Domain/Code/{Helpers.cs => Helpers/StringExtensions.cs} (50%) mode change 100755 => 100644 create mode 100644 src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs diff --git a/src/Ghosts.Client/Comms/CheckId.cs b/src/Ghosts.Client/Comms/CheckId.cs index 9223861a..c26195c6 100755 --- a/src/Ghosts.Client/Comms/CheckId.cs +++ b/src/Ghosts.Client/Comms/CheckId.cs @@ -5,7 +5,6 @@ using Ghosts.Domain.Code; using NLog; using System; -using System.Diagnostics; using System.IO; using System.Net; @@ -40,7 +39,7 @@ public static string Id } catch { - _log.Error("config file could not be opened"); + _log.Error("No ID file"); return string.Empty; } } @@ -64,30 +63,38 @@ private static string Run() var machine = new ResultMachine(); GuestInfoVars.Load(machine); - - //call home - using (WebClient client = WebClientBuilder.BuildNoId(machine)) + + try { - try + //call home + using (var client = WebClientBuilder.BuildNoId(machine)) { - using (StreamReader reader = - new StreamReader(client.OpenRead(Program.Configuration.IdUrl))) + try { - s = reader.ReadToEnd(); - _log.Debug($"{DateTime.Now} - Received client ID"); + using (var reader = + new StreamReader(client.OpenRead(Program.Configuration.IdUrl))) + { + s = reader.ReadToEnd(); + _log.Debug("ID Received"); + } } - } - catch (WebException wex) - { - if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + catch (WebException wex) { - _log.Debug("No ID returned!", wex); + if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + { + _log.Debug("No ID returned!", wex.Message); + } + } + catch (Exception e) + { + _log.Error($"General comms exception: {e.Message}"); } } - catch (Exception e) - { - _log.Error(e); - } + } + catch (Exception e) + { + _log.Error($"Cannot connect to API: {e.Message}"); + return string.Empty; } s = s.Replace("\"", ""); diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index c542a58c..cc1234cf 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -84,7 +84,7 @@ private static void GetServerUpdates() } catch (Exception e) { - _log.Error(e); + _log.Error($"Exception in connecting to server: {e.Message}"); } } diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index c15c3ac8..092841cc 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -5,6 +5,7 @@ using Ghosts.Client.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; using OpenQA.Selenium; using OpenQA.Selenium.Interactions; diff --git a/src/Ghosts.Client/Handlers/BrowserChrome.cs b/src/Ghosts.Client/Handlers/BrowserChrome.cs index 9173cdd1..bf89fa0c 100755 --- a/src/Ghosts.Client/Handlers/BrowserChrome.cs +++ b/src/Ghosts.Client/Handlers/BrowserChrome.cs @@ -5,6 +5,7 @@ using OpenQA.Selenium.Chrome; using System; using System.IO; +using Ghosts.Domain.Code.Helpers; using OpenQA.Selenium; namespace Ghosts.Client.Handlers @@ -49,33 +50,32 @@ public BrowserChrome(TimelineHandler handler) options.BinaryLocation = handler.HandlerArgs["executable-location"]; } - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) { options.AddArguments("headless"); } - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) { options.AddArguments("--incognito"); } - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) { options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); } - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) { options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); } - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) { // ? } - if (handler.HandlerArgs.ContainsKey("blockscripts") && - handler.HandlerArgs["blockscripts"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) { options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); } diff --git a/src/Ghosts.Client/Handlers/BrowserFirefox.cs b/src/Ghosts.Client/Handlers/BrowserFirefox.cs index d604822d..8de03f5b 100755 --- a/src/Ghosts.Client/Handlers/BrowserFirefox.cs +++ b/src/Ghosts.Client/Handlers/BrowserFirefox.cs @@ -8,6 +8,7 @@ using System; using System.Diagnostics; using System.IO; +using Ghosts.Domain.Code.Helpers; namespace Ghosts.Client.Handlers { @@ -80,27 +81,27 @@ private bool FirefoxEx(TimelineHandler handler) if (handler.HandlerArgs != null) { - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) { options.AddArguments("--headless"); } - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) { options.AddArguments("--incognito"); } - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) { options.Profile.SetPreference("permissions.default.stylesheet", 2); } - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) { options.Profile.SetPreference("permissions.default.image", 2); } - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) { options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); } - if (handler.HandlerArgs.ContainsKey("blockscripts") && handler.HandlerArgs["blockscripts"] == "true") + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) { options.Profile.SetPreference("permissions.default.script", 2); } diff --git a/src/Ghosts.Client/Handlers/Excel.cs b/src/Ghosts.Client/Handlers/Excel.cs index 1e2bfd23..b32e5cc6 100755 --- a/src/Ghosts.Client/Handlers/Excel.cs +++ b/src/Ghosts.Client/Handlers/Excel.cs @@ -6,11 +6,11 @@ using Microsoft.Office.Interop.Excel; using NLog; using System; -using System.Drawing; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Ghosts.Domain.Code.Helpers; using NetOffice.ExcelApi.Tools; using Excel = NetOffice.ExcelApi; using XlWindowState = NetOffice.ExcelApi.Enums.XlWindowState; @@ -92,21 +92,21 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) var excelApplication = new Excel.Application { DisplayAlerts = false, - Visible = true + //Visible = true }; - try - { - excelApplication.WindowState = XlWindowState.xlMinimized; - foreach (var item in excelApplication.Workbooks) - { - item.Windows[1].WindowState = XlWindowState.xlMinimized; - } - } - catch (Exception e) - { - _log.Trace($"Could not minimize: {e}"); - } + //try + //{ + // excelApplication.WindowState = XlWindowState.xlMinimized; + // foreach (var item in excelApplication.Workbooks) + // { + // item.Windows[1].WindowState = XlWindowState.xlMinimized; + // } + //} + //catch (Exception e) + //{ + // _log.Trace($"Could not minimize: {e}"); + //} // create a utils instance, not need for but helpful to keep the lines of code low var utils = new CommonUtils(excelApplication); @@ -117,27 +117,33 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Excel adding worksheet"); var workSheet = (Excel.Worksheet) workBook.Worksheets[1]; + var range = GetRandomRange(); // draw back color and perform the BorderAround method - workSheet.Range("$B2:$B5").Interior.Color = utils.Color.ToDouble(Color.DarkGreen); - workSheet.Range("$B2:$B5").BorderAround(XlLineStyle.xlContinuous, XlBorderWeight.xlMedium, + workSheet.Range(range).Interior.Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + workSheet.Range(range).BorderAround(XlLineStyle.xlContinuous, XlBorderWeight.xlMedium, XlColorIndex.xlColorIndexAutomatic); + range = GetRandomRange(); // draw back color and border the range explicitly - workSheet.Range("$D2:$D5").Interior.Color = utils.Color.ToDouble(Color.DarkGreen); - workSheet.Range("$D2:$D5") + workSheet.Range(range).Interior.Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + workSheet.Range(range) .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] .LineStyle = XlLineStyle.xlDouble; - workSheet.Range("$D2:$D5") + workSheet.Range(range) .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] .Weight = 4; - workSheet.Range("$D2:$D5") + workSheet.Range(range) .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .Color = utils.Color.ToDouble(Color.Black); + .Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); var writeSleep = ProcessManager.Jitter(100); Thread.Sleep(writeSleep); - workSheet.Cells[1, 1].Value = "We have 2 simple shapes created."; + var list = RandomText.GetDictionary.GetDictionaryList(); + var rt = new RandomText(list.ToArray()); + rt.AddSentence(10); + + workSheet.Cells[1, 1].Value = rt.Content; var rand = RandomFilename.Generate(); @@ -177,9 +183,21 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace($"Excel saving to path - {path}"); workBook.SaveAs(path); + FileListing.Add(path); - Report(handler.HandlerType.ToString(), timelineEvent.Command, - timelineEvent.CommandArgs[0].ToString()); + Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); + + if (timelineEvent.CommandArgs.Contains("pdf")) + { + var pdfFileName = timelineEvent.CommandArgs.Contains("pdf-vary-filenames") ? $"{RandomFilename.Generate()}.pdf" : workBook.FullName.Replace(".xlsx", ".pdf"); + // Save document into PDF Format + workBook.ExportAsFixedFormat(NetOffice.ExcelApi.Enums.XlFixedFormatType.xlTypePDF, pdfFileName); + // end save as pdf + Report(handler.HandlerType.ToString(), timelineEvent.Command, "pdf"); + FileListing.Add(pdfFileName); + } + + workBook.Close(); if (timelineEvent.DelayAfter > 0) { @@ -230,5 +248,16 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Excel closing..."); } } + + private string GetRandomRange() + { + var r = new Random(); + var x = r.Next(1, 40); + var y = r.Next(x, 50); + var a1 = RandomText.GetRandomCapitalLetter(); + var a2 = RandomText.GetRandomCapitalLetter(a1); + + return $"${a1}{x}:${a2}{y}"; + } } } diff --git a/src/Ghosts.Client/Handlers/PowerPoint.cs b/src/Ghosts.Client/Handlers/PowerPoint.cs index 0510e6af..f7d69123 100755 --- a/src/Ghosts.Client/Handlers/PowerPoint.cs +++ b/src/Ghosts.Client/Handlers/PowerPoint.cs @@ -148,9 +148,23 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) Thread.Sleep(5000); presentation.SaveAs(path); + FileListing.Add(path); - Report(handler.HandlerType.ToString(), timelineEvent.Command, - timelineEvent.CommandArgs[0].ToString()); + Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); + + if (timelineEvent.CommandArgs.Contains("pdf")) + { + // Save document into PDF Format + var outputFileName = timelineEvent.CommandArgs.Contains("pdf-vary-filenames") ? $"{RandomFilename.Generate()}.pdf" : presentation.FullName.Replace(".pptx", ".pdf"); + object fileFormat = PpSaveAsFileType.ppSaveAsPDF; + + presentation.SaveAs(outputFileName, fileFormat, MsoTriState.msoCTrue); + // end save as pdf + Report(handler.HandlerType.ToString(), timelineEvent.Command, "pdf"); + FileListing.Add(outputFileName); + } + + presentation.Close(); if (timelineEvent.DelayAfter > 0) { diff --git a/src/Ghosts.Client/Handlers/Word.cs b/src/Ghosts.Client/Handlers/Word.cs index a47d9f5e..5df35e32 100755 --- a/src/Ghosts.Client/Handlers/Word.cs +++ b/src/Ghosts.Client/Handlers/Word.cs @@ -6,11 +6,15 @@ using NetOffice.WordApi.Enums; using NLog; using System; +using System.Drawing; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Ghosts.Domain.Code.Helpers; +using Microsoft.Office.Interop.PowerPoint; using Word = NetOffice.WordApi; +using VB = Microsoft.VisualBasic; namespace Ghosts.Client.Handlers { @@ -107,20 +111,20 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) { _log.Trace($"Could not minimize: {e}"); } - + // insert some text var list = RandomText.GetDictionary.GetDictionaryList(); var rt = new RandomText(list.ToArray()); - rt.AddContentParagraphs(1, 1, 1, 10, 50); + rt.AddContentParagraphs(1, 50); wordApplication.Selection.TypeText(rt.Content); var writeSleep = ProcessManager.Jitter(100); Thread.Sleep(writeSleep); wordApplication.Selection.HomeKey(WdUnits.wdLine, WdMovementType.wdExtend); - wordApplication.Selection.Font.Color = WdColor.wdColorSeaGreen; + wordApplication.Selection.Font.Color = GetWdColor(StylingExtensions.GetRandomColor()); wordApplication.Selection.Font.Bold = 1; - wordApplication.Selection.Font.Size = 18; + wordApplication.Selection.Font.Size = 12; var rand = RandomFilename.Generate(); @@ -160,10 +164,28 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) newDocument.Saved = true; newDocument.SaveAs(path); - Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); + Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); FileListing.Add(path); + if (timelineEvent.CommandArgs.Contains("pdf")) + { + // Save document into PDF Format + object oMissing = System.Reflection.Missing.Value; + object outputFileName = timelineEvent.CommandArgs.Contains("pdf-vary-filenames") ? $"{RandomFilename.Generate()}.pdf" : newDocument.FullName.Replace(".docx", ".pdf"); + object fileFormat = WdSaveFormat.wdFormatPDF; + + newDocument.SaveAs(outputFileName, fileFormat, oMissing, oMissing, + oMissing, oMissing, oMissing, oMissing, + oMissing, oMissing, oMissing, oMissing, + oMissing, oMissing, oMissing, oMissing); + // end save as pdf + Report(handler.HandlerType.ToString(), timelineEvent.Command, "pdf"); + FileListing.Add(outputFileName.ToString()); + } + + newDocument.Close(); + if (timelineEvent.DelayAfter > 0) { //sleep and leave the app open @@ -209,5 +231,13 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Word closing..."); } } + + + private WdColor GetWdColor(Color color) + { + var rgbColor = VB.Information.RGB(color.R, color.G, color.B); + var wdColor = (WdColor)rgbColor; + return wdColor; + } } } diff --git a/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs b/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs index af6d9b34..0b8279ac 100755 --- a/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs +++ b/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs @@ -4,7 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; // ReSharper disable InconsistentNaming diff --git a/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs b/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs index 971f4cf9..ef255e66 100755 --- a/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs +++ b/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs @@ -7,6 +7,7 @@ using System.Text; using FileHelpers; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; namespace Ghosts.Client.Infrastructure.Email diff --git a/src/Ghosts.Client/Infrastructure/FileListing.cs b/src/Ghosts.Client/Infrastructure/FileListing.cs index 5fe07698..24129fd7 100755 --- a/src/Ghosts.Client/Infrastructure/FileListing.cs +++ b/src/Ghosts.Client/Infrastructure/FileListing.cs @@ -1,5 +1,6 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -87,10 +88,8 @@ public static void FlushList() continue; } - var creationTime = file.CreationTime.Hour; - _log.Trace($"Delete evaluation for {file.FullName} {file.CreationTime}"); - - if (!file.Exists || (creationTime <= Program.Configuration.OfficeDocsMaxAgeInHours)) + _log.Trace($"Delete evaluation for {file.FullName} {file.CreationTime} vs. {DateTime.Now.AddHours(-Program.Configuration.OfficeDocsMaxAgeInHours)}"); + if (!file.Exists || (file.CreationTime > (DateTime.Now.AddHours(-Program.Configuration.OfficeDocsMaxAgeInHours)))) continue; try diff --git a/src/Ghosts.Client/Infrastructure/RandomFilename.cs b/src/Ghosts.Client/Infrastructure/RandomFilename.cs index b3138396..c8b407ad 100755 --- a/src/Ghosts.Client/Infrastructure/RandomFilename.cs +++ b/src/Ghosts.Client/Infrastructure/RandomFilename.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; namespace Ghosts.Client.Infrastructure diff --git a/src/Ghosts.Client/Infrastructure/RandomText.cs b/src/Ghosts.Client/Infrastructure/RandomText.cs index 11be1b0e..bdfc7d3a 100755 --- a/src/Ghosts.Client/Infrastructure/RandomText.cs +++ b/src/Ghosts.Client/Infrastructure/RandomText.cs @@ -17,12 +17,35 @@ public class RandomText private readonly StringBuilder _builder; private readonly string[] _words; + public static char GetRandomCapitalLetter() + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + var rand = new Random(); + return chars[rand.Next(0, chars.Length)]; + } + + public static char GetRandomCapitalLetter(char after) + { + after = char.ToUpper(after); + var index = (int)after % 32; + + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + var rand = new Random(); + return chars[rand.Next(index, chars.Length)]; + } + public RandomText(string[] words) { _builder = new StringBuilder(); _words = words; } + public void AddContentParagraphs(int minParagraphs, int maxParagraphs) + { + var paragraphs = _random.Next(minParagraphs, maxParagraphs); + AddContentParagraphs(paragraphs, paragraphs, (paragraphs + 10), (paragraphs * 10), (paragraphs * 25)); + } + public void AddContentParagraphs(int numberParagraphs, int minSentences, int maxSentences, int minWords, int maxWords) { for (var i = 0; i < numberParagraphs; i++) diff --git a/src/Ghosts.Client/Properties/AssemblyInfo.cs b/src/Ghosts.Client/Properties/AssemblyInfo.cs index 7d02c501..dd982e5e 100755 --- a/src/Ghosts.Client/Properties/AssemblyInfo.cs +++ b/src/Ghosts.Client/Properties/AssemblyInfo.cs @@ -31,5 +31,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("6.0.0.0")] -[assembly: AssemblyFileVersion("6.0.0.0")] +[assembly: AssemblyVersion("6.0.1.0")] +[assembly: AssemblyFileVersion("6.0.1.0")] diff --git a/src/Ghosts.Client/Survey/SurveyManager.cs b/src/Ghosts.Client/Survey/SurveyManager.cs index ee92ed8d..5baa2506 100644 --- a/src/Ghosts.Client/Survey/SurveyManager.cs +++ b/src/Ghosts.Client/Survey/SurveyManager.cs @@ -10,9 +10,9 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading; -using Ghosts.Client.Infrastructure; using Ghosts.Client.Comms; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using Newtonsoft.Json; using NLog; diff --git a/src/Ghosts.Client/TimelineManager/Orchestrator.cs b/src/Ghosts.Client/TimelineManager/Orchestrator.cs index 61a74424..0e7f1b48 100755 --- a/src/Ghosts.Client/TimelineManager/Orchestrator.cs +++ b/src/Ghosts.Client/TimelineManager/Orchestrator.cs @@ -24,7 +24,7 @@ public class Orchestrator private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private static DateTime _lastRead = DateTime.MinValue; private Thread MonitorThread { get; set; } - private Timeline _timeline; + private static Timeline DefaultTimeline; private FileSystemWatcher timelineWatcher; private bool _isSafetyNetRunning = false; private bool _isTempCleanerRunning = false; @@ -39,6 +39,8 @@ public void Run() { try { + DefaultTimeline = TimelineBuilder.GetLocalTimeline(); + if (_isSafetyNetRunning != true) //checking if safetynet has already been started { this.StartSafetyNet(); //watch instance numbers @@ -51,8 +53,6 @@ public void Run() _isTempCleanerRunning = true; } - this._timeline = TimelineBuilder.GetLocalTimeline(); - // now watch that file for changes if(timelineWatcher == null) //you can change this to a bool if you want but checks if the object has been created { @@ -70,9 +70,9 @@ public void Run() //load into an managing object //which passes the timeline commands to handlers //and creates a thread to execute instructions over that timeline - if (this._timeline.Status == Timeline.TimelineStatus.Run) + if (DefaultTimeline.Status == Timeline.TimelineStatus.Run) { - RunEx(this._timeline); + RunEx(DefaultTimeline); } else { @@ -164,7 +164,7 @@ private void StartSafetyNet() IsBackground = true, Name = "ghosts-safetynet" }; - t.Start(); + t.Start(DefaultTimeline); } catch (Exception e) { @@ -175,17 +175,16 @@ private void StartSafetyNet() ///here lies technical debt //TODO clean up // if supposed to be one excel running, and there is more than 2, then kill race condition - private static void SafetyNet() + private static void SafetyNet(object defaultTimeline) { + var timeline = (Timeline) defaultTimeline; while (true) { try { _log.Trace("SafetyNet loop beginning"); - FileListing.FlushList(); //Added 6/10 by AMV to clear clogged while loop. - - var timeline = TimelineBuilder.GetLocalTimeline(); + FileListing.FlushList(); var handlerCount = timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.Excel); var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.Excel).ToList(); diff --git a/src/Ghosts.Client/config/application.json b/src/Ghosts.Client/config/application.json index 0a619b4f..a4c7ac4f 100755 --- a/src/Ghosts.Client/config/application.json +++ b/src/Ghosts.Client/config/application.json @@ -38,7 +38,7 @@ "ChromeExtensions": "", "FirefoxInstallLocation": "", "FirefoxMajorVersionMinimum": 48, - "OfficeDocsMaxAgeInHours": 6, + "OfficeDocsMaxAgeInHours": 72, "Email": { "RecipientsToMin": 1, "RecipientsToMax": 3, diff --git a/src/Ghosts.Client/config/timeline.json b/src/Ghosts.Client/config/timeline.json index 70c744b8..e0557c6b 100755 --- a/src/Ghosts.Client/config/timeline.json +++ b/src/Ghosts.Client/config/timeline.json @@ -96,7 +96,7 @@ "TimeLineEvents": [ { "Command": "create", - "CommandArgs": [ "%homedrive%%homepath%\\Documents" ], + "CommandArgs": [ "%homedrive%%homepath%\\Documents", "pdf", "pdf-vary-filenames" ], "DelayAfter": 900000, "DelayBefore": 0 } @@ -111,7 +111,7 @@ "TimeLineEvents": [ { "Command": "create", - "CommandArgs": [ "%homedrive%%homepath%\\Documents" ], + "CommandArgs": [ "%homedrive%%homepath%\\Documents", "pdf", "pdf-vary-filenames" ], "DelayAfter": 900000, "DelayBefore": 0 } @@ -126,7 +126,7 @@ "TimeLineEvents": [ { "Command": "create", - "CommandArgs": [ "%homedrive%%homepath%\\Documents" ], + "CommandArgs": [ "%homedrive%%homepath%\\Documents", "pdf", "pdf-vary-filenames" ], "DelayAfter": 900000, "DelayBefore": 0 } diff --git a/src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs b/src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs new file mode 100644 index 00000000..dd0a7c14 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs @@ -0,0 +1,16 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.IO; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class DateTimeExtensions + { + public static bool IsOlderThanHours(string filename, int hours) + { + var threshold = DateTime.Now.AddHours(-hours); + return File.GetCreationTime(filename) <= threshold; + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs b/src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs new file mode 100644 index 00000000..03ffa0f4 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs @@ -0,0 +1,14 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System.Collections.Generic; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class DictionaryExtensions + { + public static bool ContainsKeyWithOption(this Dictionary options, string key, string value) + { + return options.ContainsKey(key) && options[key] == value; + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs b/src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs new file mode 100644 index 00000000..f66677ee --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs @@ -0,0 +1,46 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class EnumerableExtensions + { + /// + /// Get name value of an Enum + /// + /// + /// + /// + public static T ParseEnum(string value) + { + return (T)Enum.Parse(typeof(T), value, true); + } + + /// + /// Picks 1 random object from a list T + /// + public static T PickRandom(this IEnumerable source) + { + return source.PickRandom(1).Single(); + } + + /// + /// Picks n random objects from a list T + /// + public static IEnumerable PickRandom(this IEnumerable source, int count) + { + return source.Shuffle().Take(count); + } + + /// + /// Randomize a list of T objects in list + /// + public static IEnumerable Shuffle(this IEnumerable source) + { + return source.OrderBy(x => Guid.NewGuid()); + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers.cs b/src/Ghosts.Domain/Code/Helpers/StringExtensions.cs old mode 100755 new mode 100644 similarity index 50% rename from src/Ghosts.Domain/Code/Helpers.cs rename to src/Ghosts.Domain/Code/Helpers/StringExtensions.cs index 60bb0ce9..2939a959 --- a/src/Ghosts.Domain/Code/Helpers.cs +++ b/src/Ghosts.Domain/Code/Helpers/StringExtensions.cs @@ -1,101 +1,53 @@ -// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Ghosts.Domain.Code -{ - public static class Helpers - { - /// - /// Get name value of an Enum - /// - /// - /// - /// - public static T ParseEnum(string value) - { - return (T) Enum.Parse(typeof(T), value, true); - } - - public static bool IsOlderThanHours(string filename, int hours) - { - var threshold = DateTime.Now.AddHours(-hours); - return File.GetCreationTime(filename) <= threshold; - } - } - - public static class StringExtensions - { - public static IEnumerable Split(this string o, string splitString) - { - return o.Split(new [] { splitString }, StringSplitOptions.None); - } - - public static string GetTextBetweenQuotes(this string o) - { - var result = Regex.Match(o, "\"([^\"]*)\"").ToString(); - if(!string.IsNullOrEmpty(result)) - result = result.TrimStart('"').TrimEnd('"'); - return result; - } - - public static string ReplaceCaseInsensitive(this string input, string search, string replacement) - { - var result = Regex.Replace(input, Regex.Escape(search), replacement.Replace("$", "$$"), RegexOptions.IgnoreCase); - return result; - } - - public static string RemoveFirstLines(this string text, int linesCount) - { - var lines = Regex.Split(text, "\r\n|\r|\n").Skip(linesCount); - return string.Join(Environment.NewLine, lines.ToArray()); - } - - public static string RemoveWhitespace(this string input) - { - return new string(input.Where(c => !char.IsWhiteSpace(c)).ToArray()); - } - - public static string RemoveDuplicateSpaces(this string input) - { - var regex = new Regex("[ ]{2,}", RegexOptions.None); - return regex.Replace(input, " "); - } - - public static string ToFormValueString(this IDictionary dictionary) - { - return string.Join("&", dictionary.Select(x => x.Key + "=" + x.Value).ToArray()); - } - } - - public static class EnumerableExtensions - { - /// - /// Picks 1 random object from a list T - /// - public static T PickRandom(this IEnumerable source) - { - return source.PickRandom(1).Single(); - } - - /// - /// Picks n random objects from a list T - /// - public static IEnumerable PickRandom(this IEnumerable source, int count) - { - return source.Shuffle().Take(count); - } - - /// - /// Randomize a list of T objects in list - /// - public static IEnumerable Shuffle(this IEnumerable source) - { - return source.OrderBy(x => Guid.NewGuid()); - } - } -} \ No newline at end of file +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class StringExtensions + { + public static IEnumerable Split(this string o, string splitString) + { + return o.Split(new[] { splitString }, StringSplitOptions.None); + } + + public static string GetTextBetweenQuotes(this string o) + { + var result = Regex.Match(o, "\"([^\"]*)\"").ToString(); + if (!string.IsNullOrEmpty(result)) + result = result.TrimStart('"').TrimEnd('"'); + return result; + } + + public static string ReplaceCaseInsensitive(this string input, string search, string replacement) + { + var result = Regex.Replace(input, Regex.Escape(search), replacement.Replace("$", "$$"), RegexOptions.IgnoreCase); + return result; + } + + public static string RemoveFirstLines(this string text, int linesCount) + { + var lines = Regex.Split(text, "\r\n|\r|\n").Skip(linesCount); + return string.Join(Environment.NewLine, lines.ToArray()); + } + + public static string RemoveWhitespace(this string input) + { + return new string(input.Where(c => !char.IsWhiteSpace(c)).ToArray()); + } + + public static string RemoveDuplicateSpaces(this string input) + { + var regex = new Regex("[ ]{2,}", RegexOptions.None); + return regex.Replace(input, " "); + } + + public static string ToFormValueString(this IDictionary dictionary) + { + return string.Join("&", dictionary.Select(x => x.Key + "=" + x.Value).ToArray()); + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs b/src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs new file mode 100644 index 00000000..2d3d6b52 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs @@ -0,0 +1,16 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Drawing; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class StylingExtensions + { + public static Color GetRandomColor() + { + var random = new Random(); + return Color.FromArgb((byte)random.Next(0, 255), (byte)random.Next(0, 255), (byte)random.Next(0, 255)); + } + } +} diff --git a/src/Ghosts.Domain/Code/LinkManager.cs b/src/Ghosts.Domain/Code/LinkManager.cs index ed3d39ba..d1a4c5d4 100644 --- a/src/Ghosts.Domain/Code/LinkManager.cs +++ b/src/Ghosts.Domain/Code/LinkManager.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; +using Ghosts.Domain.Code.Helpers; namespace Ghosts.Domain.Code { diff --git a/src/Ghosts.Domain/Code/TimelineBuilder.cs b/src/Ghosts.Domain/Code/TimelineBuilder.cs index 668ab118..04e23d36 100644 --- a/src/Ghosts.Domain/Code/TimelineBuilder.cs +++ b/src/Ghosts.Domain/Code/TimelineBuilder.cs @@ -29,6 +29,7 @@ public static Timeline GetLocalTimeline() _log.Trace($"Loading timeline config {TimelineFile }"); var raw = File.ReadAllText(TimelineFile); + var timeline = JsonConvert.DeserializeObject(raw); if (timeline.Id == Guid.Empty) { diff --git a/src/Ghosts.Domain/Code/TimelineTranslator.cs b/src/Ghosts.Domain/Code/TimelineTranslator.cs index 9a4f5670..fcb35685 100644 --- a/src/Ghosts.Domain/Code/TimelineTranslator.cs +++ b/src/Ghosts.Domain/Code/TimelineTranslator.cs @@ -1,8 +1,10 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + using System; using System.Collections.Generic; using System.Linq; +using Ghosts.Domain.Code.Helpers; using NLog; -using NPOI.HSSF.Record; namespace Ghosts.Domain.Code { diff --git a/src/Ghosts.Domain/Code/UserAgentManager.cs b/src/Ghosts.Domain/Code/UserAgentManager.cs index 971b9237..66242244 100755 --- a/src/Ghosts.Domain/Code/UserAgentManager.cs +++ b/src/Ghosts.Domain/Code/UserAgentManager.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using Ghosts.Domain.Code.Helpers; using NLog; namespace Ghosts.Domain.Code From 340082d8e0b3d5066ebcd54d37f341dffdd9783e Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 22 Nov 2021 18:10:26 -0500 Subject: [PATCH 29/42] updates domain.dll --- src/ghosts.api.sln | 1 + src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ghosts.api.sln b/src/ghosts.api.sln index 8f26b957..99b561ac 100644 --- a/src/ghosts.api.sln +++ b/src/ghosts.api.sln @@ -9,6 +9,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution LICENSE.md = LICENSE.md README.md = README.md compose-api.yml = compose-api.yml + Dockerfile-api = Dockerfile-api EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ghosts.domain", "Ghosts.Domain\ghosts.domain.csproj", "{3BCAE63E-7914-4BD3-A751-BF69820FF1BE}" diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index 12ab096e..02ead081 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -5,6 +5,7 @@ using ghosts.client.linux.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; using OpenQA.Selenium; using OpenQA.Selenium.Interactions; From 831550bdf4420c613d2ea76ee1b8e25abc7e277c Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Tue, 23 Nov 2021 13:46:40 -0500 Subject: [PATCH 30/42] Adds more excel randomization and some file cleanup --- src/Ghosts.Client/Handlers/Excel.cs | 72 +++++++-------- .../TimelineManager/Orchestrator.cs | 90 +++++++++---------- src/Ghosts.Domain/Code/ApplicationDetails.cs | 1 + src/Ghosts.Domain/Code/ClientConfiguration.cs | 43 ++++----- src/ghosts.windows.sln.DotSettings | 7 ++ 5 files changed, 112 insertions(+), 101 deletions(-) create mode 100644 src/ghosts.windows.sln.DotSettings diff --git a/src/Ghosts.Client/Handlers/Excel.cs b/src/Ghosts.Client/Handlers/Excel.cs index b32e5cc6..cac4491b 100755 --- a/src/Ghosts.Client/Handlers/Excel.cs +++ b/src/Ghosts.Client/Handlers/Excel.cs @@ -92,22 +92,8 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) var excelApplication = new Excel.Application { DisplayAlerts = false, - //Visible = true }; - //try - //{ - // excelApplication.WindowState = XlWindowState.xlMinimized; - // foreach (var item in excelApplication.Workbooks) - // { - // item.Windows[1].WindowState = XlWindowState.xlMinimized; - // } - //} - //catch (Exception e) - //{ - // _log.Trace($"Could not minimize: {e}"); - //} - // create a utils instance, not need for but helpful to keep the lines of code low var utils = new CommonUtils(excelApplication); @@ -117,27 +103,6 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Excel adding worksheet"); var workSheet = (Excel.Worksheet) workBook.Worksheets[1]; - var range = GetRandomRange(); - // draw back color and perform the BorderAround method - workSheet.Range(range).Interior.Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); - workSheet.Range(range).BorderAround(XlLineStyle.xlContinuous, XlBorderWeight.xlMedium, - XlColorIndex.xlColorIndexAutomatic); - - range = GetRandomRange(); - // draw back color and border the range explicitly - workSheet.Range(range).Interior.Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); - workSheet.Range(range) - .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .LineStyle = XlLineStyle.xlDouble; - workSheet.Range(range) - .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .Weight = 4; - workSheet.Range(range) - .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); - - var writeSleep = ProcessManager.Jitter(100); - Thread.Sleep(writeSleep); var list = RandomText.GetDictionary.GetDictionaryList(); var rt = new RandomText(list.ToArray()); @@ -145,6 +110,43 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) workSheet.Cells[1, 1].Value = rt.Content; + var random = new Random(); + for (var i = 2; i < 100; i++) + { + for (var j = 1; j < 100; j++) + { + if (random.Next(0, 20) != 1) // 1 in 20 cells are blank + workSheet.Cells[i, j].Value = random.Next(0, 999999999); + } + } + + for (var i = 0; i < random.Next(1,30); i++) + { + var range = GetRandomRange(); + // draw back color and perform the BorderAround method + workSheet.Range(range).Interior.Color = + utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + workSheet.Range(range).BorderAround(XlLineStyle.xlContinuous, XlBorderWeight.xlMedium, + XlColorIndex.xlColorIndexAutomatic); + + range = GetRandomRange(); + // draw back color and border the range explicitly + workSheet.Range(range).Interior.Color = + utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + workSheet.Range(range) + .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] + .LineStyle = XlLineStyle.xlDouble; + workSheet.Range(range) + .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] + .Weight = 4; + workSheet.Range(range) + .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] + .Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + } + + var writeSleep = ProcessManager.Jitter(100); + Thread.Sleep(writeSleep); + var rand = RandomFilename.Generate(); var dir = timelineEvent.CommandArgs[0].ToString(); diff --git a/src/Ghosts.Client/TimelineManager/Orchestrator.cs b/src/Ghosts.Client/TimelineManager/Orchestrator.cs index 0e7f1b48..1daef559 100755 --- a/src/Ghosts.Client/TimelineManager/Orchestrator.cs +++ b/src/Ghosts.Client/TimelineManager/Orchestrator.cs @@ -24,10 +24,10 @@ public class Orchestrator private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private static DateTime _lastRead = DateTime.MinValue; private Thread MonitorThread { get; set; } - private static Timeline DefaultTimeline; - private FileSystemWatcher timelineWatcher; - private bool _isSafetyNetRunning = false; - private bool _isTempCleanerRunning = false; + private static Timeline _defaultTimeline; + private FileSystemWatcher _timelineWatcher; + private bool _isSafetyNetRunning; + private bool _isTempCleanerRunning; private bool _isWordInstalled { get; set; } private bool _isExcelInstalled { get; set; } @@ -39,7 +39,7 @@ public void Run() { try { - DefaultTimeline = TimelineBuilder.GetLocalTimeline(); + _defaultTimeline = TimelineBuilder.GetLocalTimeline(); if (_isSafetyNetRunning != true) //checking if safetynet has already been started { @@ -53,26 +53,27 @@ public void Run() _isTempCleanerRunning = true; } + var dirName = TimelineBuilder.TimelineFilePath().DirectoryName; // now watch that file for changes - if(timelineWatcher == null) //you can change this to a bool if you want but checks if the object has been created + if (_timelineWatcher == null && dirName != null) //you can change this to a bool if you want but checks if the object has been created { _log.Trace("Timeline watcher starting and is null..."); - timelineWatcher = new FileSystemWatcher(TimelineBuilder.TimelineFilePath().DirectoryName) + _timelineWatcher = new FileSystemWatcher(dirName) { Filter = Path.GetFileName(TimelineBuilder.TimelineFilePath().Name) }; _log.Trace($"watching {Path.GetFileName(TimelineBuilder.TimelineFilePath().Name)}"); - timelineWatcher.NotifyFilter = NotifyFilters.LastWrite; - timelineWatcher.EnableRaisingEvents = true; - timelineWatcher.Changed += OnChanged; + _timelineWatcher.NotifyFilter = NotifyFilters.LastWrite; + _timelineWatcher.EnableRaisingEvents = true; + _timelineWatcher.Changed += OnChanged; } //load into an managing object //which passes the timeline commands to handlers //and creates a thread to execute instructions over that timeline - if (DefaultTimeline.Status == Timeline.TimelineStatus.Run) + if (_defaultTimeline.Status == Timeline.TimelineStatus.Run) { - RunEx(DefaultTimeline); + RunEx(_defaultTimeline); } else { @@ -164,7 +165,7 @@ private void StartSafetyNet() IsBackground = true, Name = "ghosts-safetynet" }; - t.Start(DefaultTimeline); + t.Start(_defaultTimeline); } catch (Exception e) { @@ -411,43 +412,42 @@ private void OnChanged(object source, FileSystemEventArgs e) _log.Trace($"FileWatcher event raised: {e.FullPath} {e.Name} {e.ChangeType}"); // filewatcher throws two events, we only need 1 - DateTime lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime != _lastRead) - { - _lastRead = lastWriteTime; - _log.Trace("FileWatcher Processing: " + e.FullPath + " " + e.ChangeType); - _log.Trace($"Reloading {MethodBase.GetCurrentMethod().DeclaringType}"); + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime == _lastRead) return; + + _lastRead = lastWriteTime; + _log.Trace("FileWatcher Processing: " + e.FullPath + " " + e.ChangeType); + _log.Trace($"Reloading {MethodBase.GetCurrentMethod().DeclaringType}"); - _log.Trace("terminate existing tasks and rerun orchestrator"); + _log.Trace("terminate existing tasks and rerun orchestrator"); - try - { - Stop(); - } - catch (Exception exception) - { - _log.Info(exception); - } + try + { + Stop(); + } + catch (Exception exception) + { + _log.Info(exception); + } - try - { - StartupTasks.CleanupProcesses(); - } - catch (Exception exception) - { - _log.Info(exception); - } + try + { + StartupTasks.CleanupProcesses(); + } + catch (Exception exception) + { + _log.Info(exception); + } - Thread.Sleep(7500); + Thread.Sleep(7500); - try - { - Run(); - } - catch (Exception exception) - { - _log.Info(exception); - } + try + { + Run(); + } + catch (Exception exception) + { + _log.Info(exception); } } catch (Exception exc) diff --git a/src/Ghosts.Domain/Code/ApplicationDetails.cs b/src/Ghosts.Domain/Code/ApplicationDetails.cs index df117e06..b4b18757 100755 --- a/src/Ghosts.Domain/Code/ApplicationDetails.cs +++ b/src/Ghosts.Domain/Code/ApplicationDetails.cs @@ -58,6 +58,7 @@ public static bool IsLinux() return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); } + // ReSharper disable once InconsistentNaming public static bool IsOSX() { return RuntimeInformation.IsOSPlatform(OSPlatform.OSX); diff --git a/src/Ghosts.Domain/Code/ClientConfiguration.cs b/src/Ghosts.Domain/Code/ClientConfiguration.cs index e6e1c481..16a54ad3 100755 --- a/src/Ghosts.Domain/Code/ClientConfiguration.cs +++ b/src/Ghosts.Domain/Code/ClientConfiguration.cs @@ -11,60 +11,61 @@ namespace Ghosts.Domain.Code public class ClientConfiguration { /// - /// Should each instance generate and use its own ID to identify with server? + /// Should each instance generate and use its own ID to identify with server? /// public bool IdEnabled { get; set; } /// - /// API URL for client to get its instance ID + /// API URL for client to get its instance ID /// public string IdUrl { get; set; } /// - /// guest|guestinfo + /// guest|guestinfo /// public string IdFormat { get; set; } /// - /// if using guestinfo, the key to query for the value to use as hostname + /// if using guestinfo, the key to query for the value to use as hostname /// public string IdFormatKey { get; set; } public string IdFormatValue { get; set; } /// - /// Where is vmtools? + /// Where is vmtools? /// + // ReSharper disable once InconsistentNaming public string VMWareToolsLocation { get; set; } /// - /// Are client health checks enabled? + /// Are client health checks enabled? /// public bool HealthIsEnabled { get; set; } /// - /// Is client executing a timeline of user activities? + /// Is client executing a timeline of user activities? /// public bool HandlersIsEnabled { get; set; } /// - /// Comma sep list of extensions for chrome (aka c:\path\to\extension) + /// Comma sep list of extensions for chrome (aka c:\path\to\extension) /// public string ChromeExtensions { get; set; } /// - /// The amount of hours that office docs will live before being cleaned (based on FileListing class reaper) - /// Set to -1 to disable + /// The amount of hours that office docs will live before being cleaned (based on FileListing class reaper) + /// Set to -1 to disable /// public int OfficeDocsMaxAgeInHours { get; set; } /// - /// Tokens to be replaced within email-content.csv for a more customized storyline + /// Tokens to be replaced within email-content.csv for a more customized storyline /// public Dictionary EmailContent { get; set; } /// - /// Client survey values + /// Client survey values /// public SurveySettings Survey { get; set; } @@ -74,12 +75,12 @@ public class ClientConfiguration public ContentSettings Content { get; set; } /// - /// Settable install location for non-standard "c:\program files\" or "c:\program files (x86)\" installs + /// Settable install location for non-standard "c:\program files\" or "c:\program files (x86)\" installs /// public string FirefoxInstallLocation { get; set; } /// - /// Geckodriver depends on at least this version of FF to be used + /// Geckodriver depends on at least this version of FF to be used /// public int FirefoxMajorVersionMinimum { get; set; } public bool EncodeHeaders { get; set; } @@ -98,19 +99,19 @@ public class ContentSettings public class ClientResultSettings { /// - /// Is client posting its results to server? + /// Is client posting its results to server? /// public bool IsEnabled { get; set; } public bool IsSecure { get; set; } /// - /// API URL for client to post activity results, like timeline, health, etc. + /// API URL for client to post activity results, like timeline, health, etc. /// public string PostUrl { get; set; } /// - /// How often should client post results? (this number is the ms sleep between cycles) + /// How often should client post results? (this number is the ms sleep between cycles) /// public int CycleSleep { get; set; } } @@ -118,17 +119,17 @@ public class ClientResultSettings public class ClientUpdateSettings { /// - /// Is client attempting to pull down updates from server? + /// Is client attempting to pull down updates from server? /// public bool IsEnabled { get; set; } /// - /// API URL for client to get updates like timeline, health, etc. + /// API URL for client to get updates like timeline, health, etc. /// public string PostUrl { get; set; } /// - /// How often should client poll for updates? (this number is the ms sleep between cycles) + /// How often should client poll for updates? (this number is the ms sleep between cycles) /// public int CycleSleep { get; set; } } @@ -166,7 +167,7 @@ public class EmailSettings public class ListenerSettings { /// - /// Set to -1 to disable + /// Set to -1 to disable /// public int Port { get; set; } } diff --git a/src/ghosts.windows.sln.DotSettings b/src/ghosts.windows.sln.DotSettings new file mode 100644 index 00000000..cf1de47e --- /dev/null +++ b/src/ghosts.windows.sln.DotSettings @@ -0,0 +1,7 @@ + + True + True + True + True + True + True \ No newline at end of file From ec5d68b6797ff6e2896210dce471541f32b9e4b0 Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Wed, 24 Nov 2021 09:57:33 -0500 Subject: [PATCH 31/42] Adds crawl browser action --- src/Ghosts.Client/Comms/Updates.cs | 2 +- .../Handlers/BaseBrowserHandler.cs | 129 ++++++++++++++---- src/Ghosts.Client/Handlers/BrowserChrome.cs | 3 +- src/Ghosts.Client/Handlers/BrowserFirefox.cs | 6 +- .../Code/Helpers/UriExtensions.cs | 13 ++ src/Ghosts.Domain/Code/LinkManager.cs | 82 +++++++---- 6 files changed, 183 insertions(+), 52 deletions(-) create mode 100644 src/Ghosts.Domain/Code/Helpers/UriExtensions.cs diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index cc1234cf..34792b6c 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -252,7 +252,7 @@ private static void PostClientResults() } catch (Exception e) { - _log.Error($"Problem posting logs to server {e}"); + _log.Error($"Problem posting logs to server: {e.Message}"); } finally { diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index 092841cc..9fe3a689 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Linq; using System.Threading; using Ghosts.Client.Infrastructure.Browser; using Ghosts.Domain; @@ -21,6 +22,8 @@ public abstract class BaseBrowserHandler : BaseHandler private int _stickiness = 0; private int _depthMin = 1; private int _depthMax = 10; + private LinkManager _linkManager; + private int _pageBrowseCount = 0; public void ExecuteEvents(TimelineHandler handler) { @@ -42,6 +45,36 @@ public void ExecuteEvents(TimelineHandler handler) switch (timelineEvent.Command) { + case "crawl": + if (handler.HandlerArgs.ContainsKey("stickiness")) + { + int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); + } + + Driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + + foreach (var site in timelineEvent.CommandArgs) + { + if (Driver.CurrentWindowHandle == null) + { + throw new Exception("Browser window handle not available"); + } + + this._pageBrowseCount = 0; + config = RequestConfiguration.Load(site); + this._linkManager = new LinkManager(config.Uri); + if (config.Uri.IsWellFormedOriginalString()) + { + MakeRequest(config); + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + + GetAllLinks(config, true); + CrawlAllLinks(config, handler, timelineEvent, true); + } + } + + break; case "random": // setup @@ -82,32 +115,9 @@ public void ExecuteEvents(TimelineHandler handler) { try { - var linkManager = new LinkManager(config.GetHost()); - - - //get all links - var links = Driver.FindElements(By.TagName("a")); - foreach (var l in links) - { - var node = l.GetAttribute("href"); - if (string.IsNullOrEmpty(node) || - node.ToLower().StartsWith("//")) - { - //skip, these seem ugly - } - // http|s links - else if (node.ToLower().StartsWith("http")) - { - linkManager.AddLink(node.ToLower(), 1); - } - // relative links - prefix the scheme and host - else - { - linkManager.AddLink($"{config.GetHost()}{node.ToLower()}", 2); - } - } - - var link = linkManager.Choose(); + this._linkManager = new LinkManager(config.Uri); + GetAllLinks(config, false); + var link = this._linkManager.Choose(); if (link == null) { return; @@ -248,5 +258,72 @@ public void Stop() Report(BrowserType.ToString(), "Stop", string.Empty); Close(); } + + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + if (this._pageBrowseCount > this._stickiness) + return; + + try + { + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + this._linkManager.AddLink(uri, 1); + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + } + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, TimelineEvent timelineEvent, bool sameSite) + { + if (this._linkManager?.Links == null) + return; + if (this._pageBrowseCount > this._stickiness) + return; + + foreach (var link in this._linkManager.Links.Where(x=>x.WasBrowsed == false).OrderByDescending(x=>x.Priority)) + { + if (this._pageBrowseCount > this._stickiness) + return; + if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) + continue; + + config.Method = "GET"; + config.Uri = link.Url; + + MakeRequest(config); + + foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) + l.WasBrowsed = true; + this._pageBrowseCount += 1; + if (this._pageBrowseCount > this._stickiness) + return; + + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), timelineEvent.TrackableId); + GetAllLinks(config, sameSite); + CrawlAllLinks(config, handler, timelineEvent, sameSite); + Thread.Sleep(timelineEvent.DelayAfter); + } + } } } \ No newline at end of file diff --git a/src/Ghosts.Client/Handlers/BrowserChrome.cs b/src/Ghosts.Client/Handlers/BrowserChrome.cs index bf89fa0c..2250e0b7 100755 --- a/src/Ghosts.Client/Handlers/BrowserChrome.cs +++ b/src/Ghosts.Client/Handlers/BrowserChrome.cs @@ -82,12 +82,13 @@ public BrowserChrome(TimelineHandler handler) } options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); + options.AddUserProfilePreference("profile.default_content_setting_values.geolocation", 2); options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); - + if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) { options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); diff --git a/src/Ghosts.Client/Handlers/BrowserFirefox.cs b/src/Ghosts.Client/Handlers/BrowserFirefox.cs index 8de03f5b..3e589f98 100755 --- a/src/Ghosts.Client/Handlers/BrowserFirefox.cs +++ b/src/Ghosts.Client/Handlers/BrowserFirefox.cs @@ -110,7 +110,11 @@ private bool FirefoxEx(TimelineHandler handler) options.Profile.SetPreference("permissions.default.cookies", 2); options.Profile.SetPreference("permissions.default.popups", 2); options.Profile.SetPreference("permissions.default.geolocation", 2); - options.Profile.SetPreference("permissions.default.media_stream", 2); + options.Profile.SetPreference("permissions.default.media_stream", 2); + + options.Profile.SetPreference("geo.enabled", false); + options.Profile.SetPreference("geo.prompt.testing", false); + options.Profile.SetPreference("geo.prompt.testing.allow", false); Driver = new FirefoxDriver(options); base.Driver = Driver; diff --git a/src/Ghosts.Domain/Code/Helpers/UriExtensions.cs b/src/Ghosts.Domain/Code/Helpers/UriExtensions.cs new file mode 100644 index 00000000..da5c925d --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/UriExtensions.cs @@ -0,0 +1,13 @@ +using System; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class UriExtensions + { + public static string GetDomain(this Uri uri) + { + var a = uri.Host.Split('.'); + return a.GetUpperBound(0) < 2 ? uri.Host : $"{a[a.GetUpperBound(0) - 1]}.{a[a.GetUpperBound(0)]}"; + } + } +} diff --git a/src/Ghosts.Domain/Code/LinkManager.cs b/src/Ghosts.Domain/Code/LinkManager.cs index d1a4c5d4..2c0e0ed3 100644 --- a/src/Ghosts.Domain/Code/LinkManager.cs +++ b/src/Ghosts.Domain/Code/LinkManager.cs @@ -1,10 +1,10 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using Ghosts.Domain.Code.Helpers; using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; -using Ghosts.Domain.Code.Helpers; namespace Ghosts.Domain.Code { @@ -16,63 +16,95 @@ public Link() } public Uri Url { get; set; } + /// + /// Higher priority is more important + /// public int Priority { get; set; } + + public bool WasBrowsed { get; set; } } public class LinkManager { public List Links { private set; get; } - - private readonly string _baseUrl; + + private readonly Uri _baseUri; private readonly Random _random = new Random(); - public LinkManager(string baseUrl) + public LinkManager(Uri baseUri) { Links = new List(); - _baseUrl = baseUrl; + _baseUri = baseUri; } - + public void AddLink(string url, int priority) { + if (!Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out var uri)) + { + return; + } + this.AddLink(uri, priority); + } + + public void AddLink(Uri uri, int priority) + { + string[] validSchemes = {"http", "https"}; + if (!validSchemes.Contains(uri.Scheme)) + { + return; + } + + foreach (var link in Links) + { + if (Uri.Compare(uri, link.Url, UriComponents.Host | UriComponents.PathAndQuery, UriFormat.SafeUnescaped, StringComparison.OrdinalIgnoreCase) == 0) + { + return; + } + } + + //truly a new link, add it try { - Links.Add(new Link {Url = new Uri(url), Priority = priority}); + Links.Add(new Link { Url = uri, Priority = priority }); } - catch(Exception e) + catch (Exception e) { - Console.WriteLine($"{url} {e}"); + Console.WriteLine($"{uri} {e}"); } } public Link Choose() { var pickList = new List(); - var baseUri = new Uri(_baseUrl); - var schemesToIgnore = new [] {"mailto", "skype", "tel"}; foreach (var link in Links) + { try { - if(schemesToIgnore.Any(s => s.StartsWith(link.Url.ToString()))) - continue; - // give relative links priority - if((link.Url.Scheme + link.Url.Host).Replace("www.", "").Equals((baseUri.Scheme + baseUri.Host).Replace("www.", ""), StringComparison.InvariantCultureIgnoreCase)) + if ((link.Url.Scheme + link.Url.Host).Replace("www.", "").Equals((_baseUri.Scheme + _baseUri.Host).Replace("www.", ""), StringComparison.InvariantCultureIgnoreCase)) + { link.Priority += 1; - else if(link.Url.Scheme.Equals("file", StringComparison.InvariantCultureIgnoreCase)) + } + else if (link.Url.Scheme.Equals("file", StringComparison.InvariantCultureIgnoreCase)) + { link.Priority += 1; - + } + pickList.Add(link); } catch (Exception e) { Console.WriteLine($"{link.Url} : {e}"); } + } Links = pickList.OrderByDescending(o => o.Priority).ToList(); if (Links.Count < 1) + { return null; + } var priority = Links.First().Priority; var chosen = Links.Where(x => x.Priority == priority).PickRandom(); @@ -81,16 +113,20 @@ public Link Choose() { try { - var bUrl = baseUri.ToString(); + var bUrl = _baseUri.ToString(); if (bUrl.EndsWith("/")) + { bUrl = bUrl.Substring(0, bUrl.Length - 1); - - var thisUrl = chosen.Url.ToString().Replace("file://",""); - + } + + var thisUrl = chosen.Url.ToString().Replace("file://", ""); + thisUrl = Regex.Replace(thisUrl, "////", "//"); if (thisUrl.StartsWith("/")) + { thisUrl = thisUrl.Substring(1, thisUrl.Length - 1); - + } + chosen.Url = new Uri($"{bUrl}/{thisUrl}"); } catch (Exception e) @@ -101,5 +137,5 @@ public Link Choose() return chosen; } - } + } } \ No newline at end of file From 28f6dfbb005f941e58db4f8944b9a5a4e8b94430 Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Mon, 29 Nov 2021 07:25:22 -0500 Subject: [PATCH 32/42] adds browser crawl --- src/Ghosts.Client/Ghosts.Client.csproj | 1 + .../Handlers/BaseBrowserHandler.cs | 131 +++++----------- src/Ghosts.Client/Handlers/BrowserChrome.cs | 139 +++++++++-------- src/Ghosts.Client/Handlers/BrowserCrawl.cs | 144 ++++++++++++++++++ src/Ghosts.Client/Handlers/BrowserFirefox.cs | 124 ++++++++------- 5 files changed, 320 insertions(+), 219 deletions(-) create mode 100644 src/Ghosts.Client/Handlers/BrowserCrawl.cs diff --git a/src/Ghosts.Client/Ghosts.Client.csproj b/src/Ghosts.Client/Ghosts.Client.csproj index b10cf282..05a6488d 100755 --- a/src/Ghosts.Client/Ghosts.Client.csproj +++ b/src/Ghosts.Client/Ghosts.Client.csproj @@ -546,6 +546,7 @@ + diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index 9fe3a689..4dadf994 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -1,7 +1,6 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.Linq; using System.Threading; using Ghosts.Client.Infrastructure.Browser; using Ghosts.Domain; @@ -23,8 +22,7 @@ public abstract class BaseBrowserHandler : BaseHandler private int _depthMin = 1; private int _depthMax = 10; private LinkManager _linkManager; - private int _pageBrowseCount = 0; - + public void ExecuteEvents(TimelineHandler handler) { try @@ -46,34 +44,14 @@ public void ExecuteEvents(TimelineHandler handler) switch (timelineEvent.Command) { case "crawl": - if (handler.HandlerArgs.ContainsKey("stickiness")) - { - int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); - } - - Driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); - foreach (var site in timelineEvent.CommandArgs) { - if (Driver.CurrentWindowHandle == null) + new Thread(() => { - throw new Exception("Browser window handle not available"); - } - - this._pageBrowseCount = 0; - config = RequestConfiguration.Load(site); - this._linkManager = new LinkManager(config.Uri); - if (config.Uri.IsWellFormedOriginalString()) - { - MakeRequest(config); - Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), - timelineEvent.TrackableId); - - GetAllLinks(config, true); - CrawlAllLinks(config, handler, timelineEvent, true); - } + new BrowserCrawl(handler, timelineEvent, site.ToString()); + }); + Thread.Sleep(5000); } - break; case "random": @@ -210,6 +188,38 @@ public void ExecuteEvents(TimelineHandler handler) } } + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + try + { + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + this._linkManager.AddLink(uri, 1); + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + } + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + } + private void MakeRequest(RequestConfiguration config) { // Added try here because some versions of FF (v56) throw an exception for an unresolved site, @@ -258,72 +268,5 @@ public void Stop() Report(BrowserType.ToString(), "Stop", string.Empty); Close(); } - - private void GetAllLinks(RequestConfiguration config, bool sameSite) - { - if (this._pageBrowseCount > this._stickiness) - return; - - try - { - var links = Driver.FindElements(By.TagName("a")); - foreach (var l in links) - { - var node = l.GetAttribute("href"); - if (string.IsNullOrEmpty(node)) - continue; - node = node.ToLower(); - if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) - { - if (uri.GetDomain() != config.Uri.GetDomain()) - { - if (!sameSite) - this._linkManager.AddLink(uri, 1); - } - // relative links - prefix the scheme and host - else - { - this._linkManager.AddLink(uri, 2); - } - } - } - } - catch (Exception e) - { - _log.Trace(e); - } - } - - private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, TimelineEvent timelineEvent, bool sameSite) - { - if (this._linkManager?.Links == null) - return; - if (this._pageBrowseCount > this._stickiness) - return; - - foreach (var link in this._linkManager.Links.Where(x=>x.WasBrowsed == false).OrderByDescending(x=>x.Priority)) - { - if (this._pageBrowseCount > this._stickiness) - return; - if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) - continue; - - config.Method = "GET"; - config.Uri = link.Url; - - MakeRequest(config); - - foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) - l.WasBrowsed = true; - this._pageBrowseCount += 1; - if (this._pageBrowseCount > this._stickiness) - return; - - Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), timelineEvent.TrackableId); - GetAllLinks(config, sameSite); - CrawlAllLinks(config, handler, timelineEvent, sameSite); - Thread.Sleep(timelineEvent.DelayAfter); - } - } } } \ No newline at end of file diff --git a/src/Ghosts.Client/Handlers/BrowserChrome.cs b/src/Ghosts.Client/Handlers/BrowserChrome.cs index 2250e0b7..ab00ff69 100755 --- a/src/Ghosts.Client/Handlers/BrowserChrome.cs +++ b/src/Ghosts.Client/Handlers/BrowserChrome.cs @@ -14,7 +14,7 @@ public class BrowserChrome : BaseBrowserHandler { public new IJavaScriptExecutor JS { get; private set; } - private string GetInstallLocation() + private static string GetInstallLocation() { var path = @"C:\Program Files\Google\Chrome\Application\chrome.exe"; if (File.Exists(path)) @@ -31,72 +31,9 @@ public BrowserChrome(TimelineHandler handler) BrowserType = HandlerType.BrowserChrome; try { - var options = new ChromeOptions(); - options.AddArguments("disable-infobars"); - options.AddArguments("disable-logging"); - options.AddArguments("--disable-logging"); - options.AddArgument("--log-level=3"); - options.AddArgument("--silent"); - - options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); - options.AddUserProfilePreference("disable-popup-blocking", "true"); - options.BinaryLocation = GetInstallLocation(); - - if (handler.HandlerArgs != null) - { - if (handler.HandlerArgs.ContainsKey("executable-location") && - !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) - { - options.BinaryLocation = handler.HandlerArgs["executable-location"]; - } - - if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) - { - options.AddArguments("headless"); - } - - if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) - { - options.AddArguments("--incognito"); - } - - if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) - { - options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); - } - - if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) - { - options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); - } - - if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) - { - // ? - } - - if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) - { - options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); - } - } - - options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); - options.AddUserProfilePreference("profile.default_content_setting_values.geolocation", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); - - if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) - { - options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); - } - - Driver = new ChromeDriver(options); + Driver = GetDriver(handler); base.Driver = Driver; - + JS = (IJavaScriptExecutor)Driver; base.JS = JS; @@ -124,5 +61,75 @@ public BrowserChrome(TimelineHandler handler) ProcessManager.KillProcessAndChildrenByName(ProcessManager.ProcessNames.ChromeDriver); } } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var options = new ChromeOptions(); + options.AddArguments("disable-infobars"); + options.AddArguments("disable-logging"); + options.AddArguments("--disable-logging"); + options.AddArgument("--log-level=3"); + options.AddArgument("--silent"); + + options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); + options.AddUserProfilePreference("disable-popup-blocking", "true"); + options.BinaryLocation = GetInstallLocation(); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKey("executable-location") && + !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) + { + options.BinaryLocation = handler.HandlerArgs["executable-location"]; + } + + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("headless"); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + // ? + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); + } + } + + options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); + options.AddUserProfilePreference("profile.default_content_setting_values.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + + if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + { + options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + } + + var driver = new ChromeDriver(options); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + return driver; + } } } diff --git a/src/Ghosts.Client/Handlers/BrowserCrawl.cs b/src/Ghosts.Client/Handlers/BrowserCrawl.cs new file mode 100644 index 00000000..8c8714e4 --- /dev/null +++ b/src/Ghosts.Client/Handlers/BrowserCrawl.cs @@ -0,0 +1,144 @@ +using System; +using System.Linq; +using System.Threading; +using Ghosts.Client.Infrastructure.Browser; +using Ghosts.Domain; +using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; +using NLog; +using OpenQA.Selenium; + +namespace Ghosts.Client.Handlers +{ + class BrowserCrawl : BaseHandler + { + public static readonly Logger _log = LogManager.GetCurrentClassLogger(); + public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } + private int _stickiness = 0; + private LinkManager _linkManager; + private int _pageBrowseCount = 0; + + internal BrowserCrawl(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + switch (handler.HandlerType) + { + case HandlerType.BrowserChrome: + this.Driver = BrowserChrome.GetDriver(handler); + break; + case HandlerType.BrowserFirefox: + this.Driver = BrowserFirefox.GetDriver(handler); + break; + } + + if (handler.HandlerArgs.ContainsKey("stickiness")) + { + int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); + } + + this._pageBrowseCount = 0; + var config = RequestConfiguration.Load(site); + this._linkManager = new LinkManager(config.Uri); + if (config.Uri.IsWellFormedOriginalString()) + { + MakeRequest(config); + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + + GetAllLinks(config, true); + CrawlAllLinks(config, handler, timelineEvent, true); + } + } + + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + if (this._pageBrowseCount > this._stickiness) + { + return; + } + + try + { + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + this._linkManager.AddLink(uri, 1); + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + } + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, + TimelineEvent timelineEvent, bool sameSite) + { + if (this._linkManager?.Links == null) + { + return; + } + if (this._pageBrowseCount > this._stickiness) + { + return; + } + + foreach (var link in this._linkManager.Links.Where(x => x.WasBrowsed == false) + .OrderByDescending(x => x.Priority)) + { + if (this._pageBrowseCount > this._stickiness) + { + return; + } + if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) + continue; + + config.Method = "GET"; + config.Uri = link.Url; + + MakeRequest(config); + + foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) + l.WasBrowsed = true; + this._pageBrowseCount += 1; + if (this._pageBrowseCount > this._stickiness) + { + return; + } + + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + GetAllLinks(config, sameSite); + CrawlAllLinks(config, handler, timelineEvent, sameSite); + } + } + + private void MakeRequest(RequestConfiguration config) + { + try + { + Driver.Navigate().GoToUrl(config.Uri); + } + catch (Exception e) + { + _log.Trace(e.Message); + } + } + } +} diff --git a/src/Ghosts.Client/Handlers/BrowserFirefox.cs b/src/Ghosts.Client/Handlers/BrowserFirefox.cs index 3e589f98..b6592046 100755 --- a/src/Ghosts.Client/Handlers/BrowserFirefox.cs +++ b/src/Ghosts.Client/Handlers/BrowserFirefox.cs @@ -29,7 +29,7 @@ public BrowserFirefox(TimelineHandler handler) } } - private string GetInstallLocation() + private static string GetInstallLocation() { var path = @"C:\Program Files\Mozilla Firefox\firefox.exe"; if (File.Exists(path)) @@ -41,13 +41,13 @@ private string GetInstallLocation() return File.Exists(path) ? path : Program.Configuration.FirefoxInstallLocation; } - private int GetFirefoxVersion(string path) + private static int GetFirefoxVersion(string path) { var versionInfo = FileVersionInfo.GetVersionInfo(path); return versionInfo.FileMajorPart; } - private bool IsSufficientVersion(string path) + private static bool IsSufficientVersion(string path) { int currentVersion = GetFirefoxVersion(path); int minimumVersion = Program.Configuration.FirefoxMajorVersionMinimum; @@ -63,60 +63,7 @@ private bool FirefoxEx(TimelineHandler handler) { try { - var path = GetInstallLocation(); - - if (!IsSufficientVersion(path)) - { - _log.Warn("Firefox version is not sufficient. Exiting"); - return true; - } - - FirefoxOptions options = new FirefoxOptions(); - options.AddArguments("--disable-infobars"); - options.AddArguments("--disable-extensions"); - options.AddArguments("--disable-notifications"); - - options.BrowserExecutableLocation = path; - options.Profile = new FirefoxProfile(); - - if (handler.HandlerArgs != null) - { - if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) - { - options.AddArguments("--headless"); - } - if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) - { - options.AddArguments("--incognito"); - } - if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) - { - options.Profile.SetPreference("permissions.default.stylesheet", 2); - } - if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) - { - options.Profile.SetPreference("permissions.default.image", 2); - } - if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) - { - options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); - } - if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) - { - options.Profile.SetPreference("permissions.default.script", 2); - } - } - - options.Profile.SetPreference("permissions.default.cookies", 2); - options.Profile.SetPreference("permissions.default.popups", 2); - options.Profile.SetPreference("permissions.default.geolocation", 2); - options.Profile.SetPreference("permissions.default.media_stream", 2); - - options.Profile.SetPreference("geo.enabled", false); - options.Profile.SetPreference("geo.prompt.testing", false); - options.Profile.SetPreference("geo.prompt.testing.allow", false); - - Driver = new FirefoxDriver(options); + Driver = GetDriver(handler); base.Driver = Driver; JS = (IJavaScriptExecutor)Driver; @@ -128,8 +75,8 @@ private bool FirefoxEx(TimelineHandler handler) handler.Initial.Equals("about:external", StringComparison.InvariantCultureIgnoreCase)) { handler.Initial = "about:blank"; - } - + } + Driver.Navigate().GoToUrl(handler.Initial); if (handler.Loop) @@ -162,5 +109,64 @@ private bool FirefoxEx(TimelineHandler handler) return true; } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var path = GetInstallLocation(); + + if (!IsSufficientVersion(path)) + { + _log.Warn("Firefox version is not sufficient. Exiting"); + return null; + } + + FirefoxOptions options = new FirefoxOptions(); + options.AddArguments("--disable-infobars"); + options.AddArguments("--disable-extensions"); + options.AddArguments("--disable-notifications"); + + options.BrowserExecutableLocation = path; + options.Profile = new FirefoxProfile(); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("--headless"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.Profile.SetPreference("permissions.default.stylesheet", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.Profile.SetPreference("permissions.default.image", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.Profile.SetPreference("permissions.default.script", 2); + } + } + + options.Profile.SetPreference("permissions.default.cookies", 2); + options.Profile.SetPreference("permissions.default.popups", 2); + options.Profile.SetPreference("permissions.default.geolocation", 2); + options.Profile.SetPreference("permissions.default.media_stream", 2); + + options.Profile.SetPreference("geo.enabled", false); + options.Profile.SetPreference("geo.prompt.testing", false); + options.Profile.SetPreference("geo.prompt.testing.allow", false); + var driver = new FirefoxDriver(options); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + return driver; + } } } From 9a7707f7122ee4929b6d3aa29bd3019d631741f1 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Thu, 2 Dec 2021 11:23:25 -0500 Subject: [PATCH 33/42] Fixes up handlers --- src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs | 2 +- src/ghosts.client.linux/Handlers/Curl.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index 02ead081..fed2f45c 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -83,7 +83,7 @@ protected void ExecuteEvents(TimelineHandler handler) { try { - var linkManager = new LinkManager(config.GetHost()); + var linkManager = new LinkManager(new Uri(config.GetHost())); //get all links diff --git a/src/ghosts.client.linux/Handlers/Curl.cs b/src/ghosts.client.linux/Handlers/Curl.cs index 2bd77d3b..6fc93fa1 100755 --- a/src/ghosts.client.linux/Handlers/Curl.cs +++ b/src/ghosts.client.linux/Handlers/Curl.cs @@ -169,7 +169,7 @@ private void DeepBrowse() return; } - var linkManager = new LinkManager(this._currentHost); + var linkManager = new LinkManager(new Uri(this._currentHost)); foreach (var node in nodes) { if (!node.HasAttributes From cf52486f16ebdd7f81edb08c8e3bdaa0e5a6b17f Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Thu, 2 Dec 2021 16:46:19 -0500 Subject: [PATCH 34/42] adds browser crawl in (was testing on linux) --- .../Handlers/BaseBrowserHandler.cs | 28 ++++-- src/Ghosts.Client/Handlers/BrowserCrawl.cs | 86 +++++++++++++++++-- src/Ghosts.Client/Handlers/Excel.cs | 14 +-- src/Ghosts.Client/Handlers/PowerPoint.cs | 13 ++- src/Ghosts.Client/Handlers/Word.cs | 11 ++- 5 files changed, 126 insertions(+), 26 deletions(-) diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index 4dadf994..1d769f54 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -2,6 +2,7 @@ using System; using System.Threading; +using System.Threading.Tasks; using Ghosts.Client.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; @@ -22,7 +23,13 @@ public abstract class BaseBrowserHandler : BaseHandler private int _depthMin = 1; private int _depthMax = 10; private LinkManager _linkManager; - + + private Task LaunchThread(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + var o = new BrowserCrawl(); + return o.Crawl(handler, timelineEvent, site); + } + public void ExecuteEvents(TimelineHandler handler) { try @@ -44,13 +51,24 @@ public void ExecuteEvents(TimelineHandler handler) switch (timelineEvent.Command) { case "crawl": + var _taskMax = 1; + if (handler.HandlerArgs.ContainsKey("crawl-tasks-maximum")) + { + int.TryParse(handler.HandlerArgs["crawl-tasks-maximum"], out _taskMax); + } + + var i = 0; foreach (var site in timelineEvent.CommandArgs) { - new Thread(() => - { - new BrowserCrawl(handler, timelineEvent, site.ToString()); - }); + Task.Factory.StartNew(() => LaunchThread(handler, timelineEvent, site.ToString())); Thread.Sleep(5000); + i++; + + if (i >= _taskMax) + { + Task.WaitAll(); + i = 0; + } } break; case "random": diff --git a/src/Ghosts.Client/Handlers/BrowserCrawl.cs b/src/Ghosts.Client/Handlers/BrowserCrawl.cs index 8c8714e4..931c987b 100644 --- a/src/Ghosts.Client/Handlers/BrowserCrawl.cs +++ b/src/Ghosts.Client/Handlers/BrowserCrawl.cs @@ -1,6 +1,8 @@ -using System; +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; using System.Linq; -using System.Threading; +using System.Threading.Tasks; using Ghosts.Client.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; @@ -18,8 +20,11 @@ class BrowserCrawl : BaseHandler private int _stickiness = 0; private LinkManager _linkManager; private int _pageBrowseCount = 0; + private string _proxyLocalUrl = string.Empty; + private int _siteDepthMax = 1; + private int _siteDepthCurrent = 0; - internal BrowserCrawl(TimelineHandler handler, TimelineEvent timelineEvent, string site) + internal Task Crawl(TimelineHandler handler, TimelineEvent timelineEvent, string site) { switch (handler.HandlerType) { @@ -31,11 +36,23 @@ internal BrowserCrawl(TimelineHandler handler, TimelineEvent timelineEvent, stri break; } + Console.WriteLine($"{Environment.CurrentManagedThreadId} handle: {Driver.CurrentWindowHandle}"); + if (handler.HandlerArgs.ContainsKey("stickiness")) { int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); } + if (handler.HandlerArgs.ContainsKey("crawl-site-depth")) + { + int.TryParse(handler.HandlerArgs["crawl-site-depth"], out _siteDepthMax); + } + + if (handler.HandlerArgs.ContainsKey("crawl-proxy-local-url")) + { + _proxyLocalUrl = handler.HandlerArgs["crawl-proxy-local-url"]; + } + this._pageBrowseCount = 0; var config = RequestConfiguration.Load(site); this._linkManager = new LinkManager(config.Uri); @@ -44,52 +61,89 @@ internal BrowserCrawl(TimelineHandler handler, TimelineEvent timelineEvent, stri MakeRequest(config); Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), timelineEvent.TrackableId); + this._siteDepthCurrent += 1; + + if (this._siteDepthCurrent >= this._siteDepthMax) + return Task.CompletedTask; GetAllLinks(config, true); CrawlAllLinks(config, handler, timelineEvent, true); } + + Driver.Close(); + Driver.Quit(); + _log.Trace($"Run complete for {site}"); + return Task.CompletedTask; } private void GetAllLinks(RequestConfiguration config, bool sameSite) { + _log.Trace($"Getting links for {config.Uri}..."); + var linksAdded = 0; if (this._pageBrowseCount > this._stickiness) { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness}..."); return; } try { + var isInIframe = false; + // for use with pywb and proxy scraping + var iframes = Driver.FindElements(By.TagName("iframe")); + foreach (var iframe in iframes) + { + if (iframe.GetAttribute("id") == "replay_iframe") + { + Driver.SwitchTo().Frame(iframe); + isInIframe = true; + } + } + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) { var node = l.GetAttribute("href"); if (string.IsNullOrEmpty(node)) continue; node = node.ToLower(); + if (isInIframe && !string.IsNullOrEmpty(this._proxyLocalUrl)) + node = this._proxyLocalUrl + node; if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) { if (uri.GetDomain() != config.Uri.GetDomain()) { if (!sameSite) + { this._linkManager.AddLink(uri, 1); + linksAdded += 1; + } } // relative links - prefix the scheme and host else { this._linkManager.AddLink(uri, 2); + linksAdded += 1; } } } + + if (isInIframe) + Driver.SwitchTo().DefaultContent(); + + _log.Trace($"Added {linksAdded} links for {config.Uri}"); } catch (Exception e) { _log.Trace(e); } } - + private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, TimelineEvent timelineEvent, bool sameSite) { + _log.Trace($"Crawling links for {config.Uri}"); if (this._linkManager?.Links == null) { return; @@ -104,10 +158,14 @@ private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, { if (this._pageBrowseCount > this._stickiness) { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (2)..."); return; } + if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) + { continue; + } config.Method = "GET"; config.Uri = link.Url; @@ -115,17 +173,29 @@ private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, MakeRequest(config); foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) + { l.WasBrowsed = true; + _log.Trace($"Skipping {config.Uri} (already browsed)"); + } this._pageBrowseCount += 1; if (this._pageBrowseCount > this._stickiness) { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (3)..."); return; } Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), timelineEvent.TrackableId); - GetAllLinks(config, sameSite); - CrawlAllLinks(config, handler, timelineEvent, sameSite); + + // if this is the last step down, there is no reason to keep digging, + // but we don't increase the current depth count so as to allow peer + // pages at this level to still be scraped + if (this._siteDepthCurrent + 1 < this._siteDepthMax) + { + _log.Trace($"Drilling into {config.Uri}..."); + GetAllLinks(config, sameSite); + CrawlAllLinks(config, handler, timelineEvent, sameSite); + } } } @@ -137,8 +207,8 @@ private void MakeRequest(RequestConfiguration config) } catch (Exception e) { - _log.Trace(e.Message); + _log.Trace($"Requst error for {config.Uri}: {e.Message}"); } } } -} +} \ No newline at end of file diff --git a/src/Ghosts.Client/Handlers/Excel.cs b/src/Ghosts.Client/Handlers/Excel.cs index cac4491b..4d6b05e8 100755 --- a/src/Ghosts.Client/Handlers/Excel.cs +++ b/src/Ghosts.Client/Handlers/Excel.cs @@ -13,7 +13,6 @@ using Ghosts.Domain.Code.Helpers; using NetOffice.ExcelApi.Tools; using Excel = NetOffice.ExcelApi; -using XlWindowState = NetOffice.ExcelApi.Enums.XlWindowState; namespace Ghosts.Client.Handlers { @@ -213,20 +212,23 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) excelApplication.Dispose(); excelApplication = null; - workBook = null; - workSheet = null; - try { Marshal.ReleaseComObject(excelApplication); } - catch { } + catch + { + // ignore + } try { Marshal.FinalReleaseComObject(excelApplication); } - catch { } + catch + { + // ignore + } GC.Collect(); } diff --git a/src/Ghosts.Client/Handlers/PowerPoint.cs b/src/Ghosts.Client/Handlers/PowerPoint.cs index f7d69123..ed7d3533 100755 --- a/src/Ghosts.Client/Handlers/PowerPoint.cs +++ b/src/Ghosts.Client/Handlers/PowerPoint.cs @@ -177,19 +177,24 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) powerApplication.Quit(); powerApplication.Dispose(); powerApplication = null; - presentation = null; - + try { Marshal.ReleaseComObject(powerApplication); } - catch { } + catch + { + // ignore + } try { Marshal.FinalReleaseComObject(powerApplication); } - catch { } + catch + { + // ignore + } GC.Collect(); } diff --git a/src/Ghosts.Client/Handlers/Word.cs b/src/Ghosts.Client/Handlers/Word.cs index 5df35e32..22cc3db9 100755 --- a/src/Ghosts.Client/Handlers/Word.cs +++ b/src/Ghosts.Client/Handlers/Word.cs @@ -12,7 +12,6 @@ using System.Runtime.InteropServices; using System.Threading; using Ghosts.Domain.Code.Helpers; -using Microsoft.Office.Interop.PowerPoint; using Word = NetOffice.WordApi; using VB = Microsoft.VisualBasic; @@ -201,13 +200,19 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) { Marshal.ReleaseComObject(wordApplication); } - catch { } + catch + { + // ignore + } try { Marshal.FinalReleaseComObject(wordApplication); } - catch { } + catch + { + // ignore + } GC.Collect(); } From 104c5a23d747a8f2b856f2f2c192896fa947144d Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Tue, 7 Dec 2021 16:52:25 -0500 Subject: [PATCH 35/42] adds continued production testing - particularly crawling --- .../Communications/Updates.cs | 103 +++++---- .../Handlers/BaseBrowserHandler.cs | 121 +++++++--- .../Handlers/BrowserChrome.cs | 166 +++++++------- .../Handlers/BrowserCrawl.cs | 214 ++++++++++++++++++ .../Handlers/BrowserFirefox.cs | 171 +++++++++----- src/ghosts.client.linux/Program.cs | 1 + .../Survey/SurveyManager.cs | 1 + .../ghosts.client.linux.csproj | 2 +- src/ghosts.client.linux/nlog.config | 2 +- 9 files changed, 571 insertions(+), 210 deletions(-) create mode 100644 src/ghosts.client.linux/Handlers/BrowserCrawl.cs diff --git a/src/ghosts.client.linux/Communications/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs index 15950206..6e309877 100644 --- a/src/ghosts.client.linux/Communications/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -12,10 +12,10 @@ using Ghosts.Domain; using Ghosts.Domain.Code; using Ghosts.Domain.Messages.MesssagesForServer; -using Newtonsoft.Json; using NLog; +using Newtonsoft.Json; -namespace ghosts.client.linux.Communications +namespace ghosts.client.linux.Comms { /// /// Get updates from the C2 server - could be timeline, health, etc. @@ -47,23 +47,29 @@ private static void GetServerUpdates() if (!Program.Configuration.ClientUpdates.IsEnabled) return; + // ignore all certs + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); - Thread.Sleep(Program.Configuration.ClientUpdates.CycleSleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientUpdates.CycleSleep)); while (true) { try { - var s = string.Empty; + string s = string.Empty; using (var client = WebClientBuilder.Build(machine)) { try { - using var reader = - new StreamReader(client.OpenRead(Program.Configuration.ClientUpdates.PostUrl)); - s = reader.ReadToEnd(); - _log.Debug($"{DateTime.Now} - Received new configuration"); + using (var reader = + new StreamReader(client.OpenRead(Program.Configuration.ClientUpdates.PostUrl))) + { + s = reader.ReadToEnd(); + _log.Debug($"{DateTime.Now} - Received new configuration"); + } } catch (WebException wex) { @@ -78,7 +84,7 @@ private static void GetServerUpdates() } catch (Exception e) { - _log.Error(e); + _log.Error($"Exception in connecting to server: {e.Message}"); } } @@ -86,7 +92,6 @@ private static void GetServerUpdates() { var update = JsonConvert.DeserializeObject(s); - // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault switch (update.Type) { case UpdateClientConfig.UpdateType.RequestForTimeline: @@ -104,9 +109,12 @@ private static void GetServerUpdates() { _log.Trace($"PartialTimeline found: {timelineHandler.HandlerType}"); - foreach (var timelineEvent in timelineHandler.TimeLineEvents.Where(timelineEvent => string.IsNullOrEmpty(timelineEvent.TrackableId))) + foreach (var timelineEvent in timelineHandler.TimeLineEvents) { - timelineEvent.TrackableId = Guid.NewGuid().ToString(); + if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + { + timelineEvent.TrackableId = Guid.NewGuid().ToString(); + } } Orchestrator.RunCommand(timeline, timelineHandler); @@ -122,20 +130,18 @@ private static void GetServerUpdates() { var newTimeline = JsonConvert.DeserializeObject(update.Update.ToString()); //save to local disk - using var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Health); - var serializer = new JsonSerializer + using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Health)) { - Formatting = Formatting.Indented - }; - serializer.Serialize(file, newTimeline); + var serializer = new JsonSerializer(); + serializer.Formatting = Formatting.Indented; + serializer.Serialize(file, newTimeline); + } break; } default: - { _log.Debug($"Update {update.Type} has no handler, ignoring..."); break; - } } } } @@ -145,10 +151,10 @@ private static void GetServerUpdates() _log.Error(e); } - Thread.Sleep(Program.Configuration.ClientUpdates.CycleSleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientUpdates.CycleSleep)); } } - + private static void PostCurrentTimeline(UpdateClientConfig update) { // is the config for a specific timeline id? @@ -173,7 +179,6 @@ private static void PostCurrentTimeline(UpdateClientConfig update) } } - ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; var posturl = string.Empty; @@ -196,7 +201,7 @@ private static void PostCurrentTimeline(UpdateClientConfig update) var payload = TimelineBuilder.TimelineToString(timeline); var machine = new ResultMachine(); - // GuestInfoVars.Load(machine); // TODO? + // GuestInfoVars.Load(machine); using (var client = WebClientBuilder.Build(machine)) { @@ -208,7 +213,8 @@ private static void PostCurrentTimeline(UpdateClientConfig update) } catch (Exception e) { - _log.Debug($"Problem posting timeline to server from {ApplicationDetails.ConfigurationFiles.Timeline} to {posturl}"); + _log.Debug( + $"Problem posting timeline to server from {ApplicationDetails.ConfigurationFiles.Timeline} to {posturl}"); _log.Error(e); } } @@ -219,21 +225,24 @@ private static void PostClientResults() if (!Program.Configuration.ClientResults.IsEnabled) return; + // ignore all certs + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + var fileName = ApplicationDetails.LogFiles.ClientUpdates; - var cycleSleep = Program.Configuration.ClientResults.CycleSleep; - var postUrl = Program.Configuration.ClientResults.PostUrl; + var posturl = Program.Configuration.ClientResults.PostUrl; var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); - Thread.Sleep(cycleSleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientResults.CycleSleep)); while (true) { try { - if(File.Exists(fileName)) + if (File.Exists(fileName)) { - PostResults(fileName, machine, postUrl); + PostResults(fileName, machine, posturl); } else { @@ -242,26 +251,36 @@ private static void PostClientResults() } catch (Exception e) { - _log.Error($"Problem posting logs to server {e}"); + _log.Error($"Problem posting logs to server: {e.Message}"); + } + finally + { + GC.Collect(); + GC.WaitForPendingFinalizers(); } // look for other result files that have not been posted try { - foreach(var file in Directory.GetFiles(Path.GetDirectoryName(fileName))) + foreach (var file in Directory.GetFiles(Path.GetDirectoryName(fileName))) { if (!file.EndsWith("app.log") && file != fileName) { - PostResults(file, machine, postUrl, true); + PostResults(file, machine, posturl, true); } } } catch (Exception e) { - _log.Debug($"Problem posting overflow logs from {fileName} to server {postUrl}: {e}"); + _log.Debug($"Problem posting overflow logs from {fileName} to server {posturl} : {e}"); + } + finally + { + GC.Collect(); + GC.WaitForPendingFinalizers(); } - Thread.Sleep(cycleSleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientResults.CycleSleep)); } } @@ -274,10 +293,8 @@ private static void PostResults(string fileName, ResultMachine machine, string p sb.AppendLine(d); } - var r = new TransferLogDump - { - Log = sb.ToString() - }; + var r = new TransferLogDump(); + r.Log = sb.ToString(); var payload = JsonConvert.SerializeObject(r); @@ -286,10 +303,8 @@ private static void PostResults(string fileName, ResultMachine machine, string p payload = Crypto.EncryptStringAes(payload, machine.Name); payload = Base64Encoder.Base64Encode(payload); - var p = new EncryptedPayload - { - Payload = payload - }; + var p = new EncryptedPayload(); + p.Payload = payload; payload = JsonConvert.SerializeObject(p); } @@ -311,7 +326,7 @@ private static void PostResults(string fileName, ResultMachine machine, string p _log.Trace($"{DateTime.Now} - {fileName} posted to server successfully"); } - + internal static void PostSurvey() { // ignore all certs @@ -323,7 +338,7 @@ internal static void PostSurvey() { posturl = Program.Configuration.Survey.PostUrl; } - catch + catch (Exception exc) { _log.Error("Can't get survey posturl!"); return; diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index fed2f45c..52f26851 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -2,6 +2,7 @@ using System; using System.Threading; +using System.Threading.Tasks; using ghosts.client.linux.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; @@ -15,19 +16,26 @@ namespace ghosts.client.linux.handlers { public abstract class BaseBrowserHandler : BaseHandler { - protected static readonly Logger _log = LogManager.GetCurrentClassLogger(); - protected IWebDriver Driver { get; set; } - protected IJavaScriptExecutor JS { get; set; } - protected HandlerType BrowserType { get; set; } - private int _stickiness; + public static readonly Logger _log = LogManager.GetCurrentClassLogger(); + public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } + public HandlerType BrowserType { get; set; } + private int _stickiness = 0; private int _depthMin = 1; private int _depthMax = 10; + private LinkManager _linkManager; - protected void ExecuteEvents(TimelineHandler handler) + private Task LaunchThread(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + var o = new BrowserCrawl(); + return o.Crawl(handler, timelineEvent, site); + } + + public void ExecuteEvents(TimelineHandler handler) { try { - foreach (var timelineEvent in handler.TimeLineEvents) + foreach (TimelineEvent timelineEvent in handler.TimeLineEvents) { WorkingHours.Is(handler); @@ -43,6 +51,29 @@ protected void ExecuteEvents(TimelineHandler handler) switch (timelineEvent.Command) { + case "crawl": + var _taskMax = 1; + if (handler.HandlerArgs.ContainsKey("crawl-tasks-maximum")) + { + int.TryParse(handler.HandlerArgs["crawl-tasks-maximum"], out _taskMax); + } + + var i = 0; + foreach (var site in timelineEvent.CommandArgs) + { + LaunchThread(handler, timelineEvent, site.ToString()); + + // Task.Factory.StartNew(() => LaunchThread(handler, timelineEvent, site.ToString())); + // Thread.Sleep(5000); + // i++; + // + // if (i >= _taskMax) + // { + // Task.WaitAll(); + // i = 0; + // } + } + break; case "random": // setup @@ -83,32 +114,9 @@ protected void ExecuteEvents(TimelineHandler handler) { try { - var linkManager = new LinkManager(new Uri(config.GetHost())); - - - //get all links - var links = Driver.FindElements(By.TagName("a")); - foreach (var l in links) - { - var node = l.GetAttribute("href"); - if (string.IsNullOrEmpty(node) || - node.ToLower().StartsWith("//")) - { - //skip, these seem ugly - } - // http|s links - else if (node.ToLower().StartsWith("http")) - { - linkManager.AddLink(node.ToLower(), 1); - } - // relative links - prefix the scheme and host - else - { - linkManager.AddLink($"{config.GetHost()}{node.ToLower()}", 2); - } - } - - var link = linkManager.Choose(); + this._linkManager = new LinkManager(config.Uri); + GetAllLinks(config, false); + var link = this._linkManager.Choose(); if (link == null) { return; @@ -201,6 +209,38 @@ protected void ExecuteEvents(TimelineHandler handler) } } + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + try + { + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + this._linkManager.AddLink(uri, 1); + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + } + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + } + private void MakeRequest(RequestConfiguration config) { // Added try here because some versions of FF (v56) throw an exception for an unresolved site, @@ -234,5 +274,20 @@ private void MakeRequest(RequestConfiguration config) _log.Trace(e.Message); } } + + /// + /// Close browser + /// + public void Close() + { + Report(BrowserType.ToString(), "Close", string.Empty); + Driver.Close(); + } + + public void Stop() + { + Report(BrowserType.ToString(), "Stop", string.Empty); + Close(); + } } } \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserChrome.cs b/src/ghosts.client.linux/Handlers/BrowserChrome.cs index 40cf60e3..a66e3266 100644 --- a/src/ghosts.client.linux/Handlers/BrowserChrome.cs +++ b/src/ghosts.client.linux/Handlers/BrowserChrome.cs @@ -3,6 +3,7 @@ using Ghosts.Domain; using OpenQA.Selenium.Chrome; using System; +using Ghosts.Domain.Code.Helpers; using OpenQA.Selenium; // ReSharper disable StringLiteralTypo @@ -10,112 +11,121 @@ namespace ghosts.client.linux.handlers { public class BrowserChrome : BaseBrowserHandler { - private new IWebDriver Driver { get; set; } - private new IJavaScriptExecutor JS { get; set; } + public new IJavaScriptExecutor JS { get; private set; } public BrowserChrome(TimelineHandler handler) - { - BrowserType = HandlerType.BrowserChrome; - var hasRunSuccessfully = false; - while (!hasRunSuccessfully) - { - hasRunSuccessfully = ChromeEx(handler); - } - } - - private bool ChromeEx(TimelineHandler handler) { BrowserType = HandlerType.BrowserChrome; try { - var options = new ChromeOptions(); - options.AddArguments("disable-infobars"); - options.AddArguments("disable-logging"); - options.AddArguments("--disable-logging"); - options.AddArgument("--log-level=3"); - options.AddArgument("--silent"); - - options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); - options.AddUserProfilePreference("disable-popup-blocking", "true"); - //options.BinaryLocation = GetInstallLocation(); - - if (handler.HandlerArgs != null) - { - if (handler.HandlerArgs.ContainsKey("executable-location") && - !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) - { - options.BinaryLocation = handler.HandlerArgs["executable-location"]; - } - - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") - { - options.AddArguments("headless"); - } + Driver = GetDriver(handler); + base.Driver = Driver; + + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") - { - options.AddArguments("--incognito"); - } + Driver.Navigate().GoToUrl(handler.Initial); - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") + if (handler.Loop) + { + while (true) { - options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); + ExecuteEvents(handler); } + } + else + { + ExecuteEvents(handler); + } + } + catch (Exception e) + { + _log.Error(e); + } + } - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") - { - options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); - } + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var options = new ChromeOptions(); + options.AddArguments("disable-infobars"); + options.AddArguments("disable-logging"); + options.AddArguments("--disable-logging"); + options.AddArgument("--log-level=3"); + options.AddArgument("--silent"); - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") - { - // ? - } + options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); + options.AddUserProfilePreference("disable-popup-blocking", "true"); - if (handler.HandlerArgs.ContainsKey("blockscripts") && - handler.HandlerArgs["blockscripts"] == "true") - { - options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); - } + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKey("executable-location") && + !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) + { + options.BinaryLocation = handler.HandlerArgs["executable-location"]; } - options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("headless"); + } - if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) { - options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + options.AddArguments("--incognito"); } - Driver = new ChromeDriver(options); - base.Driver = Driver; + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); + } - JS = (IJavaScriptExecutor)Driver; - base.JS = JS; + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); + } - Driver.Navigate().GoToUrl(handler.Initial); + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + // ? + } - if (handler.Loop) + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) { - while (true) - { - ExecuteEvents(handler); - } + options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); } - - ExecuteEvents(handler); + } + + options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); + options.AddUserProfilePreference("profile.default_content_setting_values.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + + if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + { + options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + } + + _log.Trace("Browser preferences set successfully, getting driver..."); + + ChromeDriver driver; + + try + { + driver = new ChromeDriver(options); } catch (Exception e) { - _log.Error(e); - return false; + _log.Trace("Driver could not be instantiated. Does the proper driver exist? Are you running as a user and not root? Sometimes running the driver directly will uncover the underlaying issue."); + throw; } - return true; + _log.Trace("Driver instantiated successfully, setting timeouts..."); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + _log.Trace("Driver timeouts set successfully, continuing..."); + return driver; } } -} +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserCrawl.cs b/src/ghosts.client.linux/Handlers/BrowserCrawl.cs new file mode 100644 index 00000000..c85eae7a --- /dev/null +++ b/src/ghosts.client.linux/Handlers/BrowserCrawl.cs @@ -0,0 +1,214 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Linq; +using System.Threading.Tasks; +using ghosts.client.linux.Infrastructure.Browser; +using Ghosts.Domain; +using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; +using NLog; +using OpenQA.Selenium; + +namespace ghosts.client.linux.handlers +{ + class BrowserCrawl : BaseHandler + { + public static readonly Logger _log = LogManager.GetCurrentClassLogger(); + public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } + private int _stickiness = 0; + private LinkManager _linkManager; + private int _pageBrowseCount = 0; + private string _proxyLocalUrl = string.Empty; + private int _siteDepthMax = 1; + private int _siteDepthCurrent = 0; + + internal Task Crawl(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + switch (handler.HandlerType) + { + case HandlerType.BrowserChrome: + this.Driver = BrowserChrome.GetDriver(handler); + break; + case HandlerType.BrowserFirefox: + this.Driver = BrowserFirefox.GetDriver(handler); + break; + } + + Console.WriteLine($"{Environment.CurrentManagedThreadId} handle: {Driver.CurrentWindowHandle}"); + + if (handler.HandlerArgs.ContainsKey("stickiness")) + { + int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); + } + + if (handler.HandlerArgs.ContainsKey("crawl-site-depth")) + { + int.TryParse(handler.HandlerArgs["crawl-site-depth"], out _siteDepthMax); + } + + if (handler.HandlerArgs.ContainsKey("crawl-proxy-local-url")) + { + _proxyLocalUrl = handler.HandlerArgs["crawl-proxy-local-url"]; + } + + this._pageBrowseCount = 0; + var config = RequestConfiguration.Load(site); + this._linkManager = new LinkManager(config.Uri); + if (config.Uri.IsWellFormedOriginalString()) + { + MakeRequest(config); + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + this._siteDepthCurrent += 1; + + if (this._siteDepthCurrent >= this._siteDepthMax) + return Task.CompletedTask; + + GetAllLinks(config, true); + CrawlAllLinks(config, handler, timelineEvent, true); + } + + Driver.Close(); + Driver.Quit(); + _log.Trace($"Run complete for {site}"); + return Task.CompletedTask; + } + + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + _log.Trace($"Getting links for {config.Uri}..."); + var linksAdded = 0; + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness}..."); + return; + } + + try + { + var isInIframe = false; + // for use with pywb and proxy scraping + var iframes = Driver.FindElements(By.TagName("iframe")); + foreach(var iframe in iframes) + { + if (iframe.GetAttribute("id") == "replay_iframe") + { + Driver.SwitchTo().Frame(iframe); + isInIframe = true; + } + } + + var links = Driver.FindElements(By.TagName("a")); + + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (isInIframe && !string.IsNullOrEmpty(this._proxyLocalUrl)) + node = this._proxyLocalUrl + node; + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + { + this._linkManager.AddLink(uri, 1); + linksAdded += 1; + } + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + linksAdded += 1; + } + } + } + + if (isInIframe) + Driver.SwitchTo().DefaultContent(); + + _log.Trace($"Added {linksAdded} links for {config.Uri}"); + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, + TimelineEvent timelineEvent, bool sameSite) + { + _log.Trace($"Crawling links for {config.Uri}"); + if (this._linkManager?.Links == null) + { + return; + } + if (this._pageBrowseCount > this._stickiness) + { + return; + } + + foreach (var link in this._linkManager.Links.Where(x => x.WasBrowsed == false) + .OrderByDescending(x => x.Priority)) + { + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (2)..."); + return; + } + + if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) + { + continue; + } + + config.Method = "GET"; + config.Uri = link.Url; + + MakeRequest(config); + + foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) + { + l.WasBrowsed = true; + _log.Trace($"Skipping {config.Uri} (already browsed)"); + } + this._pageBrowseCount += 1; + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (3)..."); + return; + } + + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + + // if this is the last step down, there is no reason to keep digging, + // but we don't increase the current depth count so as to allow peer + // pages at this level to still be scraped + if (this._siteDepthCurrent + 1 < this._siteDepthMax) + { + _log.Trace($"Drilling into {config.Uri}..."); + GetAllLinks(config, sameSite); + CrawlAllLinks(config, handler, timelineEvent, sameSite); + } + } + } + + private void MakeRequest(RequestConfiguration config) + { + try + { + Driver.Navigate().GoToUrl(config.Uri); + } + catch (Exception e) + { + _log.Trace($"Requst error for {config.Uri}: {e.Message}"); + } + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index 79fec24a..0ac3c42f 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -3,7 +3,11 @@ using Ghosts.Domain; using NLog; using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; using System.Text; +using Ghosts.Domain.Code.Helpers; using OpenQA.Selenium; using OpenQA.Selenium.Firefox; // ReSharper disable StringLiteralTypo @@ -14,70 +18,58 @@ public class BrowserFirefox : BaseBrowserHandler { private new static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private new IWebDriver Driver { get; set; } - private new IJavaScriptExecutor JS { get; set; } + public new IWebDriver Driver { get; private set; } + public new IJavaScriptExecutor JS { get; private set; } public BrowserFirefox(TimelineHandler handler) { BrowserType = HandlerType.BrowserFirefox; - var hasRunSuccessfully = false; + bool hasRunSuccessfully = false; while (!hasRunSuccessfully) { hasRunSuccessfully = FirefoxEx(handler); } } - private bool FirefoxEx(TimelineHandler handler) + private static int GetFirefoxVersion(string path) { - try - { - var options = new FirefoxOptions(); - options.AddArguments("--disable-infobars"); - options.AddArguments("--disable-extensions"); - options.AddArguments("--disable-notifications"); - - //options.BrowserExecutableLocation = path; - options.Profile = new FirefoxProfile(); + var versionInfo = FileVersionInfo.GetVersionInfo(path); + return versionInfo.FileMajorPart; + } - if (handler.HandlerArgs != null) - { - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") - { - options.AddArguments("--headless"); - } - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") - { - options.AddArguments("--incognito"); - } - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") - { - options.Profile.SetPreference("permissions.default.stylesheet", 2); - } - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") - { - options.Profile.SetPreference("permissions.default.image", 2); - } - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") - { - options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); - } - if (handler.HandlerArgs.ContainsKey("blockscripts") && handler.HandlerArgs["blockscripts"] == "true") - { - options.Profile.SetPreference("permissions.default.script", 2); - } - } + private static bool IsSufficientVersion(string path) + { + int currentVersion = GetFirefoxVersion(path); + int minimumVersion = Program.Configuration.FirefoxMajorVersionMinimum; + if (currentVersion < minimumVersion) + { + _log.Debug($"Firefox version ({currentVersion}) is incompatible - requires at least {minimumVersion}"); + return false; + } + return true; + } + + internal static string GetInstallLocation() + { + var path = "/bin/firefox"; + if (File.Exists(path)) + { + return path; + } - options.Profile.SetPreference("permissions.default.cookies", 2); - options.Profile.SetPreference("permissions.default.popups", 2); - options.Profile.SetPreference("permissions.default.geolocation", 2); - options.Profile.SetPreference("permissions.default.media_stream", 2); + path = "/usr/bin/firefox"; + var retVal = File.Exists(path) ? path : Program.Configuration.FirefoxInstallLocation; + _log.Trace($"Using install location of [{retVal}]"); + return retVal; + } - CodePagesEncodingProvider.Instance.GetEncoding(437); - Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); - - Driver = new FirefoxDriver(options); + private bool FirefoxEx(TimelineHandler handler) + { + try + { + Driver = GetDriver(handler); base.Driver = Driver; - + JS = (IJavaScriptExecutor)Driver; base.JS = JS; @@ -103,16 +95,89 @@ private bool FirefoxEx(TimelineHandler handler) ExecuteEvents(handler); } } - - ExecuteEvents(handler); + else + { + ExecuteEvents(handler); + } } catch (Exception e) { _log.Debug(e); return false; } - + return true; } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var path = GetInstallLocation(); + + var options = new FirefoxOptions(); + options.BrowserExecutableLocation = path; + options.AddArguments("--disable-infobars"); + options.AddArguments("--disable-extensions"); + options.AddArguments("--disable-notifications"); + + options.Profile = new FirefoxProfile(); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("--headless"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.Profile.SetPreference("permissions.default.stylesheet", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.Profile.SetPreference("permissions.default.image", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.Profile.SetPreference("permissions.default.script", 2); + } + } + + options.Profile.SetPreference("permissions.default.cookies", 2); + options.Profile.SetPreference("permissions.default.popups", 2); + options.Profile.SetPreference("permissions.default.geolocation", 2); + options.Profile.SetPreference("permissions.default.media_stream", 2); + + options.Profile.SetPreference("geo.enabled", false); + options.Profile.SetPreference("geo.prompt.testing", false); + options.Profile.SetPreference("geo.prompt.testing.allow", false); + + _log.Trace("Browser preferences set successfully, getting driver..."); + + CodePagesEncodingProvider.Instance.GetEncoding(437); + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + FirefoxDriver driver; + + try + { + driver = new FirefoxDriver(options); + } + catch (Exception e) + { + _log.Trace("Driver could not be instantiated. Does the proper driver exist? Are you running as a user and not root? Sometimes running the driver directly will uncover the underlaying issue."); + throw; + } + + _log.Trace("Driver instantiated successfully, setting timeouts..."); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + _log.Trace("Driver timeouts set successfully, continuing..."); + return driver; + } } -} +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Program.cs b/src/ghosts.client.linux/Program.cs index bb5d8c34..cabb9ad9 100755 --- a/src/ghosts.client.linux/Program.cs +++ b/src/ghosts.client.linux/Program.cs @@ -7,6 +7,7 @@ using System.Net; using System.Reflection; using System.Threading; +using ghosts.client.linux.Comms; using ghosts.client.linux.Communications; using ghosts.client.linux.Infrastructure; using ghosts.client.linux.timelineManager; diff --git a/src/ghosts.client.linux/Survey/SurveyManager.cs b/src/ghosts.client.linux/Survey/SurveyManager.cs index f139e239..26a83330 100644 --- a/src/ghosts.client.linux/Survey/SurveyManager.cs +++ b/src/ghosts.client.linux/Survey/SurveyManager.cs @@ -7,6 +7,7 @@ using System.Net.NetworkInformation; using System.Text.RegularExpressions; using System.Threading; +using ghosts.client.linux.Comms; using ghosts.client.linux.Communications; using Ghosts.Domain.Code; using Newtonsoft.Json; diff --git a/src/ghosts.client.linux/ghosts.client.linux.csproj b/src/ghosts.client.linux/ghosts.client.linux.csproj index d9baf23c..db5d5c8c 100755 --- a/src/ghosts.client.linux/ghosts.client.linux.csproj +++ b/src/ghosts.client.linux/ghosts.client.linux.csproj @@ -16,7 +16,7 @@ GHOSTS NPC Orchestration Platform - please email ddupdyke@sei.cmu.edu with bugs/requests/other Carnegie Mellon University 2017 NU1701 - net6.0;netcoreapp3.1 + net6.0 diff --git a/src/ghosts.client.linux/nlog.config b/src/ghosts.client.linux/nlog.config index 1ecbc4f1..3cc176ab 100644 --- a/src/ghosts.client.linux/nlog.config +++ b/src/ghosts.client.linux/nlog.config @@ -8,7 +8,7 @@ - + From bd10d5e790f6b2fcb1c9c6804e15093d67760eec Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 8 Dec 2021 11:55:42 -0500 Subject: [PATCH 36/42] updates handler --- .../Handlers/BaseBrowserHandler.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index 52f26851..8e5a5974 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -61,17 +61,17 @@ public void ExecuteEvents(TimelineHandler handler) var i = 0; foreach (var site in timelineEvent.CommandArgs) { - LaunchThread(handler, timelineEvent, site.ToString()); + //LaunchThread(handler, timelineEvent, site.ToString()); - // Task.Factory.StartNew(() => LaunchThread(handler, timelineEvent, site.ToString())); - // Thread.Sleep(5000); - // i++; - // - // if (i >= _taskMax) - // { - // Task.WaitAll(); - // i = 0; - // } + Task.Factory.StartNew(() => LaunchThread(handler, timelineEvent, site.ToString())); + Thread.Sleep(5000); + i++; + + if (i >= _taskMax) + { + Task.WaitAll(); + i = 0; + } } break; case "random": From dd99c47c8f8351d06371eed6b2e56c8843384022 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 8 Dec 2021 12:25:52 -0500 Subject: [PATCH 37/42] Adds logging --- src/ghosts.client.linux/Program.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/ghosts.client.linux/Program.cs b/src/ghosts.client.linux/Program.cs index cabb9ad9..a0a3afea 100755 --- a/src/ghosts.client.linux/Program.cs +++ b/src/ghosts.client.linux/Program.cs @@ -86,6 +86,7 @@ private static void Run(IEnumerable args) //local survey gathers information such as drives, accounts, logs, etc. if (Configuration.Survey.IsEnabled) { + _log.Trace("Survey enabled, initalizing..."); try { Survey.SurveyManager.Run(); @@ -95,18 +96,32 @@ private static void Run(IEnumerable args) _log.Error(exc); } } + else + { + _log.Trace("Survey disabled, continuing."); + } if (Configuration.HealthIsEnabled) { + _log.Trace("Health checks enabled, initalizing..."); var h = new Health.Check(); h.Run(); } + else + { + _log.Trace("Health checks disabled, continuing."); + } if (Configuration.HandlersIsEnabled) { + _log.Trace("Handlers enabled, initalizing..."); var o = new Orchestrator(); o.Run(); } + else + { + _log.Trace("Handling disabed, continuing."); + } new ManualResetEvent(false).WaitOne(); } From 04e35f5e9856ef9bcb8ccf5cafb160a8edfb1a98 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Wed, 8 Dec 2021 13:53:50 -0500 Subject: [PATCH 38/42] Adds example crawl timeline --- .../Sample Timelines/Browser Crawl.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/Ghosts.Client/Sample Timelines/Browser Crawl.json diff --git a/src/Ghosts.Client/Sample Timelines/Browser Crawl.json b/src/Ghosts.Client/Sample Timelines/Browser Crawl.json new file mode 100644 index 00000000..79fdf608 --- /dev/null +++ b/src/Ghosts.Client/Sample Timelines/Browser Crawl.json @@ -0,0 +1,39 @@ +{ + "Status": "Run", + "TimeLineHandlers": [ + { + "HandlerType": "BrowserFirefox", + "HandlerArgs": { + "isheadless": "false", + "blockimages": "false", + "blockstyles": "false", + "blockflash": "false", + "blockscripts": "false", + "stickiness": 100, + "stickiness-depth-min": 5, + "stickiness-depth-max": 10, + "incognito": "true", + "crawl-proxy-local-url": "http://localhost:8080/test-001/record/", + "crawl-tasks-maximum": 2, + "crawl-site-depth": 2 + }, + "Initial": "about:blank", + "UtcTimeOn": "00:00:00", + "UtcTimeOff": "24:00:00", + "Loop": "false", + "TimeLineEvents": [ + { + "Command": "crawl", + "CommandArgs": [ + "http://localhost:8080/test-001/record/https://www.usatoday.com", + "http://localhost:8080/test-001/record/https://www.cnn.com", + "http://localhost:8080/test-001/record/https://www.cmu.edu", + "http://localhost:8080/test-001/record/https://www.espn.com" + ], + "DelayAfter": 5000, + "DelayBefore": 0 + } + ] + } + ] +} From 29696156d59ee4471baeb443b74c9c63c917e3fd Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Thu, 9 Dec 2021 11:08:28 -0500 Subject: [PATCH 39/42] updates readme --- README.md | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5e4399b6..21eb975c 100755 --- a/README.md +++ b/README.md @@ -1,32 +1,42 @@ ![GHOSTS Logo](https://github.com/cmu-sei/GHOSTS/blob/master/assets/ghosts-logo.jpg) -Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - # GHOSTS NPC AUTOMATION -GHOSTS is a framework for highly-complex, realistic non-player character (NPC) orchestration. It essentially realistically mimics the behavior of the different types of people you might encounter on any array of different typical office or enterprise networks. The system makes it possible for cybersecurity experts to test their skills and realistically train to defend real networks with real NPC players operating on those networks doing the things we might expect them to do: Create documents, access systems, browse the web, click, run commands, and so on. +GHOSTS is a framework for highly-complex, realistic non-player character (NPC) orchestration. It essentially realistically mimics the behavior of the different types of people you might encounter on typical office or enterprise networks. The system makes it possible for cybersecurity experts to test their skills and realistically train to defend real networks with real NPC players operating on those networks doing the things we might expect them to do: Create documents, access systems, browse the web, click, run commands, and so on. -As a result of the system checks required in order for NPCs to be situationally aware, GHOSTS also does health reporting for all configured clients on a given instance. +As a result of the system checks required for NPCs to be situationally aware, GHOSTS also does health reporting for all configured clients on a given instance. ## Key Links -[Installation and configuration information is maintained on our wiki](https://github.com/cmu-sei/GHOSTS/wiki) +* [Quick start: Installation from distribution binaries](https://github.com/cmu-sei/GHOSTS/wiki/Installation-from-distribution-binaries) -[Don't hesitate to submit issues and feature requests here](https://github.com/cmu-sei/GHOSTS/issues) +* [Detailed installation and configuration information](https://github.com/cmu-sei/GHOSTS/wiki) + +* [Don't hesitate to submit issues and feature requests](https://github.com/cmu-sei/GHOSTS/issues) ## Platform Components -### Ghosts.Client (Windows) -.NET Console app (but built as forms app so that it is hidden) - requires .NET framework v4.6.1 or higher. Client works on both Windows 7 and Windows 10. +### Ghosts Clients (Windows & Linux) + +GHOSTS clients simulate users on a machine doing "user-like" things. They [can be configured](https://github.com/cmu-sei/GHOSTS/wiki/Configuring-the-Windows-Client) to perform actions including: -### Ghosts.Client (Linux) -dotnetcore app built to run silently. Client tested on centos, alpine and kali distributions. We typically use this for red teaming and "outside" traffic generation or administration simulation. +* Browse the web +* Create and edit office documents +* Send and respond to email +* Run terminal commands +* Etc. -### Ghosts.Api -.NET Core 2.0 MVC Code-First containing both the api calls for the client (and corresponding api calls the ui needs) in one. +### Ghosts API Server -Uses postgres on the backend because there is not much that postgres can't do. +The API server is a RESTful web service that provides a way for clients to interact with the GHOSTS system and its clients. It can: -## LEGAL +* Manage clients, add/remove them from groups, etc. +* Get/manage information from clients with regards to their activity, current activities, etc. +* Orchestrate new activities for particular clients to perform + + +--- [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. + +Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. From c4d1bb736d5359733227d1b252c5a587c00ac0e2 Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Thu, 9 Dec 2021 11:09:14 -0500 Subject: [PATCH 40/42] updates readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 21eb975c..e4fe6ae5 100755 --- a/README.md +++ b/README.md @@ -34,7 +34,6 @@ The API server is a RESTful web service that provides a way for clients to inter * Get/manage information from clients with regards to their activity, current activities, etc. * Orchestrate new activities for particular clients to perform - --- [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. From 2187645024c588560bed4ec69be2e7e9a09e6fbd Mon Sep 17 00:00:00 2001 From: sei-dupdyke Date: Mon, 20 Dec 2021 13:06:57 -0500 Subject: [PATCH 41/42] browser updates --- src/ghosts.client.linux/Communications/Updates.cs | 2 +- src/ghosts.client.linux/Handlers/BrowserChrome.cs | 11 +++++++---- src/ghosts.client.linux/Handlers/BrowserCrawl.cs | 5 +++++ src/ghosts.client.linux/Handlers/BrowserFirefox.cs | 9 ++++++--- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/ghosts.client.linux/Communications/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs index 6e309877..2067bd9d 100644 --- a/src/ghosts.client.linux/Communications/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -338,7 +338,7 @@ internal static void PostSurvey() { posturl = Program.Configuration.Survey.PostUrl; } - catch (Exception exc) + catch { _log.Error("Can't get survey posturl!"); return; diff --git a/src/ghosts.client.linux/Handlers/BrowserChrome.cs b/src/ghosts.client.linux/Handlers/BrowserChrome.cs index a66e3266..8147a8a6 100644 --- a/src/ghosts.client.linux/Handlers/BrowserChrome.cs +++ b/src/ghosts.client.linux/Handlers/BrowserChrome.cs @@ -20,9 +20,12 @@ public BrowserChrome(TimelineHandler handler) { Driver = GetDriver(handler); base.Driver = Driver; - - JS = (IJavaScriptExecutor)Driver; - base.JS = JS; + + if (handler.HandlerArgs.ContainsKey("javascript-enable")) + { + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + } Driver.Navigate().GoToUrl(handler.Initial); @@ -116,7 +119,7 @@ internal static IWebDriver GetDriver(TimelineHandler handler) { driver = new ChromeDriver(options); } - catch (Exception e) + catch { _log.Trace("Driver could not be instantiated. Does the proper driver exist? Are you running as a user and not root? Sometimes running the driver directly will uncover the underlaying issue."); throw; diff --git a/src/ghosts.client.linux/Handlers/BrowserCrawl.cs b/src/ghosts.client.linux/Handlers/BrowserCrawl.cs index c85eae7a..6f51e1e7 100644 --- a/src/ghosts.client.linux/Handlers/BrowserCrawl.cs +++ b/src/ghosts.client.linux/Handlers/BrowserCrawl.cs @@ -97,7 +97,9 @@ private void GetAllLinks(RequestConfiguration config, bool sameSite) { Driver.SwitchTo().Frame(iframe); isInIframe = true; + _log.Trace("replay_iframe found. Made that the focus..."); } + _log.Trace($"Iframe found: {iframe.GetAttribute("id")}"); } var links = Driver.FindElements(By.TagName("a")); @@ -130,7 +132,10 @@ private void GetAllLinks(RequestConfiguration config, bool sameSite) } if (isInIframe) + { Driver.SwitchTo().DefaultContent(); + _log.Trace("Switched back to main window focus"); + } _log.Trace($"Added {linksAdded} links for {config.Uri}"); } diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index 0ac3c42f..5d5e54af 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -70,8 +70,11 @@ private bool FirefoxEx(TimelineHandler handler) Driver = GetDriver(handler); base.Driver = Driver; - JS = (IJavaScriptExecutor)Driver; - base.JS = JS; + if (handler.HandlerArgs.ContainsKey("javascript-enable")) + { + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + } //hack: bad urls used in the past... if (handler.Initial.Equals("") || @@ -168,7 +171,7 @@ internal static IWebDriver GetDriver(TimelineHandler handler) { driver = new FirefoxDriver(options); } - catch (Exception e) + catch { _log.Trace("Driver could not be instantiated. Does the proper driver exist? Are you running as a user and not root? Sometimes running the driver directly will uncover the underlaying issue."); throw; From 0564fadf3dae755b83ba863e0bc9354e9ee454a4 Mon Sep 17 00:00:00 2001 From: Dustin Updyke Date: Wed, 8 Dec 2021 13:53:47 -0500 Subject: [PATCH 42/42] browser cleanup --- src/Ghosts.Client/Comms/Updates.cs | 2 +- src/Ghosts.Client/Handlers/BrowserChrome.cs | 9 ++++++--- src/Ghosts.Client/Handlers/BrowserFirefox.cs | 7 +++++-- src/Ghosts.Client/config/timeline.json | 3 ++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index 34792b6c..a2fb8d13 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -339,7 +339,7 @@ internal static void PostSurvey() { posturl = Program.Configuration.Survey.PostUrl; } - catch (Exception exc) + catch { _log.Error("Can't get survey posturl!"); return; diff --git a/src/Ghosts.Client/Handlers/BrowserChrome.cs b/src/Ghosts.Client/Handlers/BrowserChrome.cs index ab00ff69..ac479fb4 100755 --- a/src/Ghosts.Client/Handlers/BrowserChrome.cs +++ b/src/Ghosts.Client/Handlers/BrowserChrome.cs @@ -33,9 +33,12 @@ public BrowserChrome(TimelineHandler handler) { Driver = GetDriver(handler); base.Driver = Driver; - - JS = (IJavaScriptExecutor)Driver; - base.JS = JS; + + if (handler.HandlerArgs.ContainsKey("javascript-enable")) + { + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + } Driver.Navigate().GoToUrl(handler.Initial); diff --git a/src/Ghosts.Client/Handlers/BrowserFirefox.cs b/src/Ghosts.Client/Handlers/BrowserFirefox.cs index b6592046..3ffba4ff 100755 --- a/src/Ghosts.Client/Handlers/BrowserFirefox.cs +++ b/src/Ghosts.Client/Handlers/BrowserFirefox.cs @@ -66,8 +66,11 @@ private bool FirefoxEx(TimelineHandler handler) Driver = GetDriver(handler); base.Driver = Driver; - JS = (IJavaScriptExecutor)Driver; - base.JS = JS; + if (handler.HandlerArgs.ContainsKey("javascript-enable")) + { + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + } //hack: bad urls used in the past... if (handler.Initial.Equals("") || diff --git a/src/Ghosts.Client/config/timeline.json b/src/Ghosts.Client/config/timeline.json index e0557c6b..dda4103e 100755 --- a/src/Ghosts.Client/config/timeline.json +++ b/src/Ghosts.Client/config/timeline.json @@ -143,7 +143,8 @@ "stickiness": 75, "stickiness-depth-min": 5, "stickiness-depth-max": 10000, - "incognito": "true" + "incognito": "true", + "javascript-enable": "true" }, "Initial": "about:blank", "UtcTimeOn": "00:00:00",