diff --git a/README.md b/README.md index 3e6c7fab..4c6a6cac 100755 --- a/README.md +++ b/README.md @@ -1,32 +1,41 @@ -![GHOSTS Logo](https://github.com/cmu-sei/GHOSTS/blob/master/assets/ghosts-logo.jpg) - -Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - -# GHOSTS NPC AUTOMATION - -GHOSTS is a framework for highly-complex, realistic non-player character (NPC) orchestration. It essentially realistically mimics the behavior of the different types of people you might encounter on any array of different typical office or enterprise networks. The system makes it possible for cybersecurity experts to test their skills and realistically train to defend real networks with real NPC players operating on those networks doing the things we might expect them to do: Create documents, access systems, browse the web, click, run commands, and so on. - -As a result of the system checks required in order for NPCs to be situationally aware, GHOSTS also does health reporting for all configured clients on a given instance. - -## Key Links - -[Installation and configuration information is maintained on our wiki](https://github.com/cmu-sei/GHOSTS/wiki) - -[Don't hesitate to submit issues and feature requests here](https://github.com/cmu-sei/GHOSTS/issues) - -## Platform Components - -### Ghosts.Client (Windows) -.NET Console app (but built as forms app so that it is hidden) - requires .NET framework v4.6.1 or higher. Client works on both Windows 7 and Windows 10. - -### Ghosts.Client (Linux) -dotnetcore app built to run silently. Client tested on centos, alpine and kali distributions. We typically use this for red teaming and "outside" traffic generation or administration simulation. - -### Ghosts.Api -Dotnetcore API containing both the api calls for the client (and corresponding api calls you need for integration into other systems) in one. - -Uses postgres on the backend because there is not much that postgres can't do. - -## LEGAL - -[DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. +![GHOSTS Logo](https://github.com/cmu-sei/GHOSTS/blob/master/assets/ghosts-logo.jpg) + +# GHOSTS NPC AUTOMATION + +GHOSTS is a framework for highly-complex, realistic non-player character (NPC) orchestration. It essentially realistically mimics the behavior of the different types of people you might encounter on typical office or enterprise networks. The system makes it possible for cybersecurity experts to test their skills and realistically train to defend real networks with real NPC players operating on those networks doing the things we might expect them to do: Create documents, access systems, browse the web, click, run commands, and so on. + +As a result of the system checks required for NPCs to be situationally aware, GHOSTS also does health reporting for all configured clients on a given instance. + +## Key Links + +* [Quick start: Installation from distribution binaries](https://github.com/cmu-sei/GHOSTS/wiki/Installation-from-distribution-binaries) + +* [Detailed installation and configuration information](https://github.com/cmu-sei/GHOSTS/wiki) + +* [Don't hesitate to submit issues and feature requests](https://github.com/cmu-sei/GHOSTS/issues) + +## Platform Components + +### Ghosts Clients (Windows & Linux) + +GHOSTS clients simulate users on a machine doing "user-like" things. They [can be configured](https://github.com/cmu-sei/GHOSTS/wiki/Configuring-the-Windows-Client) to perform actions including: + +* Browse the web +* Create and edit office documents +* Send and respond to email +* Run terminal commands +* Etc. + +### Ghosts API Server + +The API server is a RESTful web service that provides a way for clients to interact with the GHOSTS system and its clients. It can: + +* Manage clients, add/remove them from groups, etc. +* Get/manage information from clients with regards to their activity, current activities, etc. +* Orchestrate new activities for particular clients to perform + +--- + +[DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. + +Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. diff --git a/src/Dockerfile-api b/src/Dockerfile-api index 2bcdfffb..fe885e48 100644 --- a/src/Dockerfile-api +++ b/src/Dockerfile-api @@ -5,7 +5,7 @@ # #multi-stage target: dev # -FROM mcr.microsoft.com/dotnet/sdk:5.0-alpine AS dev +FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS dev ENV ASPNETCORE_URLS=http://*:5000 \ ASPNETCORE_ENVIRONMENT=DEVELOPMENT @@ -20,14 +20,14 @@ CMD ["dotnet", "run"] # #multi-stage target: prod # -FROM mcr.microsoft.com/dotnet/aspnet:5.0-alpine AS prod +FROM mcr.microsoft.com/dotnet/aspnet:6.0-alpine AS prod ARG commit ENV COMMIT=$commit COPY --from=dev /app/dist /app WORKDIR /app -ENV GHOSTS_VERSION=5.0.0.0 -ENV GHOSTS_API_VERSION=v5 +ENV GHOSTS_VERSION=6.0.0.0 +ENV GHOSTS_API_VERSION=v6 ENV ASPNETCORE_URLS=http://*:5000 EXPOSE 5000 diff --git a/src/Ghosts.Api/Controllers/ClientTimeline.cs b/src/Ghosts.Api/Controllers/ClientTimeline.cs index 6961dae0..93570de2 100644 --- a/src/Ghosts.Api/Controllers/ClientTimeline.cs +++ b/src/Ghosts.Api/Controllers/ClientTimeline.cs @@ -22,10 +22,10 @@ namespace ghosts.api.Controllers public class ClientTimelineController : Controller { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private readonly IMachineTimelineService _service; + private readonly IMachineTimelinesService _service; private readonly IMachineService _machineService; - public ClientTimelineController(IMachineTimelineService service, IMachineService machineService) + public ClientTimelineController(IMachineTimelinesService service, IMachineService machineService) { _service = service; _machineService = machineService; diff --git a/src/Ghosts.Api/Controllers/ClientUpdatesController.cs b/src/Ghosts.Api/Controllers/ClientUpdatesController.cs index aa9a5a42..3d881b33 100755 --- a/src/Ghosts.Api/Controllers/ClientUpdatesController.cs +++ b/src/Ghosts.Api/Controllers/ClientUpdatesController.cs @@ -9,6 +9,7 @@ using Ghosts.Domain; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json.Linq; using NLog; namespace Ghosts.Api.Controllers @@ -86,6 +87,20 @@ public async Task Index(CancellationToken ct) await _updateService.DeleteAsync(u.Id, ct); + // integrators want to know that a timeline was actually delivered + // (the service only guarantees that the update was received) + _queue.Enqueue( + new QueueEntry + { + Payload = + new NotificationQueueEntry() + { + Type = NotificationQueueEntry.NotificationType.TimelineDelivered, + Payload = (JObject) JToken.FromObject(update) + }, + Type = QueueEntry.Types.Notification + }); + return Json(update); } } diff --git a/src/Ghosts.Api/Controllers/SurveysController.cs b/src/Ghosts.Api/Controllers/SurveysController.cs new file mode 100644 index 00000000..14388f10 --- /dev/null +++ b/src/Ghosts.Api/Controllers/SurveysController.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Services; +using Ghosts.Domain.Messages.MesssagesForServer; +using Microsoft.AspNetCore.Mvc; + +namespace ghosts.api.Controllers +{ + public class SurveysController : Controller + { + private readonly ISurveyService _surveyService; + + public SurveysController(ISurveyService surveyService) + { + _surveyService = surveyService; + } + + [ProducesResponseType(typeof(Survey), 200)] + [HttpGet("surveys/{machineId}")] + public async Task Survey([FromRoute] Guid machineId, CancellationToken ct) + { + return Ok(await _surveyService.GetLatestAsync(machineId, ct)); + } + + [ProducesResponseType(typeof(IEnumerable), 200)] + [HttpGet("surveys/{machineId}/all")] + public async Task SurveyAll([FromRoute] Guid machineId, CancellationToken ct) + { + return Ok(await _surveyService.GetAllAsync(machineId, ct)); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Controllers/TimelineController.cs b/src/Ghosts.Api/Controllers/TimelinesController.cs similarity index 58% rename from src/Ghosts.Api/Controllers/TimelineController.cs rename to src/Ghosts.Api/Controllers/TimelinesController.cs index 83285c09..284dd3b5 100644 --- a/src/Ghosts.Api/Controllers/TimelineController.cs +++ b/src/Ghosts.Api/Controllers/TimelinesController.cs @@ -1,14 +1,12 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.Net; using System.Threading; using System.Threading.Tasks; using Ghosts.Api.Models; using Ghosts.Api.Services; using Ghosts.Api.ViewModels; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; using Swashbuckle.AspNetCore.Annotations; namespace ghosts.api.Controllers @@ -16,29 +14,44 @@ namespace ghosts.api.Controllers /// /// Get or update a machine timeline via the API /// - public class TimelineController : Controller + public class TimelinesController : Controller { private readonly ITimelineService _timelineService; - private readonly IMachineTimelineService _machineTimelineService; + private readonly IMachineTimelinesService _machineTimelinesService; - public TimelineController(ITimelineService timelineService, IMachineTimelineService machineTimelineService) + public TimelinesController(ITimelineService timelineService, IMachineTimelinesService machineTimelinesService) { _timelineService = timelineService; - _machineTimelineService = machineTimelineService; + _machineTimelinesService = machineTimelinesService; } /// - /// This returns the timeline for a requested machine. If the timeline is not available, + /// This returns all timelines for a requested machine. If all or a specific timeline is not available, + /// a MachineUpdate request can be made to retrieve the machine timelines + /// + /// Machine Guid + /// Cancellation token + /// MachineTimelines + [ProducesResponseType(typeof(MachineTimeline), 200)] + [HttpGet("timelines/{machineId}")] + public async Task Timeline([FromRoute] Guid machineId, CancellationToken ct) + { + return Ok(await _machineTimelinesService.GetByMachineIdAsync(machineId, ct)); + } + + /// + /// This returns a specific timeline for a requested machine. If the timeline is not available, /// a MachineUpdate request can be made to retrieve the machine timeline /// /// Machine Guid + /// /// Timeline Id Guid /// Cancellation token /// MachineTimeline [ProducesResponseType(typeof(MachineTimeline), 200)] - [HttpGet("timeline/{machineId}")] - public async Task Timeline([FromRoute] Guid machineId, CancellationToken ct) + [HttpGet("timelines/{machineId}/{timelineId}")] + public async Task TimelineById([FromRoute] Guid machineId, [FromRoute] Guid timelineId, CancellationToken ct) { - return Ok(await _machineTimelineService.GetByMachineIdAsync(machineId, ct)); + return Ok(await _machineTimelinesService.GetByMachineIdAndTimelineIdAsync(machineId, timelineId, ct)); } /// @@ -47,7 +60,7 @@ public async Task Timeline([FromRoute] Guid machineId, Cancellati /// The update to send /// Cancellation token /// 204 No content - [HttpPost("timeline")] + [HttpPost("timelines")] // [ProducesResponseType(typeof(Task), (int) HttpStatusCode.NoContent)] Swagger hates this https://stackoverflow.com/questions/35605427/swagger-ui-freezes-after-api-fetch-and-browser-crashes [SwaggerOperation(OperationId = "createTimeline")] public async Task Timeline([FromBody] MachineUpdateViewModel machineUpdate, CancellationToken ct) @@ -55,6 +68,14 @@ public async Task Timeline([FromBody] MachineUpdateViewModel mach await _timelineService.UpdateAsync(machineUpdate, ct); return NoContent(); } + + [HttpPost("timelines/{machineId}/{timelineId}/stop")] + [SwaggerOperation(OperationId = "stopTimeline")] + public async Task Timeline([FromRoute] Guid machineId, [FromRoute] Guid timelineId, CancellationToken ct) + { + await _timelineService.StopAsync(machineId, timelineId, ct); + return NoContent(); + } /// /// Send a new timeline to an entire group of machines @@ -63,7 +84,7 @@ public async Task Timeline([FromBody] MachineUpdateViewModel mach /// The update to send /// Cancellation token /// 204 No content - [HttpPost("timeline/bygroup/{groupId}")] + [HttpPost("timelines/bygroup/{groupId}")] // [ProducesResponseType(typeof(Task), (int) HttpStatusCode.NoContent)] Swagger hates this [SwaggerOperation(OperationId = "createTimelineForGroup")] public async Task GroupTimeline([FromRoute] int groupId, [FromBody] MachineUpdateViewModel machineUpdate, CancellationToken ct) diff --git a/src/Ghosts.Api/Controllers/TrackablesController.cs b/src/Ghosts.Api/Controllers/TrackablesController.cs new file mode 100644 index 00000000..e3c0e64d --- /dev/null +++ b/src/Ghosts.Api/Controllers/TrackablesController.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Services; +using Microsoft.AspNetCore.Mvc; + +namespace ghosts.api.Controllers +{ + [Produces("application/json")] + [Route("api/[controller]")] + [ResponseCache(Duration = 5)] + public class TrackablesController : Controller + { + private readonly ITrackableService _service; + + public TrackablesController(ITrackableService service) + { + _service = service; + } + + /// + /// Gets all trackables in the system + /// + /// Cancellation Token + /// List of Trackables + [HttpGet] + public async Task GetTrackables(CancellationToken ct) + { + var list = await _service.GetAsync(ct); + if (list == null) return NotFound(); + return Ok(list); + } + + [HttpGet("{id}")] + public async Task GetTrackableHistory([FromRoute] Guid id, CancellationToken ct) + { + var list = await _service.GetActivityByTrackableId(id, ct); + if (list == null) return NotFound(); + return Ok(list); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Models/QueueEntries.cs b/src/Ghosts.Api/Models/QueueEntries.cs index 39a891f0..d7247ce5 100644 --- a/src/Ghosts.Api/Models/QueueEntries.cs +++ b/src/Ghosts.Api/Models/QueueEntries.cs @@ -30,7 +30,8 @@ public class NotificationQueueEntry public enum NotificationType { Timeline = 0, - WebhookCreate = 1 + WebhookCreate = 1, + TimelineDelivered = 10 } public NotificationType Type { get; set; } diff --git a/src/Ghosts.Api/Models/WebHook.cs b/src/Ghosts.Api/Models/WebHook.cs index b3d354d1..a1992f59 100755 --- a/src/Ghosts.Api/Models/WebHook.cs +++ b/src/Ghosts.Api/Models/WebHook.cs @@ -27,8 +27,7 @@ public Webhook() public Webhook(WebhookViewModel model) { - var id = Guid.NewGuid(); - if (Guid.TryParse(model.Id, out id)) + if (Guid.TryParse(model.Id, out var id)) Id = id; Status = model.Status; Description = model.Description; diff --git a/src/Ghosts.Api/Services/MachineTimelineService.cs b/src/Ghosts.Api/Services/MachineTimelineService.cs index f8d758e1..588eddfd 100755 --- a/src/Ghosts.Api/Services/MachineTimelineService.cs +++ b/src/Ghosts.Api/Services/MachineTimelineService.cs @@ -1,37 +1,55 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Ghosts.Api.Infrastructure.Data; using Ghosts.Api.Models; using Ghosts.Domain; +using Ghosts.Domain.Code; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using NLog; namespace Ghosts.Api.Services { - public interface IMachineTimelineService + public interface IMachineTimelinesService { - Task GetByMachineIdAsync(Guid id, CancellationToken ct); + Task> GetByMachineIdAsync(Guid id, CancellationToken ct); + Task GetByMachineIdAndTimelineIdAsync(Guid id, Guid timelineId, CancellationToken ct); Task CreateAsync(Machine model, Timeline timeline, CancellationToken ct); Task DeleteByMachineIdAsync(Guid model, CancellationToken ct); } - public class MachineTimelineService : IMachineTimelineService + public class MachineTimelinesService : IMachineTimelinesService { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private readonly ApplicationDbContext _context; - public MachineTimelineService(ApplicationDbContext context) + public MachineTimelinesService(ApplicationDbContext context) { _context = context; } - public async Task GetByMachineIdAsync(Guid id, CancellationToken ct) + public async Task> GetByMachineIdAsync(Guid id, CancellationToken ct) { - return await _context.MachineTimelines.FirstOrDefaultAsync(x => x.MachineId == id, cancellationToken: ct); + return await _context.MachineTimelines.Where(x => x.MachineId == id).ToListAsync(ct); + } + + public async Task GetByMachineIdAndTimelineIdAsync(Guid id, Guid timelineId, CancellationToken ct) + { + var timelines = await _context.MachineTimelines.Where(x => x.MachineId == id).ToListAsync(ct); + foreach (var timeline in timelines) + { + var t = TimelineBuilder.StringToTimeline(timeline.Timeline); + if (t.Id == timelineId) + return timeline; + } + + return null; } public async Task CreateAsync(Machine model, Timeline timeline, CancellationToken ct) diff --git a/src/Ghosts.Api/Services/QueueSyncService.cs b/src/Ghosts.Api/Services/QueueSyncService.cs index 5ec854d1..845e7743 100755 --- a/src/Ghosts.Api/Services/QueueSyncService.cs +++ b/src/Ghosts.Api/Services/QueueSyncService.cs @@ -71,18 +71,18 @@ private async Task Sync() { using var scope = _scopeFactory.CreateScope(); await using var context = scope.ServiceProvider.GetRequiredService(); - + foreach (var item in Queue.GetAll()) switch (item.Type) { case QueueEntry.Types.Machine: - await ProcessMachine(scope, context, (MachineQueueEntry) item.Payload); + await ProcessMachine(scope, context, (MachineQueueEntry)item.Payload); break; case QueueEntry.Types.Notification: - await ProcessNotification(context, (NotificationQueueEntry) item.Payload); + await ProcessNotification(context, (NotificationQueueEntry)item.Payload); break; case QueueEntry.Types.Survey: - await ProcessSurvey(context, (Survey) item.Payload); + await ProcessSurvey(context, (Survey)item.Payload); break; default: throw new ArgumentOutOfRangeException(); @@ -115,7 +115,7 @@ private async Task ProcessNotification(ApplicationDbContext context, Notificatio foreach (var webhook in webhooks) { - var t = new Thread(() => { HandleWebhook(webhook, item); }) {IsBackground = true}; + var t = new Thread(() => { HandleWebhook(webhook, item); }) { IsBackground = true }; t.Start(); } @@ -128,73 +128,6 @@ private async Task ProcessNotification(ApplicationDbContext context, Notificatio } } - internal static async void HandleWebhook(Webhook webhook, NotificationQueueEntry payload) - { - var historyTimeline = JsonConvert.DeserializeObject(payload.Payload.ToString()); - // Serialize our concrete class into a JSON String - - var formattedResponse = webhook.PostbackFormat; - - var isValid = false; - var reg = new Regex(@"\[(.*?)\]"); - foreach (Match match in reg.Matches(formattedResponse)) - switch (match.Value.ToLower()) - { - case "[machinename]": - formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.MachineId.ToString()); - break; - case "[datetime.utcnow]": - //json formatted date! - formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.CreatedUtc.ToString("s")); - break; - case "[messagetype]": - formattedResponse = formattedResponse.Replace(match.Value, "Binary"); - break; - case "[messagepayload]": - if (payload.Payload["Result"] != null && !string.IsNullOrEmpty(payload.Payload["Result"].ToString())) - { - var p = payload.Payload["Result"].ToString().Trim().Trim('"').Trim().Trim('"'); - - p = $"\"{HttpUtility.JavaScriptStringEncode(p)}\""; - - formattedResponse = formattedResponse.Replace(match.Value, p); - isValid = true; - } - - break; - } - - if (!isValid) - { - log.Trace("Webhook has no payload, exiting"); - return; - } - - try - { - // Wrap our JSON inside a StringContent which then can be used by the HttpClient class - var httpContent = new StringContent(formattedResponse, Encoding.UTF8, "application/json"); - - using var httpClient = new HttpClient(); - // Do the actual request and await the response - var httpResponse = await httpClient.PostAsync(webhook.PostbackUrl, httpContent); - - log.Trace($"Webhook response {webhook.PostbackUrl} {webhook.PostbackMethod} {httpResponse.StatusCode}"); - - // If the response contains content we want to read it! - if (httpResponse.Content != null) - { - var responseContent = await httpResponse.Content.ReadAsStringAsync(); - log.Trace($"Webhook notification sent with {responseContent}"); - // From here on you could deserialize the ResponseContent back again to a concrete C# type using Json.Net - } - } - catch (Exception e) - { - log.Trace($"Webhook failed response {webhook.PostbackUrl} {webhook.PostbackMethod} - {e}"); - } - } - private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext context, MachineQueueEntry item) { var service = scope.ServiceProvider.GetRequiredService(); @@ -261,7 +194,7 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont if (item.LogDump.Log.Length > 0) log.Trace(item.LogDump.Log); - var lines = item.LogDump.Log.Split(new[] {Environment.NewLine}, StringSplitOptions.None); + var lines = item.LogDump.Log.Split(new[] { Environment.NewLine }, StringSplitOptions.None); foreach (var line in lines) try { @@ -307,7 +240,7 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont }; if (data.Tags != null) - timeline.Tags = data.Tags; + timeline.Tags = data.Tags; if (data.Result != null) timeline.Result = data.Result; @@ -323,7 +256,7 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont new NotificationQueueEntry { Type = NotificationQueueEntry.NotificationType.Timeline, - Payload = (JObject) JToken.FromObject(timeline) + Payload = (JObject)JToken.FromObject(timeline) } }); @@ -481,5 +414,91 @@ private async Task ProcessMachine(IServiceScope scope, ApplicationDbContext cont } } } + + internal static async void HandleWebhook(Webhook webhook, NotificationQueueEntry payload) + { + string formattedResponse; + if (payload.Type == NotificationQueueEntry.NotificationType.TimelineDelivered) + { + formattedResponse = payload.Payload.ToString(); + } + else + { + var historyTimeline = JsonConvert.DeserializeObject(payload.Payload.ToString()); + // Serialize our concrete class into a JSON String + + formattedResponse = webhook.PostbackFormat; + + var isValid = false; + var reg = new Regex(@"\[(.*?)\]"); + foreach (Match match in reg.Matches(formattedResponse)) + switch (match.Value.ToLower()) + { + case "[machinename]": + formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.MachineId.ToString()); + break; + case "[datetime.utcnow]": + //json formatted date! + formattedResponse = formattedResponse.Replace(match.Value, historyTimeline.CreatedUtc.ToString("s")); + break; + case "[messagetype]": + formattedResponse = formattedResponse.Replace(match.Value, "Binary"); + break; + case "[messagepayload]": + if (payload.Payload["Result"] != null && !string.IsNullOrEmpty(payload.Payload["Result"].ToString())) + { + var p = payload.Payload["Result"].ToString().Trim().Trim('"').Trim().Trim('"'); + + p = $"\"{HttpUtility.JavaScriptStringEncode(p)}\""; + + formattedResponse = formattedResponse.Replace(match.Value, p); + isValid = true; + } + + break; + } + + if (!isValid) + { + log.Trace("Webhook has no payload, exiting"); + return; + } + } + + try + { + // Wrap our JSON inside a StringContent which then can be used by the HttpClient class + var httpContent = new StringContent(formattedResponse, Encoding.UTF8, "application/json"); + + using var httpClient = new HttpClient(); + // Do the actual request and await the response + HttpResponseMessage httpResponse; + switch (webhook.PostbackMethod) + { + default: + throw new ArgumentException("webhook configuration encountered unspecified postback method"); + case Webhook.WebhookMethod.POST: + httpResponse = await httpClient.PostAsync(webhook.PostbackUrl, httpContent); + break; + case Webhook.WebhookMethod.GET: + httpResponse = await httpClient.GetAsync($"{webhook.PostbackUrl}?message={formattedResponse}"); + break; + } + + log.Trace($"Webhook response {webhook.PostbackUrl} {webhook.PostbackMethod} {httpResponse.StatusCode}"); + + // If the response contains content we want to read it! + if (httpResponse.Content != null) + { + var responseContent = await httpResponse.Content.ReadAsStringAsync(); + log.Trace($"Webhook notification sent with {responseContent}"); + // From here on you could deserialize the ResponseContent back again to a concrete C# type using Json.Net + } + } + catch (Exception e) + { + log.Trace($"Webhook failed response {webhook.PostbackUrl} {webhook.PostbackMethod} - {e}"); + } + } } } \ No newline at end of file diff --git a/src/Ghosts.Api/Services/SurveyService.cs b/src/Ghosts.Api/Services/SurveyService.cs new file mode 100644 index 00000000..5f7aff73 --- /dev/null +++ b/src/Ghosts.Api/Services/SurveyService.cs @@ -0,0 +1,57 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Infrastructure.Data; +using Ghosts.Domain.Messages.MesssagesForServer; +using Microsoft.EntityFrameworkCore; + +namespace Ghosts.Api.Services +{ + public interface ISurveyService + { + Task GetLatestAsync(Guid machineId, CancellationToken ct); + Task> GetAllAsync(Guid machineId, CancellationToken ct); + } + + public class SurveyService : ISurveyService + { + private readonly ApplicationDbContext _context; + + public SurveyService(ApplicationDbContext context) + { + _context = context; + } + + public async Task GetLatestAsync(Guid machineId, CancellationToken ct) + { + return await _context.Surveys + .Include(x=>x.Drives) + .Include("Interfaces.Bindings") + .Include(x=>x.Ports) + .Include(x=>x.Processes) + .Include(x=>x.EventLogs) + .Include(x=>x.LocalUsers) + .Where(x=>x.MachineId == machineId) + .OrderByDescending(x => x.Created) + .FirstOrDefaultAsync(ct); + } + + public async Task> GetAllAsync(Guid machineId, CancellationToken ct) + { + return await _context.Surveys + .Include(x=>x.Drives) + .Include("Interfaces.Bindings") + .Include(x=>x.Ports) + .Include(x=>x.Processes) + .Include(x=>x.EventLogs) + .Include(x=>x.LocalUsers) + .Where(x=>x.MachineId == machineId) + .OrderByDescending(x => x.Created) + .ToArrayAsync(ct); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Services/TimelineService.cs b/src/Ghosts.Api/Services/TimelineService.cs index 4bb6bb8a..d8389ba8 100644 --- a/src/Ghosts.Api/Services/TimelineService.cs +++ b/src/Ghosts.Api/Services/TimelineService.cs @@ -1,11 +1,16 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Ghosts.Api.Infrastructure.Data; +using Ghosts.Api.Models; using Ghosts.Api.ViewModels; +using Ghosts.Domain; using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; namespace Ghosts.Api.Services { @@ -13,6 +18,7 @@ public interface ITimelineService { Task UpdateAsync(MachineUpdateViewModel machineUpdate, CancellationToken ct); Task UpdateGroupAsync(int groupId, MachineUpdateViewModel machineUpdate, CancellationToken ct); + Task StopAsync(Guid machineId, Guid timelineId, CancellationToken ct); } public class TimelineService : ITimelineService @@ -49,5 +55,42 @@ public async Task UpdateGroupAsync(int groupId, MachineUpdateViewModel machineUp await _context.SaveChangesAsync(ct); } + + public async Task StopAsync(Guid machineId, Guid timelineId, CancellationToken ct) + { + var timelineEvent = new TimelineEvent + { + Command = "stop" + }; + + var handler = new TimelineHandler + { + HandlerType = HandlerType.NpcSystem + }; + handler.TimeLineEvents.Add(timelineEvent); + + var handlers = new List(); + handlers.Add(handler); + + var timeline = new Timeline + { + Id = timelineId, + Status = Timeline.TimelineStatus.Run, + TimeLineHandlers = handlers + }; + + var o = new MachineUpdate + { + Status = StatusType.Active, + Update = JsonConvert.SerializeObject(timeline), + ActiveUtc = DateTime.UtcNow, + CreatedUtc = DateTime.UtcNow, + MachineId = machineId, + Type = UpdateClientConfig.UpdateType.TimelinePartial + }; + + await _context.MachineUpdates.AddAsync(o, ct); + await _context.SaveChangesAsync(ct); + } } } \ No newline at end of file diff --git a/src/Ghosts.Api/Services/TrackableService.cs b/src/Ghosts.Api/Services/TrackableService.cs new file mode 100755 index 00000000..6a3287ae --- /dev/null +++ b/src/Ghosts.Api/Services/TrackableService.cs @@ -0,0 +1,42 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Ghosts.Api.Infrastructure.Data; +using Ghosts.Api.Models; +using Microsoft.EntityFrameworkCore; +using NLog; + +namespace Ghosts.Api.Services +{ + public interface ITrackableService + { + Task> GetAsync(CancellationToken ct); + Task> GetActivityByTrackableId(Guid trackableId, CancellationToken ct); + } + + public class TrackableService : ITrackableService + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private readonly ApplicationDbContext _context; + private readonly int _lookBack = Program.ClientConfig.LookbackRecords; + + public TrackableService(ApplicationDbContext context) + { + _context = context; + } + + public async Task> GetAsync(CancellationToken ct) + { + return await _context.Trackables.ToListAsync(ct); + } + + public async Task> GetActivityByTrackableId(Guid trackableId, CancellationToken ct) + { + return await _context.HistoryTrackables.Where(o => o.TrackableId == trackableId).ToListAsync(ct); + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Api/Startup.cs b/src/Ghosts.Api/Startup.cs index 34caeca0..57c579ae 100755 --- a/src/Ghosts.Api/Startup.cs +++ b/src/Ghosts.Api/Startup.cs @@ -21,7 +21,7 @@ namespace Ghosts.Api { public class Startup { - public const int apiVersion = 5; + public const int apiVersion = 6; public Startup(IConfiguration configuration) { @@ -67,7 +67,9 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Ghosts.Api/docker-compose.yml b/src/Ghosts.Api/docker-compose.yml index 25e309b6..318fe1f9 100644 --- a/src/Ghosts.Api/docker-compose.yml +++ b/src/Ghosts.Api/docker-compose.yml @@ -38,7 +38,7 @@ services: depends_on: - postgres ports: - - '5000:5000' + - '52388:5000' networks: - ghosts-network restart: always diff --git a/src/Ghosts.Api/ghosts.api.csproj b/src/Ghosts.Api/ghosts.api.csproj index e602b19f..21c338f4 100644 --- a/src/Ghosts.Api/ghosts.api.csproj +++ b/src/Ghosts.Api/ghosts.api.csproj @@ -1,12 +1,12 @@  - net5.0 + net6.0 ghosts.api - 5.0.0.0 - 5.0.0.0 - 5.0.0.0 - 5.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 false Dustin Updyke for Carnegie Mellon University diff --git a/src/Ghosts.Client/Comms/CheckId.cs b/src/Ghosts.Client/Comms/CheckId.cs index 9223861a..c26195c6 100755 --- a/src/Ghosts.Client/Comms/CheckId.cs +++ b/src/Ghosts.Client/Comms/CheckId.cs @@ -5,7 +5,6 @@ using Ghosts.Domain.Code; using NLog; using System; -using System.Diagnostics; using System.IO; using System.Net; @@ -40,7 +39,7 @@ public static string Id } catch { - _log.Error("config file could not be opened"); + _log.Error("No ID file"); return string.Empty; } } @@ -64,30 +63,38 @@ private static string Run() var machine = new ResultMachine(); GuestInfoVars.Load(machine); - - //call home - using (WebClient client = WebClientBuilder.BuildNoId(machine)) + + try { - try + //call home + using (var client = WebClientBuilder.BuildNoId(machine)) { - using (StreamReader reader = - new StreamReader(client.OpenRead(Program.Configuration.IdUrl))) + try { - s = reader.ReadToEnd(); - _log.Debug($"{DateTime.Now} - Received client ID"); + using (var reader = + new StreamReader(client.OpenRead(Program.Configuration.IdUrl))) + { + s = reader.ReadToEnd(); + _log.Debug("ID Received"); + } } - } - catch (WebException wex) - { - if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + catch (WebException wex) { - _log.Debug("No ID returned!", wex); + if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + { + _log.Debug("No ID returned!", wex.Message); + } + } + catch (Exception e) + { + _log.Error($"General comms exception: {e.Message}"); } } - catch (Exception e) - { - _log.Error(e); - } + } + catch (Exception e) + { + _log.Error($"Cannot connect to API: {e.Message}"); + return string.Empty; } s = s.Replace("\"", ""); diff --git a/src/Ghosts.Client/Comms/Updates.cs b/src/Ghosts.Client/Comms/Updates.cs index 964de6a0..a2fb8d13 100755 --- a/src/Ghosts.Client/Comms/Updates.cs +++ b/src/Ghosts.Client/Comms/Updates.cs @@ -1,7 +1,9 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Text; using System.Threading; @@ -71,14 +73,18 @@ private static void GetServerUpdates() } catch (WebException wex) { - if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + if (wex?.Response == null) + { + _log.Debug($"{DateTime.Now} - API Server appears to be not responding"); + } + else if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) { _log.Debug($"{DateTime.Now} - No new configuration found"); } } catch (Exception e) { - _log.Error(e); + _log.Error($"Exception in connecting to server: {e.Message}"); } } @@ -89,7 +95,7 @@ private static void GetServerUpdates() switch (update.Type) { case UpdateClientConfig.UpdateType.RequestForTimeline: - PostCurrentTimeline(); + PostCurrentTimeline(update); break; case UpdateClientConfig.UpdateType.Timeline: TimelineBuilder.SetLocalTimeline(update.Update.ToString()); @@ -112,7 +118,7 @@ private static void GetServerUpdates() } var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) @@ -150,8 +156,30 @@ private static void GetServerUpdates() } } - private static void PostCurrentTimeline() + private static void PostCurrentTimeline(UpdateClientConfig update) { + // is the config for a specific timeline id? + var timelineId = TimelineUpdateClientConfigManager.GetConfigUpdateTimelineId(update); + + // get all timelines + var localTimelines = Domain.Code.TimelineManager.GetLocalTimelines(); + + var timelines = localTimelines as Timeline[] ?? localTimelines.ToArray(); + if (timelineId != Guid.Empty) + { + foreach (var timeline in timelines) + { + if (timeline.Id == timelineId) + { + timelines = new List() + { + timeline + }.ToArray(); + break; + } + } + } + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; var posturl = string.Empty; @@ -160,32 +188,36 @@ private static void PostCurrentTimeline() { posturl = Program.Configuration.IdUrl.Replace("clientid", "clienttimeline"); } - catch (Exception exc) + catch { _log.Error("Can't get timeline posturl!"); return; } - try + foreach (var timeline in timelines) { - _log.Trace("posting timeline"); + try + { + _log.Trace("posting timeline"); - var payload = File.ReadAllText(ApplicationDetails.ConfigurationFiles.Timeline); - var machine = new ResultMachine(); - GuestInfoVars.Load(machine); + var payload = TimelineBuilder.TimelineToString(timeline); + var machine = new ResultMachine(); + GuestInfoVars.Load(machine); - using (var client = WebClientBuilder.Build(machine)) + using (var client = WebClientBuilder.Build(machine)) + { + client.Headers[HttpRequestHeader.ContentType] = "application/json"; + client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + } + + _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); + } + catch (Exception e) { - client.Headers[HttpRequestHeader.ContentType] = "application/json"; - client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + _log.Debug( + $"Problem posting timeline to server from {ApplicationDetails.ConfigurationFiles.Timeline} to {posturl}"); + _log.Error(e); } - - _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); - } - catch (Exception e) - { - _log.Debug($"Problem posting timeline to server from { ApplicationDetails.ConfigurationFiles.Timeline } to { posturl }"); - _log.Error(e); } } @@ -220,7 +252,7 @@ private static void PostClientResults() } catch (Exception e) { - _log.Error($"Problem posting logs to server {e}"); + _log.Error($"Problem posting logs to server: {e.Message}"); } finally { @@ -307,7 +339,7 @@ internal static void PostSurvey() { posturl = Program.Configuration.Survey.PostUrl; } - catch (Exception exc) + catch { _log.Error("Can't get survey posturl!"); return; diff --git a/src/Ghosts.Client/Ghosts.Client.csproj b/src/Ghosts.Client/Ghosts.Client.csproj index c66856d5..05a6488d 100755 --- a/src/Ghosts.Client/Ghosts.Client.csproj +++ b/src/Ghosts.Client/Ghosts.Client.csproj @@ -68,6 +68,26 @@ false + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + ..\packages\CommandLineParser.2.8.0\lib\net461\CommandLine.dll @@ -526,6 +546,7 @@ + @@ -611,7 +632,6 @@ - diff --git a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs index 965265f6..1d769f54 100755 --- a/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs +++ b/src/Ghosts.Client/Handlers/BaseBrowserHandler.cs @@ -2,9 +2,11 @@ using System; using System.Threading; +using System.Threading.Tasks; using Ghosts.Client.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; using OpenQA.Selenium; using OpenQA.Selenium.Interactions; @@ -15,10 +17,18 @@ public abstract class BaseBrowserHandler : BaseHandler { public static readonly Logger _log = LogManager.GetCurrentClassLogger(); public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } public HandlerType BrowserType { get; set; } private int _stickiness = 0; private int _depthMin = 1; private int _depthMax = 10; + private LinkManager _linkManager; + + private Task LaunchThread(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + var o = new BrowserCrawl(); + return o.Crawl(handler, timelineEvent, site); + } public void ExecuteEvents(TimelineHandler handler) { @@ -40,6 +50,27 @@ public void ExecuteEvents(TimelineHandler handler) switch (timelineEvent.Command) { + case "crawl": + var _taskMax = 1; + if (handler.HandlerArgs.ContainsKey("crawl-tasks-maximum")) + { + int.TryParse(handler.HandlerArgs["crawl-tasks-maximum"], out _taskMax); + } + + var i = 0; + foreach (var site in timelineEvent.CommandArgs) + { + Task.Factory.StartNew(() => LaunchThread(handler, timelineEvent, site.ToString())); + Thread.Sleep(5000); + i++; + + if (i >= _taskMax) + { + Task.WaitAll(); + i = 0; + } + } + break; case "random": // setup @@ -80,32 +111,9 @@ public void ExecuteEvents(TimelineHandler handler) { try { - var linkManager = new LinkManager(config.GetHost()); - - - //get all links - var links = Driver.FindElements(By.TagName("a")); - foreach (var l in links) - { - var node = l.GetAttribute("href"); - if (string.IsNullOrEmpty(node) || - node.ToLower().StartsWith("//")) - { - //skip, these seem ugly - } - // http|s links - else if (node.ToLower().StartsWith("http")) - { - linkManager.AddLink(node.ToLower()); - } - // relative links - prefix the scheme and host - else - { - linkManager.AddLink($"{config.GetHost()}{node.ToLower()}"); - } - } - - var link = linkManager.Choose(); + this._linkManager = new LinkManager(config.Uri); + GetAllLinks(config, false); + var link = this._linkManager.Choose(); if (link == null) { return; @@ -157,15 +165,33 @@ public void ExecuteEvents(TimelineHandler handler) actions.SendKeys(element, timelineEvent.CommandArgs[1].ToString()).Build().Perform(); break; case "click": + case "click.by.name": element = Driver.FindElement(By.Name(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; case "clickbyid": + case "click.by.id": element = Driver.FindElement(By.Id(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; + case "click.by.linktext": + element = Driver.FindElement(By.LinkText(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "click.by.cssselector": + element = Driver.FindElement(By.CssSelector(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "js.executescript": + JS.ExecuteScript(timelineEvent.CommandArgs[0].ToString()); + break; + case "manage.window.size": + Driver.Manage().Window.Size = new System.Drawing.Size(Convert.ToInt32(timelineEvent.CommandArgs[0]), Convert.ToInt32(timelineEvent.CommandArgs[1])); + break; } if (timelineEvent.DelayAfter > 0) @@ -180,6 +206,38 @@ public void ExecuteEvents(TimelineHandler handler) } } + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + try + { + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + this._linkManager.AddLink(uri, 1); + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + } + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + } + private void MakeRequest(RequestConfiguration config) { // Added try here because some versions of FF (v56) throw an exception for an unresolved site, diff --git a/src/Ghosts.Client/Handlers/BrowserChrome.cs b/src/Ghosts.Client/Handlers/BrowserChrome.cs index 6d26af02..ac479fb4 100755 --- a/src/Ghosts.Client/Handlers/BrowserChrome.cs +++ b/src/Ghosts.Client/Handlers/BrowserChrome.cs @@ -5,12 +5,16 @@ using OpenQA.Selenium.Chrome; using System; using System.IO; +using Ghosts.Domain.Code.Helpers; +using OpenQA.Selenium; namespace Ghosts.Client.Handlers { public class BrowserChrome : BaseBrowserHandler { - private string GetInstallLocation() + public new IJavaScriptExecutor JS { get; private set; } + + private static string GetInstallLocation() { var path = @"C:\Program Files\Google\Chrome\Application\chrome.exe"; if (File.Exists(path)) @@ -27,72 +31,15 @@ public BrowserChrome(TimelineHandler handler) BrowserType = HandlerType.BrowserChrome; try { - var options = new ChromeOptions(); - options.AddArguments("disable-infobars"); - options.AddArguments("disable-logging"); - options.AddArguments("--disable-logging"); - options.AddArgument("--log-level=3"); - options.AddArgument("--silent"); - - options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); - options.AddUserProfilePreference("disable-popup-blocking", "true"); - options.BinaryLocation = GetInstallLocation(); - - if (handler.HandlerArgs != null) - { - if (handler.HandlerArgs.ContainsKey("executable-location") && - !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) - { - options.BinaryLocation = handler.HandlerArgs["executable-location"]; - } - - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") - { - options.AddArguments("headless"); - } - - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") - { - options.AddArguments("--incognito"); - } - - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") - { - options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); - } - - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") - { - options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); - } - - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") - { - // ? - } - - if (handler.HandlerArgs.ContainsKey("blockscripts") && - handler.HandlerArgs["blockscripts"] == "true") - { - options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); - } - } - - options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); - options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + Driver = GetDriver(handler); + base.Driver = Driver; - if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + if (handler.HandlerArgs.ContainsKey("javascript-enable")) { - options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; } - Driver = new ChromeDriver(options); - base.Driver = Driver; - Driver.Navigate().GoToUrl(handler.Initial); if (handler.Loop) @@ -117,5 +64,75 @@ public BrowserChrome(TimelineHandler handler) ProcessManager.KillProcessAndChildrenByName(ProcessManager.ProcessNames.ChromeDriver); } } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var options = new ChromeOptions(); + options.AddArguments("disable-infobars"); + options.AddArguments("disable-logging"); + options.AddArguments("--disable-logging"); + options.AddArgument("--log-level=3"); + options.AddArgument("--silent"); + + options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); + options.AddUserProfilePreference("disable-popup-blocking", "true"); + options.BinaryLocation = GetInstallLocation(); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKey("executable-location") && + !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) + { + options.BinaryLocation = handler.HandlerArgs["executable-location"]; + } + + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("headless"); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + // ? + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); + } + } + + options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); + options.AddUserProfilePreference("profile.default_content_setting_values.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + + if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + { + options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + } + + var driver = new ChromeDriver(options); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + return driver; + } } } diff --git a/src/Ghosts.Client/Handlers/BrowserCrawl.cs b/src/Ghosts.Client/Handlers/BrowserCrawl.cs new file mode 100644 index 00000000..931c987b --- /dev/null +++ b/src/Ghosts.Client/Handlers/BrowserCrawl.cs @@ -0,0 +1,214 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Ghosts.Client.Infrastructure.Browser; +using Ghosts.Domain; +using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; +using NLog; +using OpenQA.Selenium; + +namespace Ghosts.Client.Handlers +{ + class BrowserCrawl : BaseHandler + { + public static readonly Logger _log = LogManager.GetCurrentClassLogger(); + public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } + private int _stickiness = 0; + private LinkManager _linkManager; + private int _pageBrowseCount = 0; + private string _proxyLocalUrl = string.Empty; + private int _siteDepthMax = 1; + private int _siteDepthCurrent = 0; + + internal Task Crawl(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + switch (handler.HandlerType) + { + case HandlerType.BrowserChrome: + this.Driver = BrowserChrome.GetDriver(handler); + break; + case HandlerType.BrowserFirefox: + this.Driver = BrowserFirefox.GetDriver(handler); + break; + } + + Console.WriteLine($"{Environment.CurrentManagedThreadId} handle: {Driver.CurrentWindowHandle}"); + + if (handler.HandlerArgs.ContainsKey("stickiness")) + { + int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); + } + + if (handler.HandlerArgs.ContainsKey("crawl-site-depth")) + { + int.TryParse(handler.HandlerArgs["crawl-site-depth"], out _siteDepthMax); + } + + if (handler.HandlerArgs.ContainsKey("crawl-proxy-local-url")) + { + _proxyLocalUrl = handler.HandlerArgs["crawl-proxy-local-url"]; + } + + this._pageBrowseCount = 0; + var config = RequestConfiguration.Load(site); + this._linkManager = new LinkManager(config.Uri); + if (config.Uri.IsWellFormedOriginalString()) + { + MakeRequest(config); + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + this._siteDepthCurrent += 1; + + if (this._siteDepthCurrent >= this._siteDepthMax) + return Task.CompletedTask; + + GetAllLinks(config, true); + CrawlAllLinks(config, handler, timelineEvent, true); + } + + Driver.Close(); + Driver.Quit(); + _log.Trace($"Run complete for {site}"); + return Task.CompletedTask; + } + + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + _log.Trace($"Getting links for {config.Uri}..."); + var linksAdded = 0; + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness}..."); + return; + } + + try + { + var isInIframe = false; + // for use with pywb and proxy scraping + var iframes = Driver.FindElements(By.TagName("iframe")); + foreach (var iframe in iframes) + { + if (iframe.GetAttribute("id") == "replay_iframe") + { + Driver.SwitchTo().Frame(iframe); + isInIframe = true; + } + } + + var links = Driver.FindElements(By.TagName("a")); + + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (isInIframe && !string.IsNullOrEmpty(this._proxyLocalUrl)) + node = this._proxyLocalUrl + node; + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + { + this._linkManager.AddLink(uri, 1); + linksAdded += 1; + } + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + linksAdded += 1; + } + } + } + + if (isInIframe) + Driver.SwitchTo().DefaultContent(); + + _log.Trace($"Added {linksAdded} links for {config.Uri}"); + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, + TimelineEvent timelineEvent, bool sameSite) + { + _log.Trace($"Crawling links for {config.Uri}"); + if (this._linkManager?.Links == null) + { + return; + } + if (this._pageBrowseCount > this._stickiness) + { + return; + } + + foreach (var link in this._linkManager.Links.Where(x => x.WasBrowsed == false) + .OrderByDescending(x => x.Priority)) + { + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (2)..."); + return; + } + + if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) + { + continue; + } + + config.Method = "GET"; + config.Uri = link.Url; + + MakeRequest(config); + + foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) + { + l.WasBrowsed = true; + _log.Trace($"Skipping {config.Uri} (already browsed)"); + } + this._pageBrowseCount += 1; + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (3)..."); + return; + } + + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + + // if this is the last step down, there is no reason to keep digging, + // but we don't increase the current depth count so as to allow peer + // pages at this level to still be scraped + if (this._siteDepthCurrent + 1 < this._siteDepthMax) + { + _log.Trace($"Drilling into {config.Uri}..."); + GetAllLinks(config, sameSite); + CrawlAllLinks(config, handler, timelineEvent, sameSite); + } + } + } + + private void MakeRequest(RequestConfiguration config) + { + try + { + Driver.Navigate().GoToUrl(config.Uri); + } + catch (Exception e) + { + _log.Trace($"Requst error for {config.Uri}: {e.Message}"); + } + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Client/Handlers/BrowserFirefox.cs b/src/Ghosts.Client/Handlers/BrowserFirefox.cs index b7780a57..3ffba4ff 100755 --- a/src/Ghosts.Client/Handlers/BrowserFirefox.cs +++ b/src/Ghosts.Client/Handlers/BrowserFirefox.cs @@ -8,14 +8,16 @@ using System; using System.Diagnostics; using System.IO; +using Ghosts.Domain.Code.Helpers; namespace Ghosts.Client.Handlers { public class BrowserFirefox : BaseBrowserHandler { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private new static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public IWebDriver Driver { get; private set; } + public new IWebDriver Driver { get; private set; } + public new IJavaScriptExecutor JS { get; private set; } public BrowserFirefox(TimelineHandler handler) { @@ -27,7 +29,7 @@ public BrowserFirefox(TimelineHandler handler) } } - private string GetInstallLocation() + private static string GetInstallLocation() { var path = @"C:\Program Files\Mozilla Firefox\firefox.exe"; if (File.Exists(path)) @@ -39,13 +41,13 @@ private string GetInstallLocation() return File.Exists(path) ? path : Program.Configuration.FirefoxInstallLocation; } - private int GetFirefoxVersion(string path) + private static int GetFirefoxVersion(string path) { var versionInfo = FileVersionInfo.GetVersionInfo(path); return versionInfo.FileMajorPart; } - private bool IsSufficientVersion(string path) + private static bool IsSufficientVersion(string path) { int currentVersion = GetFirefoxVersion(path); int minimumVersion = Program.Configuration.FirefoxMajorVersionMinimum; @@ -61,66 +63,23 @@ private bool FirefoxEx(TimelineHandler handler) { try { - var path = GetInstallLocation(); + Driver = GetDriver(handler); + base.Driver = Driver; - if (!IsSufficientVersion(path)) + if (handler.HandlerArgs.ContainsKey("javascript-enable")) { - _log.Warn("Firefox version is not sufficient. Exiting"); - return true; + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; } - FirefoxOptions options = new FirefoxOptions(); - options.AddArguments("--disable-infobars"); - options.AddArguments("--disable-extensions"); - options.AddArguments("--disable-notifications"); - - options.BrowserExecutableLocation = path; - options.Profile = new FirefoxProfile(); - - if (handler.HandlerArgs != null) - { - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") - { - options.AddArguments("--headless"); - } - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") - { - options.AddArguments("--incognito"); - } - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") - { - options.Profile.SetPreference("permissions.default.stylesheet", 2); - } - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") - { - options.Profile.SetPreference("permissions.default.image", 2); - } - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") - { - options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); - } - if (handler.HandlerArgs.ContainsKey("blockscripts") && handler.HandlerArgs["blockscripts"] == "true") - { - options.Profile.SetPreference("permissions.default.script", 2); - } - } - - options.Profile.SetPreference("permissions.default.cookies", 2); - options.Profile.SetPreference("permissions.default.popups", 2); - options.Profile.SetPreference("permissions.default.geolocation", 2); - options.Profile.SetPreference("permissions.default.media_stream", 2); - - Driver = new FirefoxDriver(options); - base.Driver = Driver; - //hack: bad urls used in the past... if (handler.Initial.Equals("") || handler.Initial.Equals("about:internal", StringComparison.InvariantCultureIgnoreCase) || handler.Initial.Equals("about:external", StringComparison.InvariantCultureIgnoreCase)) { handler.Initial = "about:blank"; - } - + } + Driver.Navigate().GoToUrl(handler.Initial); if (handler.Loop) @@ -153,5 +112,64 @@ private bool FirefoxEx(TimelineHandler handler) return true; } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var path = GetInstallLocation(); + + if (!IsSufficientVersion(path)) + { + _log.Warn("Firefox version is not sufficient. Exiting"); + return null; + } + + FirefoxOptions options = new FirefoxOptions(); + options.AddArguments("--disable-infobars"); + options.AddArguments("--disable-extensions"); + options.AddArguments("--disable-notifications"); + + options.BrowserExecutableLocation = path; + options.Profile = new FirefoxProfile(); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("--headless"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.Profile.SetPreference("permissions.default.stylesheet", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.Profile.SetPreference("permissions.default.image", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.Profile.SetPreference("permissions.default.script", 2); + } + } + + options.Profile.SetPreference("permissions.default.cookies", 2); + options.Profile.SetPreference("permissions.default.popups", 2); + options.Profile.SetPreference("permissions.default.geolocation", 2); + options.Profile.SetPreference("permissions.default.media_stream", 2); + + options.Profile.SetPreference("geo.enabled", false); + options.Profile.SetPreference("geo.prompt.testing", false); + options.Profile.SetPreference("geo.prompt.testing.allow", false); + var driver = new FirefoxDriver(options); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + return driver; + } } } diff --git a/src/Ghosts.Client/Handlers/Excel.cs b/src/Ghosts.Client/Handlers/Excel.cs index 1e2bfd23..4d6b05e8 100755 --- a/src/Ghosts.Client/Handlers/Excel.cs +++ b/src/Ghosts.Client/Handlers/Excel.cs @@ -6,14 +6,13 @@ using Microsoft.Office.Interop.Excel; using NLog; using System; -using System.Drawing; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Ghosts.Domain.Code.Helpers; using NetOffice.ExcelApi.Tools; using Excel = NetOffice.ExcelApi; -using XlWindowState = NetOffice.ExcelApi.Enums.XlWindowState; namespace Ghosts.Client.Handlers { @@ -92,22 +91,8 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) var excelApplication = new Excel.Application { DisplayAlerts = false, - Visible = true }; - try - { - excelApplication.WindowState = XlWindowState.xlMinimized; - foreach (var item in excelApplication.Workbooks) - { - item.Windows[1].WindowState = XlWindowState.xlMinimized; - } - } - catch (Exception e) - { - _log.Trace($"Could not minimize: {e}"); - } - // create a utils instance, not need for but helpful to keep the lines of code low var utils = new CommonUtils(excelApplication); @@ -117,28 +102,50 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Excel adding worksheet"); var workSheet = (Excel.Worksheet) workBook.Worksheets[1]; - // draw back color and perform the BorderAround method - workSheet.Range("$B2:$B5").Interior.Color = utils.Color.ToDouble(Color.DarkGreen); - workSheet.Range("$B2:$B5").BorderAround(XlLineStyle.xlContinuous, XlBorderWeight.xlMedium, - XlColorIndex.xlColorIndexAutomatic); - - // draw back color and border the range explicitly - workSheet.Range("$D2:$D5").Interior.Color = utils.Color.ToDouble(Color.DarkGreen); - workSheet.Range("$D2:$D5") - .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .LineStyle = XlLineStyle.xlDouble; - workSheet.Range("$D2:$D5") - .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .Weight = 4; - workSheet.Range("$D2:$D5") - .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] - .Color = utils.Color.ToDouble(Color.Black); + + var list = RandomText.GetDictionary.GetDictionaryList(); + var rt = new RandomText(list.ToArray()); + rt.AddSentence(10); + + workSheet.Cells[1, 1].Value = rt.Content; + + var random = new Random(); + for (var i = 2; i < 100; i++) + { + for (var j = 1; j < 100; j++) + { + if (random.Next(0, 20) != 1) // 1 in 20 cells are blank + workSheet.Cells[i, j].Value = random.Next(0, 999999999); + } + } + + for (var i = 0; i < random.Next(1,30); i++) + { + var range = GetRandomRange(); + // draw back color and perform the BorderAround method + workSheet.Range(range).Interior.Color = + utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + workSheet.Range(range).BorderAround(XlLineStyle.xlContinuous, XlBorderWeight.xlMedium, + XlColorIndex.xlColorIndexAutomatic); + + range = GetRandomRange(); + // draw back color and border the range explicitly + workSheet.Range(range).Interior.Color = + utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + workSheet.Range(range) + .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] + .LineStyle = XlLineStyle.xlDouble; + workSheet.Range(range) + .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] + .Weight = 4; + workSheet.Range(range) + .Borders[(Excel.Enums.XlBordersIndex) XlBordersIndex.xlInsideHorizontal] + .Color = utils.Color.ToDouble(StylingExtensions.GetRandomColor()); + } var writeSleep = ProcessManager.Jitter(100); Thread.Sleep(writeSleep); - workSheet.Cells[1, 1].Value = "We have 2 simple shapes created."; - var rand = RandomFilename.Generate(); var dir = timelineEvent.CommandArgs[0].ToString(); @@ -177,9 +184,21 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace($"Excel saving to path - {path}"); workBook.SaveAs(path); + FileListing.Add(path); - Report(handler.HandlerType.ToString(), timelineEvent.Command, - timelineEvent.CommandArgs[0].ToString()); + Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); + + if (timelineEvent.CommandArgs.Contains("pdf")) + { + var pdfFileName = timelineEvent.CommandArgs.Contains("pdf-vary-filenames") ? $"{RandomFilename.Generate()}.pdf" : workBook.FullName.Replace(".xlsx", ".pdf"); + // Save document into PDF Format + workBook.ExportAsFixedFormat(NetOffice.ExcelApi.Enums.XlFixedFormatType.xlTypePDF, pdfFileName); + // end save as pdf + Report(handler.HandlerType.ToString(), timelineEvent.Command, "pdf"); + FileListing.Add(pdfFileName); + } + + workBook.Close(); if (timelineEvent.DelayAfter > 0) { @@ -193,20 +212,23 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) excelApplication.Dispose(); excelApplication = null; - workBook = null; - workSheet = null; - try { Marshal.ReleaseComObject(excelApplication); } - catch { } + catch + { + // ignore + } try { Marshal.FinalReleaseComObject(excelApplication); } - catch { } + catch + { + // ignore + } GC.Collect(); } @@ -230,5 +252,16 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Excel closing..."); } } + + private string GetRandomRange() + { + var r = new Random(); + var x = r.Next(1, 40); + var y = r.Next(x, 50); + var a1 = RandomText.GetRandomCapitalLetter(); + var a2 = RandomText.GetRandomCapitalLetter(a1); + + return $"${a1}{x}:${a2}{y}"; + } } } diff --git a/src/Ghosts.Client/Handlers/NpcSystem.cs b/src/Ghosts.Client/Handlers/NpcSystem.cs index ce0c7b2b..6f6e05b1 100755 --- a/src/Ghosts.Client/Handlers/NpcSystem.cs +++ b/src/Ghosts.Client/Handlers/NpcSystem.cs @@ -1,8 +1,10 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; using Ghosts.Client.Infrastructure; using Ghosts.Client.TimelineManager; using Ghosts.Domain; +using Ghosts.Domain.Code; using NLog; namespace Ghosts.Client.Handlers @@ -11,7 +13,7 @@ public class NpcSystem : BaseHandler { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public NpcSystem(TimelineHandler handler) + public NpcSystem(Timeline timeline, TimelineHandler handler) { _log.Trace($"Handling NpcSystem call: {handler}"); @@ -20,26 +22,32 @@ public NpcSystem(TimelineHandler handler) if (string.IsNullOrEmpty(timelineEvent.Command)) continue; - Timeline timeline; + Timeline t; switch (timelineEvent.Command.ToLower()) { case "start": - timeline = TimelineBuilder.GetLocalTimeline(); - timeline.Status = Timeline.TimelineStatus.Run; - TimelineBuilder.SetLocalTimeline(timeline); + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Run; + TimelineBuilder.SetLocalTimeline(t); break; case "stop": - timeline = TimelineBuilder.GetLocalTimeline(); - timeline.Status = Timeline.TimelineStatus.Stop; + if (timeline.Id != Guid.Empty) + { + var o = new Orchestrator(); + o.StopTimeline(timeline.Id); + } + else + { + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Stop; + StartupTasks.CleanupProcesses(); + TimelineBuilder.SetLocalTimeline(t); + } - StartupTasks.CleanupProcesses(); - - TimelineBuilder.SetLocalTimeline(timeline); break; } } } - } } \ No newline at end of file diff --git a/src/Ghosts.Client/Handlers/PowerPoint.cs b/src/Ghosts.Client/Handlers/PowerPoint.cs index 0510e6af..ed7d3533 100755 --- a/src/Ghosts.Client/Handlers/PowerPoint.cs +++ b/src/Ghosts.Client/Handlers/PowerPoint.cs @@ -148,9 +148,23 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) Thread.Sleep(5000); presentation.SaveAs(path); + FileListing.Add(path); - Report(handler.HandlerType.ToString(), timelineEvent.Command, - timelineEvent.CommandArgs[0].ToString()); + Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); + + if (timelineEvent.CommandArgs.Contains("pdf")) + { + // Save document into PDF Format + var outputFileName = timelineEvent.CommandArgs.Contains("pdf-vary-filenames") ? $"{RandomFilename.Generate()}.pdf" : presentation.FullName.Replace(".pptx", ".pdf"); + object fileFormat = PpSaveAsFileType.ppSaveAsPDF; + + presentation.SaveAs(outputFileName, fileFormat, MsoTriState.msoCTrue); + // end save as pdf + Report(handler.HandlerType.ToString(), timelineEvent.Command, "pdf"); + FileListing.Add(outputFileName); + } + + presentation.Close(); if (timelineEvent.DelayAfter > 0) { @@ -163,19 +177,24 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) powerApplication.Quit(); powerApplication.Dispose(); powerApplication = null; - presentation = null; - + try { Marshal.ReleaseComObject(powerApplication); } - catch { } + catch + { + // ignore + } try { Marshal.FinalReleaseComObject(powerApplication); } - catch { } + catch + { + // ignore + } GC.Collect(); } diff --git a/src/Ghosts.Client/Handlers/Word.cs b/src/Ghosts.Client/Handlers/Word.cs index a47d9f5e..22cc3db9 100755 --- a/src/Ghosts.Client/Handlers/Word.cs +++ b/src/Ghosts.Client/Handlers/Word.cs @@ -6,11 +6,14 @@ using NetOffice.WordApi.Enums; using NLog; using System; +using System.Drawing; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Ghosts.Domain.Code.Helpers; using Word = NetOffice.WordApi; +using VB = Microsoft.VisualBasic; namespace Ghosts.Client.Handlers { @@ -107,20 +110,20 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) { _log.Trace($"Could not minimize: {e}"); } - + // insert some text var list = RandomText.GetDictionary.GetDictionaryList(); var rt = new RandomText(list.ToArray()); - rt.AddContentParagraphs(1, 1, 1, 10, 50); + rt.AddContentParagraphs(1, 50); wordApplication.Selection.TypeText(rt.Content); var writeSleep = ProcessManager.Jitter(100); Thread.Sleep(writeSleep); wordApplication.Selection.HomeKey(WdUnits.wdLine, WdMovementType.wdExtend); - wordApplication.Selection.Font.Color = WdColor.wdColorSeaGreen; + wordApplication.Selection.Font.Color = GetWdColor(StylingExtensions.GetRandomColor()); wordApplication.Selection.Font.Bold = 1; - wordApplication.Selection.Font.Size = 18; + wordApplication.Selection.Font.Size = 12; var rand = RandomFilename.Generate(); @@ -160,10 +163,28 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) newDocument.Saved = true; newDocument.SaveAs(path); - Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); + Report(handler.HandlerType.ToString(), timelineEvent.Command, timelineEvent.CommandArgs[0].ToString()); FileListing.Add(path); + if (timelineEvent.CommandArgs.Contains("pdf")) + { + // Save document into PDF Format + object oMissing = System.Reflection.Missing.Value; + object outputFileName = timelineEvent.CommandArgs.Contains("pdf-vary-filenames") ? $"{RandomFilename.Generate()}.pdf" : newDocument.FullName.Replace(".docx", ".pdf"); + object fileFormat = WdSaveFormat.wdFormatPDF; + + newDocument.SaveAs(outputFileName, fileFormat, oMissing, oMissing, + oMissing, oMissing, oMissing, oMissing, + oMissing, oMissing, oMissing, oMissing, + oMissing, oMissing, oMissing, oMissing); + // end save as pdf + Report(handler.HandlerType.ToString(), timelineEvent.Command, "pdf"); + FileListing.Add(outputFileName.ToString()); + } + + newDocument.Close(); + if (timelineEvent.DelayAfter > 0) { //sleep and leave the app open @@ -179,13 +200,19 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) { Marshal.ReleaseComObject(wordApplication); } - catch { } + catch + { + // ignore + } try { Marshal.FinalReleaseComObject(wordApplication); } - catch { } + catch + { + // ignore + } GC.Collect(); } @@ -209,5 +236,13 @@ private void ExecuteEvents(Timeline timeline, TimelineHandler handler) _log.Trace("Word closing..."); } } + + + private WdColor GetWdColor(Color color) + { + var rgbColor = VB.Information.RGB(color.R, color.G, color.B); + var wdColor = (WdColor)rgbColor; + return wdColor; + } } } diff --git a/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs b/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs index af6d9b34..0b8279ac 100755 --- a/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs +++ b/src/Ghosts.Client/Infrastructure/Email/EmailConfiguration.cs @@ -4,7 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; // ReSharper disable InconsistentNaming diff --git a/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs b/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs index 971f4cf9..ef255e66 100755 --- a/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs +++ b/src/Ghosts.Client/Infrastructure/Email/EmailContent.cs @@ -7,6 +7,7 @@ using System.Text; using FileHelpers; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; namespace Ghosts.Client.Infrastructure.Email diff --git a/src/Ghosts.Client/Infrastructure/FileListing.cs b/src/Ghosts.Client/Infrastructure/FileListing.cs index 5fe07698..24129fd7 100755 --- a/src/Ghosts.Client/Infrastructure/FileListing.cs +++ b/src/Ghosts.Client/Infrastructure/FileListing.cs @@ -1,5 +1,6 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -87,10 +88,8 @@ public static void FlushList() continue; } - var creationTime = file.CreationTime.Hour; - _log.Trace($"Delete evaluation for {file.FullName} {file.CreationTime}"); - - if (!file.Exists || (creationTime <= Program.Configuration.OfficeDocsMaxAgeInHours)) + _log.Trace($"Delete evaluation for {file.FullName} {file.CreationTime} vs. {DateTime.Now.AddHours(-Program.Configuration.OfficeDocsMaxAgeInHours)}"); + if (!file.Exists || (file.CreationTime > (DateTime.Now.AddHours(-Program.Configuration.OfficeDocsMaxAgeInHours)))) continue; try diff --git a/src/Ghosts.Client/Infrastructure/RandomFilename.cs b/src/Ghosts.Client/Infrastructure/RandomFilename.cs index b3138396..c8b407ad 100755 --- a/src/Ghosts.Client/Infrastructure/RandomFilename.cs +++ b/src/Ghosts.Client/Infrastructure/RandomFilename.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; namespace Ghosts.Client.Infrastructure diff --git a/src/Ghosts.Client/Infrastructure/RandomText.cs b/src/Ghosts.Client/Infrastructure/RandomText.cs index 11be1b0e..bdfc7d3a 100755 --- a/src/Ghosts.Client/Infrastructure/RandomText.cs +++ b/src/Ghosts.Client/Infrastructure/RandomText.cs @@ -17,12 +17,35 @@ public class RandomText private readonly StringBuilder _builder; private readonly string[] _words; + public static char GetRandomCapitalLetter() + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + var rand = new Random(); + return chars[rand.Next(0, chars.Length)]; + } + + public static char GetRandomCapitalLetter(char after) + { + after = char.ToUpper(after); + var index = (int)after % 32; + + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + var rand = new Random(); + return chars[rand.Next(index, chars.Length)]; + } + public RandomText(string[] words) { _builder = new StringBuilder(); _words = words; } + public void AddContentParagraphs(int minParagraphs, int maxParagraphs) + { + var paragraphs = _random.Next(minParagraphs, maxParagraphs); + AddContentParagraphs(paragraphs, paragraphs, (paragraphs + 10), (paragraphs * 10), (paragraphs * 25)); + } + public void AddContentParagraphs(int numberParagraphs, int minSentences, int maxSentences, int minWords, int maxWords) { for (var i = 0; i < numberParagraphs; i++) diff --git a/src/Ghosts.Client/Program.cs b/src/Ghosts.Client/Program.cs index 97a72110..5e7cf472 100755 --- a/src/Ghosts.Client/Program.cs +++ b/src/Ghosts.Client/Program.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Net; @@ -11,6 +12,7 @@ using Ghosts.Client.Infrastructure; using Ghosts.Client.TimelineManager; using Ghosts.Domain.Code; +using Ghosts.Domain.Models; using NLog; namespace Ghosts.Client @@ -28,6 +30,7 @@ class Program private const int SwHide = 0; private const int SwShow = 5; + internal static List ThreadJobs { get; set; } internal static ClientConfiguration Configuration { get; set; } internal static Options OptionFlags; internal static bool IsDebug; @@ -78,7 +81,7 @@ static void Main(string[] args) { MinimizeFootprint(); minimizeMemory(); - + try { Run(args); @@ -98,6 +101,8 @@ static void Main(string[] args) private static void Run(string[] args) { + ThreadJobs = new List(); + // ignore all certs ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; diff --git a/src/Ghosts.Client/Properties/AssemblyInfo.cs b/src/Ghosts.Client/Properties/AssemblyInfo.cs index 340bd6c9..dd982e5e 100755 --- a/src/Ghosts.Client/Properties/AssemblyInfo.cs +++ b/src/Ghosts.Client/Properties/AssemblyInfo.cs @@ -9,7 +9,7 @@ [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("CMU > SEI > CERT > MSE > Realism Team")] [assembly: AssemblyProduct("GHOSTS NPC Framework Client - please email ddupdyke@sei.cmu.edu with bugs/requests/other")] -[assembly: AssemblyCopyright("Copyright © 2017 - 2020")] +[assembly: AssemblyCopyright("Copyright © 2017 - 2021")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] @@ -31,5 +31,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("4.0.0.0")] -[assembly: AssemblyFileVersion("4.0.0.0")] +[assembly: AssemblyVersion("6.0.1.0")] +[assembly: AssemblyFileVersion("6.0.1.0")] diff --git a/src/Ghosts.Client/Sample Timelines/Browser Crawl.json b/src/Ghosts.Client/Sample Timelines/Browser Crawl.json new file mode 100644 index 00000000..79fdf608 --- /dev/null +++ b/src/Ghosts.Client/Sample Timelines/Browser Crawl.json @@ -0,0 +1,39 @@ +{ + "Status": "Run", + "TimeLineHandlers": [ + { + "HandlerType": "BrowserFirefox", + "HandlerArgs": { + "isheadless": "false", + "blockimages": "false", + "blockstyles": "false", + "blockflash": "false", + "blockscripts": "false", + "stickiness": 100, + "stickiness-depth-min": 5, + "stickiness-depth-max": 10, + "incognito": "true", + "crawl-proxy-local-url": "http://localhost:8080/test-001/record/", + "crawl-tasks-maximum": 2, + "crawl-site-depth": 2 + }, + "Initial": "about:blank", + "UtcTimeOn": "00:00:00", + "UtcTimeOff": "24:00:00", + "Loop": "false", + "TimeLineEvents": [ + { + "Command": "crawl", + "CommandArgs": [ + "http://localhost:8080/test-001/record/https://www.usatoday.com", + "http://localhost:8080/test-001/record/https://www.cnn.com", + "http://localhost:8080/test-001/record/https://www.cmu.edu", + "http://localhost:8080/test-001/record/https://www.espn.com" + ], + "DelayAfter": 5000, + "DelayBefore": 0 + } + ] + } + ] +} diff --git a/src/Ghosts.Client/Survey/SurveyManager.cs b/src/Ghosts.Client/Survey/SurveyManager.cs index ee92ed8d..5baa2506 100644 --- a/src/Ghosts.Client/Survey/SurveyManager.cs +++ b/src/Ghosts.Client/Survey/SurveyManager.cs @@ -10,9 +10,9 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading; -using Ghosts.Client.Infrastructure; using Ghosts.Client.Comms; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using Newtonsoft.Json; using NLog; diff --git a/src/Ghosts.Client/TimelineManager/Listener.cs b/src/Ghosts.Client/TimelineManager/Listener.cs index 7c93ee54..26002d83 100755 --- a/src/Ghosts.Client/TimelineManager/Listener.cs +++ b/src/Ghosts.Client/TimelineManager/Listener.cs @@ -6,6 +6,7 @@ using SimpleTCP; using System; using System.IO; +using System.Linq; using System.Threading; using Ghosts.Domain.Code; @@ -87,7 +88,7 @@ public DirectoryListener() { Path = _in, NotifyFilter = NotifyFilters.LastWrite, - Filter = "*.json" + Filter = "*.*" }; watcher.Changed += new FileSystemEventHandler(OnChanged); watcher.EnableRaisingEvents = true; @@ -95,50 +96,81 @@ public DirectoryListener() private void OnChanged(object source, FileSystemEventArgs e) { - // filewatcher throws multiple events, we only need 1 - if (!string.IsNullOrEmpty(_currentlyProcessing) && _currentlyProcessing == e.FullPath) return; - if (!File.Exists(e.FullPath)) return; - - _currentlyProcessing = e.FullPath; - - _log.Trace("DirectoryListener found file: " + e.FullPath + " " + e.ChangeType); - - try - { - var raw = File.ReadAllText(e.FullPath); - - var timeline = JsonConvert.DeserializeObject(raw); - - foreach (var timelineHandler in timeline.TimeLineHandlers) - { - _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); - - foreach (var timelineEvent in timelineHandler.TimeLineEvents) - { - if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + // filewatcher throws multiple events, we only need 1 + if (!string.IsNullOrEmpty(_currentlyProcessing) && _currentlyProcessing == e.FullPath) return; + _currentlyProcessing = e.FullPath; + + _log.Trace("DirectoryListener found file: " + e.FullPath + " " + e.ChangeType); + + if (!File.Exists(e.FullPath)) + return; + + if (e.FullPath.EndsWith(".json")) + { + try + { + var raw = File.ReadAllText(e.FullPath); + + var timeline = JsonConvert.DeserializeObject(raw); + + foreach (var timelineHandler in timeline.TimeLineHandlers) + { + _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); + + foreach (var timelineEvent in timelineHandler.TimeLineEvents) + { + if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + { + timelineEvent.TrackableId = Guid.NewGuid().ToString(); + } + } + + var orchestrator = new Orchestrator(); + orchestrator.RunCommand(timeline, timelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); + } + } + else if (e.FullPath.EndsWith(".cs")) + { + try + { + var commands = File.ReadAllText(e.FullPath).Split(Convert.ToChar("\n")).ToList(); + if (commands.Count > 0) + { + var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); + var orchestrator = new Orchestrator(); + var t = new Timeline { - timelineEvent.TrackableId = Guid.NewGuid().ToString(); - } - } - - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); - } - - var outfile = e.FullPath.Replace(_in, _out); - outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); - - File.Move(e.FullPath, outfile); - } - catch (Exception exc) - { - _log.Debug(exc); - } - finally - { - Thread.Sleep(1000); - _currentlyProcessing = string.Empty; - } + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(constructedTimelineHandler); + orchestrator.RunCommand(t, constructedTimelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); + } + } + + try + { + var outfile = e.FullPath.Replace(_in, _out); + outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); + + File.Move(e.FullPath, outfile); + } + catch (Exception exception) + { + _log.Debug(exception); + } + + _currentlyProcessing = string.Empty; } } @@ -172,13 +204,13 @@ public PortListener() private string Handle(Message message) { - string tempMsg = + var tempMsg = $"PortListener received raw {message.TcpClient.Client.RemoteEndPoint}: {message.MessageString}"; Console.WriteLine(tempMsg); _log.Trace(tempMsg); - string command = message.MessageString; - int index = command.LastIndexOf("}", StringComparison.InvariantCultureIgnoreCase); + var command = message.MessageString; + var index = command.LastIndexOf("}", StringComparison.InvariantCultureIgnoreCase); if (index > 0) { command = command.Substring(0, index + 1); @@ -188,9 +220,9 @@ private string Handle(Message message) try { - TimelineHandler timelineHandler = JsonConvert.DeserializeObject(command); + var timelineHandler = JsonConvert.DeserializeObject(command); - foreach (TimelineEvent evs in timelineHandler.TimeLineEvents) + foreach (var evs in timelineHandler.TimeLineEvents) { if (string.IsNullOrEmpty(evs.TrackableId)) { @@ -200,10 +232,17 @@ private string Handle(Message message) _log.Trace($"PortListener command found: {timelineHandler.HandlerType}"); - Orchestrator o = new Orchestrator(); - o.RunCommand(timelineHandler); + var o = new Orchestrator(); + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(timelineHandler); + + o.RunCommand(t, timelineHandler); - string obj = JsonConvert.SerializeObject(timelineHandler); + var obj = JsonConvert.SerializeObject(timelineHandler); return obj; } diff --git a/src/Ghosts.Client/TimelineManager/Orchestrator.cs b/src/Ghosts.Client/TimelineManager/Orchestrator.cs index e0e0d95e..1daef559 100755 --- a/src/Ghosts.Client/TimelineManager/Orchestrator.cs +++ b/src/Ghosts.Client/TimelineManager/Orchestrator.cs @@ -6,12 +6,13 @@ using Microsoft.Win32; using NLog; using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Threading; using System.Security.Permissions; +using Ghosts.Domain.Code; +using Ghosts.Domain.Models; namespace Ghosts.Client.TimelineManager { @@ -22,13 +23,11 @@ public class Orchestrator { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); private static DateTime _lastRead = DateTime.MinValue; - private List _threads { get; set; } - private List _threadJobs { get; set; } private Thread MonitorThread { get; set; } - private Timeline _timeline; - private FileSystemWatcher timelineWatcher; - private bool _isSafetyNetRunning = false; - private bool _isTempCleanerRunning = false; + private static Timeline _defaultTimeline; + private FileSystemWatcher _timelineWatcher; + private bool _isSafetyNetRunning; + private bool _isTempCleanerRunning; private bool _isWordInstalled { get; set; } private bool _isExcelInstalled { get; set; } @@ -40,6 +39,8 @@ public void Run() { try { + _defaultTimeline = TimelineBuilder.GetLocalTimeline(); + if (_isSafetyNetRunning != true) //checking if safetynet has already been started { this.StartSafetyNet(); //watch instance numbers @@ -52,30 +53,27 @@ public void Run() _isTempCleanerRunning = true; } - this._timeline = TimelineBuilder.GetLocalTimeline(); - + var dirName = TimelineBuilder.TimelineFilePath().DirectoryName; // now watch that file for changes - if(timelineWatcher == null) //you can change this to a bool if you want but checks if the object has been created + if (_timelineWatcher == null && dirName != null) //you can change this to a bool if you want but checks if the object has been created { _log.Trace("Timeline watcher starting and is null..."); - timelineWatcher = new FileSystemWatcher(TimelineBuilder.TimelineFilePath().DirectoryName) + _timelineWatcher = new FileSystemWatcher(dirName) { Filter = Path.GetFileName(TimelineBuilder.TimelineFilePath().Name) }; _log.Trace($"watching {Path.GetFileName(TimelineBuilder.TimelineFilePath().Name)}"); - timelineWatcher.NotifyFilter = NotifyFilters.LastWrite; - timelineWatcher.EnableRaisingEvents = true; - timelineWatcher.Changed += OnChanged; + _timelineWatcher.NotifyFilter = NotifyFilters.LastWrite; + _timelineWatcher.EnableRaisingEvents = true; + _timelineWatcher.Changed += OnChanged; } - - _threadJobs = new List(); - + //load into an managing object //which passes the timeline commands to handlers //and creates a thread to execute instructions over that timeline - if (this._timeline.Status == Timeline.TimelineStatus.Run) + if (_defaultTimeline.Status == Timeline.TimelineStatus.Run) { - RunEx(this._timeline); + RunEx(_defaultTimeline); } else { @@ -92,34 +90,68 @@ public void Run() } } - public void Shutdown() + public void StopTimeline(Guid timelineId) { - try + foreach (var threadJob in Program.ThreadJobs.Where(x => x.TimelineId == timelineId)) { - foreach (var thread in _threads) + try + { + threadJob.Thread.Abort(null); + } + catch (Exception e) + { + _log.Debug(e); + } + + try + { + threadJob.Thread.Join(); + } + catch (Exception e) { - thread.Abort(null); + _log.Debug(e); } } - catch { } } - private void RunEx(Timeline timeline) + public void Stop() { - _threads = new List(); + foreach (var threadJob in Program.ThreadJobs) + { + try + { + threadJob.Thread.Abort(null); + } + catch (Exception e) + { + _log.Debug(e); + } + try + { + threadJob.Thread.Join(); + } + catch (Exception e) + { + _log.Debug(e); + } + } + } + + private void RunEx(Timeline timeline) + { WhatsInstalled(); - foreach (TimelineHandler handler in timeline.TimeLineHandlers) + foreach (var handler in timeline.TimeLineHandlers) { ThreadLaunch(timeline, handler); } } - public void RunCommand(TimelineHandler handler) + public void RunCommand(Timeline timeline, TimelineHandler handler) { WhatsInstalled(); - ThreadLaunch(null, handler); + ThreadLaunch(timeline, handler); } ///here lies technical debt @@ -133,7 +165,7 @@ private void StartSafetyNet() IsBackground = true, Name = "ghosts-safetynet" }; - t.Start(); + t.Start(_defaultTimeline); } catch (Exception e) { @@ -144,17 +176,16 @@ private void StartSafetyNet() ///here lies technical debt //TODO clean up // if supposed to be one excel running, and there is more than 2, then kill race condition - private static void SafetyNet() + private static void SafetyNet(object defaultTimeline) { + var timeline = (Timeline) defaultTimeline; while (true) { try { _log.Trace("SafetyNet loop beginning"); - FileListing.FlushList(); //Added 6/10 by AMV to clear clogged while loop. - - var timeline = TimelineBuilder.GetLocalTimeline(); + FileListing.FlushList(); var handlerCount = timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.Excel); var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.Excel).ToList(); @@ -194,7 +225,7 @@ private static void SafetyNet() private void WhatsInstalled() { - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("Outlook.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("Outlook.Application")) { if (regWord != null) { @@ -204,7 +235,7 @@ private void WhatsInstalled() _log.Trace($"Outlook is installed: {_isOutlookInstalled}"); } - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("Word.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("Word.Application")) { if (regWord != null) { @@ -214,7 +245,7 @@ private void WhatsInstalled() _log.Trace($"Word is installed: {_isWordInstalled}"); } - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("Excel.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("Excel.Application")) { if (regWord != null) { @@ -224,7 +255,7 @@ private void WhatsInstalled() _log.Trace($"Excel is installed: {_isExcelInstalled}"); } - using (RegistryKey regWord = Registry.ClassesRoot.OpenSubKey("PowerPoint.Application")) + using (var regWord = Registry.ClassesRoot.OpenSubKey("PowerPoint.Application")) { if (regWord != null) { @@ -237,57 +268,35 @@ private void WhatsInstalled() private void ThreadLaunch(Timeline timeline, TimelineHandler handler) { - try { _log.Trace($"Attempting new thread for: {handler.HandlerType}"); Thread t = null; - ThreadJob threadJob = new ThreadJob - { - Id = Guid.NewGuid().ToString(), - Handler = handler - }; - + object o; switch (handler.HandlerType) { case HandlerType.NpcSystem: - NpcSystem npc = new NpcSystem(handler); + var npc = new NpcSystem(timeline, handler); break; case HandlerType.Command: - t = new Thread(() => + t = new Thread(start: () => { - Cmd o = new Cmd(handler); - - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Command; - + o = new Cmd(handler); + }); break; case HandlerType.Word: _log.Trace("Launching thread for word"); if (_isWordInstalled) { var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.Word).ToList(); - if (pids.Count > timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.Word)) + if (pids.Count > timeline.TimeLineHandlers.Count(x => x.HandlerType == HandlerType.Word)) return; t = new Thread(() => { - WordHandler o = new WordHandler(timeline, handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Word; + o = new WordHandler(timeline, handler); + }); } break; case HandlerType.Excel: @@ -295,176 +304,100 @@ private void ThreadLaunch(Timeline timeline, TimelineHandler handler) if (_isExcelInstalled) { var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.Excel).ToList(); - if (pids.Count > timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.Excel)) + if (pids.Count > timeline.TimeLineHandlers.Count(x => x.HandlerType == HandlerType.Excel)) return; t = new Thread(() => { - ExcelHandler o = new ExcelHandler(timeline, handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Excel; + o = new ExcelHandler(timeline, handler); + }); } break; case HandlerType.Clicks: _log.Trace("Launching thread to handle clicks"); t = new Thread(() => { - Clicks o = new Clicks(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + o = new Clicks(handler); + }); break; case HandlerType.Reboot: _log.Trace("Launching thread to handle reboot"); t = new Thread(() => { - Reboot o = new Reboot(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); + o = new Reboot(handler); + }); break; case HandlerType.PowerPoint: _log.Trace("Launching thread for powerpoint"); if (_isPowerPointInstalled) { var pids = ProcessManager.GetPids(ProcessManager.ProcessNames.PowerPoint).ToList(); - if (pids.Count > timeline.TimeLineHandlers.Count(o => o.HandlerType == HandlerType.PowerPoint)) + if (pids.Count > timeline.TimeLineHandlers.Count(x => x.HandlerType == HandlerType.PowerPoint)) return; t = new Thread(() => { - PowerPointHandler o = new PowerPointHandler(timeline, handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.PowerPoint; + o = new PowerPointHandler(timeline, handler); + }); } break; case HandlerType.Outlook: _log.Trace("Launching thread for outlook - note we're not checking if outlook installed, just going for it"); - //if (this.IsOutlookInstalled) - //{ t = new Thread(() => { - Outlook o = new Outlook(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Outlook; - //} - + o = new Outlook(handler); + }); break; case HandlerType.BrowserIE: //IE demands COM apartmentstate be STA so diff thread creation required t = new Thread(() => { - BrowserIE o = new BrowserIE(handler); + o = new BrowserIE(handler); }); t.SetApartmentState(ApartmentState.STA); - t.IsBackground = true; - t.Name = threadJob.Id; - t.Start(); - break; case HandlerType.Notepad: //TODO t = new Thread(() => { - Notepad o = new Notepad(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - + o = new Notepad(handler); + }); break; - case HandlerType.BrowserChrome: t = new Thread(() => { - BrowserChrome o = new BrowserChrome(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Chrome; - + o = new BrowserChrome(handler); + }); break; case HandlerType.BrowserFirefox: t = new Thread(() => { - BrowserFirefox o = new BrowserFirefox(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - threadJob.ProcessName = ProcessManager.ProcessNames.Firefox; - + o = new BrowserFirefox(handler); + }); break; case HandlerType.Watcher: t = new Thread(() => { - Watcher o = new Watcher(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - - //threadJob.ProcessName = ProcessManager.ProcessNames.Watcher; - + o = new Watcher(handler); + }); break; case HandlerType.Print: t = new Thread(() => { - var p = new Print(handler); - }) - { - IsBackground = true, - Name = threadJob.Id - }; - t.Start(); - + o = new Print(handler); + }); break; } - if (threadJob.ProcessName != null) - { - _threadJobs.Add(threadJob); - } + if (t == null) return; - if (t != null) + t.IsBackground = true; + t.Start(); + Program.ThreadJobs.Add(new ThreadJob { - _threads.Add(t); - } + TimelineId = timeline.Id, + Thread = t + }); } catch (Exception e) { @@ -479,57 +412,48 @@ private void OnChanged(object source, FileSystemEventArgs e) _log.Trace($"FileWatcher event raised: {e.FullPath} {e.Name} {e.ChangeType}"); // filewatcher throws two events, we only need 1 - DateTime lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime != _lastRead) - { - _lastRead = lastWriteTime; - _log.Trace("FileWatcher Processing: " + e.FullPath + " " + e.ChangeType); - _log.Trace($"Reloading {MethodBase.GetCurrentMethod().DeclaringType}"); + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime == _lastRead) return; + + _lastRead = lastWriteTime; + _log.Trace("FileWatcher Processing: " + e.FullPath + " " + e.ChangeType); + _log.Trace($"Reloading {MethodBase.GetCurrentMethod().DeclaringType}"); - _log.Trace("terminate existing tasks and rerun orchestrator"); + _log.Trace("terminate existing tasks and rerun orchestrator"); - try - { - Shutdown(); - } - catch (Exception exception) - { - _log.Info(exception); - } + try + { + Stop(); + } + catch (Exception exception) + { + _log.Info(exception); + } - try - { - StartupTasks.CleanupProcesses(); - } - catch (Exception exception) - { - _log.Info(exception); - } + try + { + StartupTasks.CleanupProcesses(); + } + catch (Exception exception) + { + _log.Info(exception); + } - Thread.Sleep(7500); + Thread.Sleep(7500); - try - { - Run(); - } - catch (Exception exception) - { - _log.Info(exception); - } + try + { + Run(); + } + catch (Exception exception) + { + _log.Info(exception); } } catch (Exception exc) { _log.Info(exc); } - } } - - public class ThreadJob - { - public string Id { get; set; } - public TimelineHandler Handler { get; set; } - public string ProcessName { get; set; } - } } diff --git a/src/Ghosts.Client/TimelineManager/TimelineBuilder.cs b/src/Ghosts.Client/TimelineManager/TimelineBuilder.cs deleted file mode 100755 index bb8c8251..00000000 --- a/src/Ghosts.Client/TimelineManager/TimelineBuilder.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - -using System; -using System.IO; -using Ghosts.Domain; -using Ghosts.Domain.Code; -using Newtonsoft.Json; -using NLog; - -namespace Ghosts.Client.TimelineManager -{ - /// - /// Helper class that loads timeline and watches it for future changes - /// - public class TimelineBuilder - { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - - public static string TimelineFile = ApplicationDetails.ConfigurationFiles.Timeline; - - public static FileInfo TimelineFilePath() - { - return new FileInfo(TimelineFile); - } - - /// - /// Get from local disk - /// - /// The local timeline to be executed - public static Timeline GetLocalTimeline() - { - Timeline timeline; - _log.Trace($"Loading timeline config {TimelineFile }"); - - try - { - var raw = File.ReadAllText(TimelineFile); - timeline = JsonConvert.DeserializeObject(raw); - } - catch (Exception e) - { - var err = $"ERROR: Could not deserialize timeline json file! {e.Message} {e.StackTrace}"; - Console.WriteLine(err); - _log.Error(err); - throw; - } - - _log.Trace("Timeline config loaded successfully"); - - return timeline; - } - - /// - /// Save to local disk - /// - /// Raw timeline string (to be converted to `Timeline` type) - public static void SetLocalTimeline(string timelineString) - { - var timelineObject = JsonConvert.DeserializeObject(timelineString); - SetLocalTimeline(timelineObject); - } - - /// - /// Save to local disk - /// - /// `Timeline` type - public static void SetLocalTimeline(Timeline timeline) - { - using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Timeline)) - { - var serializer = new JsonSerializer(); - serializer.Formatting = Formatting.Indented; - serializer.Serialize(file, timeline); - } - } - } -} diff --git a/src/Ghosts.Client/config/application.json b/src/Ghosts.Client/config/application.json index c8e107ac..a4c7ac4f 100755 --- a/src/Ghosts.Client/config/application.json +++ b/src/Ghosts.Client/config/application.json @@ -1,95 +1,96 @@ { - "IdEnabled": true, - "IdUrl": "http://ghosts-c2:52388/api/clientid", - "IdFormat": "guestlocal", - "IdFormatKey": "guestinfo.id", - "IdFormatValue": "$formatkeyvalue$-$machinename$", - "VMWareToolsLocation": "C:\\progra~1\\VMware\\VMware Tools\\vmtoolsd.exe", - "EncodeHeaders": true, - "ClientResults": { - "IsEnabled": true, - "IsSecure": false, - "PostUrl": "http://ghosts-c2:52388/api/clientresults", - "CycleSleep": 9000 - }, - "ClientUpdates": { - "IsEnabled": true, - "PostUrl": "http://ghosts-c2:52388/api/clientupdates", - "CycleSleep": 9000 - }, - "Survey": { - "IsEnabled": true, - "IsSecure": false, - "Frequency": "once", - "CycleSleepMinutes": 5, - "OutputFormat": "indent", - "PostUrl": "http://ghosts-c2:52388/api/clientsurvey" - }, - "Content": { - "EmailContent": "", - "EmailReply": "", - "EmailDomain": "", - "EmailOutside": "", - "FileNames": "", - "Dictionary": "" - }, - "HealthIsEnabled": true, - "HandlersIsEnabled": true, - "ChromeExtensions": "", - "FirefoxInstallLocation": "", - "FirefoxMajorVersionMinimum": 48, - "OfficeDocsMaxAgeInHours": 6, - "Email": { - "RecipientsToMin": 1, - "RecipientsToMax": 3, - "RecipientsCcMin": 0, - "RecipientsCcMax": 2, - "RecipientsBccMin": 2, - "RecipientsBccMax": 2, - "RecipientsOutsideMin": 0, - "RecipientsOutsideMax": 1, - "SetAccountFromConfig": false, - "SetAccountFromLocal": false, - "SetForcedSendReceive": true, - "SaveToOutbox": false, - "EmailDomainSearchString": "Get-ADUser -filter * -searchbase \"CN=USERS,DC=JRSS,DC=GOV\" -properties UserPrincipalName | select -expand UserPrincipalName" - }, - "Listener": { - "Port": -1 - }, - "EmailContent": { - "conflict_1_capital": "", - "conflict_1_name": "", - "conflict_1_peoples": "", - "conflict_1_president": "", - "localized_flashpoint_locale": "", - "friendly_nation_leader_lastname": "", - "friendly_nation_leader_name": "", - "friendly_nation_name": "", - "friendly_nation_peoples": "", - "commander_title": "", - "commander_name": "", - "commander_initials": "", - "commander_lastname": "", - "commander_email": "", - "commander_sub1": "", - "commander_sub2": "", - "us_president": "", - "iraq": "", - "iraqi": "", - "iran": "", - "iranian": "", - "china": "", - "chinese": "", - "russia": "", - "azerbaijan": "", - "turkish": "", - "turkey": "", - "pakistani": "", - "pakistan": "", - "palestinian": "", - "palestine": "", - "gaza": "", - "korea": "" - } -} + "IdEnabled": true, + "IdUrl": "http://ghosts-api:52388/api/clientid", + "IdFormat": "guestlocal", + "IdFormatKey": "guestinfo.id", + "IdFormatValue": "$formatkeyvalue$-$machinename$", + "VMWareToolsLocation": "C:\\progra~1\\VMware\\VMware Tools\\vmtoolsd.exe", + "EncodeHeaders": true, + "ClientResults": { + "IsEnabled": true, + "IsSecure": false, + "PostUrl": "http://ghosts-api:52388/api/clientresults", + "CycleSleep": 9000 + }, + "ClientUpdates": { + "IsEnabled": true, + "PostUrl": "http://ghosts-api:52388/api/clientupdates", + "CycleSleep": 9000 + }, + "Survey": { + "IsEnabled": true, + "IsSecure": false, + "Frequency": "once", + "CycleSleepMinutes": 5, + "OutputFormat": "indent", + "PostUrl": "http://ghosts-api:52388/api/clientsurvey" + }, + "Content": { + "EmailContent": "", + "EmailReply": "", + "EmailDomain": "", + "EmailOutside": "", + "FileNames": "", + "Dictionary": "" + }, + "HealthIsEnabled": true, + "HandlersIsEnabled": true, + "ChromeExtensions": "", + "FirefoxInstallLocation": "", + "FirefoxMajorVersionMinimum": 48, + "OfficeDocsMaxAgeInHours": 72, + "Email": { + "RecipientsToMin": 1, + "RecipientsToMax": 3, + "RecipientsCcMin": 0, + "RecipientsCcMax": 2, + "RecipientsBccMin": 2, + "RecipientsBccMax": 2, + "RecipientsOutsideMin": 0, + "RecipientsOutsideMax": 1, + "SetAccountFromConfig": false, + "SetAccountFromLocal": false, + "SetForcedSendReceive": true, + "SaveToOutbox": false, + "EmailDomainSearchString": + "Get-ADUser -filter * -searchbase \"CN=USERS,DC=JRSS,DC=GOV\" -properties UserPrincipalName | select -expand UserPrincipalName" + }, + "Listener": { + "Port": -1 + }, + "EmailContent": { + "conflict_1_capital": "", + "conflict_1_name": "", + "conflict_1_peoples": "", + "conflict_1_president": "", + "localized_flashpoint_locale": "", + "friendly_nation_leader_lastname": "", + "friendly_nation_leader_name": "", + "friendly_nation_name": "", + "friendly_nation_peoples": "", + "commander_title": "", + "commander_name": "", + "commander_initials": "", + "commander_lastname": "", + "commander_email": "", + "commander_sub1": "", + "commander_sub2": "", + "us_president": "", + "iraq": "", + "iraqi": "", + "iran": "", + "iranian": "", + "china": "", + "chinese": "", + "russia": "", + "azerbaijan": "", + "turkish": "", + "turkey": "", + "pakistani": "", + "pakistan": "", + "palestinian": "", + "palestine": "", + "gaza": "", + "korea": "" + } +} \ No newline at end of file diff --git a/src/Ghosts.Client/config/timeline.json b/src/Ghosts.Client/config/timeline.json index 70c744b8..dda4103e 100755 --- a/src/Ghosts.Client/config/timeline.json +++ b/src/Ghosts.Client/config/timeline.json @@ -96,7 +96,7 @@ "TimeLineEvents": [ { "Command": "create", - "CommandArgs": [ "%homedrive%%homepath%\\Documents" ], + "CommandArgs": [ "%homedrive%%homepath%\\Documents", "pdf", "pdf-vary-filenames" ], "DelayAfter": 900000, "DelayBefore": 0 } @@ -111,7 +111,7 @@ "TimeLineEvents": [ { "Command": "create", - "CommandArgs": [ "%homedrive%%homepath%\\Documents" ], + "CommandArgs": [ "%homedrive%%homepath%\\Documents", "pdf", "pdf-vary-filenames" ], "DelayAfter": 900000, "DelayBefore": 0 } @@ -126,7 +126,7 @@ "TimeLineEvents": [ { "Command": "create", - "CommandArgs": [ "%homedrive%%homepath%\\Documents" ], + "CommandArgs": [ "%homedrive%%homepath%\\Documents", "pdf", "pdf-vary-filenames" ], "DelayAfter": 900000, "DelayBefore": 0 } @@ -143,7 +143,8 @@ "stickiness": 75, "stickiness-depth-min": 5, "stickiness-depth-max": 10000, - "incognito": "true" + "incognito": "true", + "javascript-enable": "true" }, "Initial": "about:blank", "UtcTimeOn": "00:00:00", diff --git a/src/Ghosts.Domain/Code/ApplicationDetails.cs b/src/Ghosts.Domain/Code/ApplicationDetails.cs index df117e06..b4b18757 100755 --- a/src/Ghosts.Domain/Code/ApplicationDetails.cs +++ b/src/Ghosts.Domain/Code/ApplicationDetails.cs @@ -58,6 +58,7 @@ public static bool IsLinux() return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); } + // ReSharper disable once InconsistentNaming public static bool IsOSX() { return RuntimeInformation.IsOSPlatform(OSPlatform.OSX); diff --git a/src/Ghosts.Domain/Code/ClientConfiguration.cs b/src/Ghosts.Domain/Code/ClientConfiguration.cs index e6e1c481..16a54ad3 100755 --- a/src/Ghosts.Domain/Code/ClientConfiguration.cs +++ b/src/Ghosts.Domain/Code/ClientConfiguration.cs @@ -11,60 +11,61 @@ namespace Ghosts.Domain.Code public class ClientConfiguration { /// - /// Should each instance generate and use its own ID to identify with server? + /// Should each instance generate and use its own ID to identify with server? /// public bool IdEnabled { get; set; } /// - /// API URL for client to get its instance ID + /// API URL for client to get its instance ID /// public string IdUrl { get; set; } /// - /// guest|guestinfo + /// guest|guestinfo /// public string IdFormat { get; set; } /// - /// if using guestinfo, the key to query for the value to use as hostname + /// if using guestinfo, the key to query for the value to use as hostname /// public string IdFormatKey { get; set; } public string IdFormatValue { get; set; } /// - /// Where is vmtools? + /// Where is vmtools? /// + // ReSharper disable once InconsistentNaming public string VMWareToolsLocation { get; set; } /// - /// Are client health checks enabled? + /// Are client health checks enabled? /// public bool HealthIsEnabled { get; set; } /// - /// Is client executing a timeline of user activities? + /// Is client executing a timeline of user activities? /// public bool HandlersIsEnabled { get; set; } /// - /// Comma sep list of extensions for chrome (aka c:\path\to\extension) + /// Comma sep list of extensions for chrome (aka c:\path\to\extension) /// public string ChromeExtensions { get; set; } /// - /// The amount of hours that office docs will live before being cleaned (based on FileListing class reaper) - /// Set to -1 to disable + /// The amount of hours that office docs will live before being cleaned (based on FileListing class reaper) + /// Set to -1 to disable /// public int OfficeDocsMaxAgeInHours { get; set; } /// - /// Tokens to be replaced within email-content.csv for a more customized storyline + /// Tokens to be replaced within email-content.csv for a more customized storyline /// public Dictionary EmailContent { get; set; } /// - /// Client survey values + /// Client survey values /// public SurveySettings Survey { get; set; } @@ -74,12 +75,12 @@ public class ClientConfiguration public ContentSettings Content { get; set; } /// - /// Settable install location for non-standard "c:\program files\" or "c:\program files (x86)\" installs + /// Settable install location for non-standard "c:\program files\" or "c:\program files (x86)\" installs /// public string FirefoxInstallLocation { get; set; } /// - /// Geckodriver depends on at least this version of FF to be used + /// Geckodriver depends on at least this version of FF to be used /// public int FirefoxMajorVersionMinimum { get; set; } public bool EncodeHeaders { get; set; } @@ -98,19 +99,19 @@ public class ContentSettings public class ClientResultSettings { /// - /// Is client posting its results to server? + /// Is client posting its results to server? /// public bool IsEnabled { get; set; } public bool IsSecure { get; set; } /// - /// API URL for client to post activity results, like timeline, health, etc. + /// API URL for client to post activity results, like timeline, health, etc. /// public string PostUrl { get; set; } /// - /// How often should client post results? (this number is the ms sleep between cycles) + /// How often should client post results? (this number is the ms sleep between cycles) /// public int CycleSleep { get; set; } } @@ -118,17 +119,17 @@ public class ClientResultSettings public class ClientUpdateSettings { /// - /// Is client attempting to pull down updates from server? + /// Is client attempting to pull down updates from server? /// public bool IsEnabled { get; set; } /// - /// API URL for client to get updates like timeline, health, etc. + /// API URL for client to get updates like timeline, health, etc. /// public string PostUrl { get; set; } /// - /// How often should client poll for updates? (this number is the ms sleep between cycles) + /// How often should client poll for updates? (this number is the ms sleep between cycles) /// public int CycleSleep { get; set; } } @@ -166,7 +167,7 @@ public class EmailSettings public class ListenerSettings { /// - /// Set to -1 to disable + /// Set to -1 to disable /// public int Port { get; set; } } diff --git a/src/Ghosts.Domain/Code/Helpers.cs b/src/Ghosts.Domain/Code/Helpers.cs deleted file mode 100755 index a74931b5..00000000 --- a/src/Ghosts.Domain/Code/Helpers.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Ghosts.Domain.Code -{ - public static class Helpers - { - /// - /// Get name value of an Enum - /// - /// - /// - /// - public static T ParseEnum(string value) - { - return (T) Enum.Parse(typeof(T), value, true); - } - - public static bool IsOlderThanHours(string filename, int hours) - { - var threshold = DateTime.Now.AddHours(-hours); - return File.GetCreationTime(filename) <= threshold; - } - } - - public static class StringExtensions - { - public static string ReplaceCaseInsensitive(this string input, string search, string replacement) - { - var result = Regex.Replace(input, Regex.Escape(search), replacement.Replace("$", "$$"), RegexOptions.IgnoreCase); - return result; - } - - public static string RemoveFirstLines(this string text, int linesCount) - { - var lines = Regex.Split(text, "\r\n|\r|\n").Skip(linesCount); - return string.Join(Environment.NewLine, lines.ToArray()); - } - - public static string RemoveWhitespace(this string input) - { - return new string(input.Where(c => !char.IsWhiteSpace(c)).ToArray()); - } - - public static string RemoveDuplicateSpaces(this string input) - { - var regex = new Regex("[ ]{2,}", RegexOptions.None); - return regex.Replace(input, " "); - } - - public static string ToFormValueString(this IDictionary dictionary) - { - return string.Join("&", dictionary.Select(x => x.Key + "=" + x.Value).ToArray()); - } - } - - public static class EnumerableExtensions - { - /// - /// Picks 1 random object from a list T - /// - public static T PickRandom(this IEnumerable source) - { - return source.PickRandom(1).Single(); - } - - /// - /// Picks n random objects from a list T - /// - public static IEnumerable PickRandom(this IEnumerable source, int count) - { - return source.Shuffle().Take(count); - } - - /// - /// Randomize a list of T objects in list - /// - public static IEnumerable Shuffle(this IEnumerable source) - { - return source.OrderBy(x => Guid.NewGuid()); - } - } -} \ No newline at end of file diff --git a/src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs b/src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs new file mode 100644 index 00000000..dd0a7c14 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/DateTimeExtensions.cs @@ -0,0 +1,16 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.IO; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class DateTimeExtensions + { + public static bool IsOlderThanHours(string filename, int hours) + { + var threshold = DateTime.Now.AddHours(-hours); + return File.GetCreationTime(filename) <= threshold; + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs b/src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs new file mode 100644 index 00000000..03ffa0f4 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/DictionaryExtensions.cs @@ -0,0 +1,14 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System.Collections.Generic; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class DictionaryExtensions + { + public static bool ContainsKeyWithOption(this Dictionary options, string key, string value) + { + return options.ContainsKey(key) && options[key] == value; + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs b/src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs new file mode 100644 index 00000000..f66677ee --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/EnumerableExtensions.cs @@ -0,0 +1,46 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class EnumerableExtensions + { + /// + /// Get name value of an Enum + /// + /// + /// + /// + public static T ParseEnum(string value) + { + return (T)Enum.Parse(typeof(T), value, true); + } + + /// + /// Picks 1 random object from a list T + /// + public static T PickRandom(this IEnumerable source) + { + return source.PickRandom(1).Single(); + } + + /// + /// Picks n random objects from a list T + /// + public static IEnumerable PickRandom(this IEnumerable source, int count) + { + return source.Shuffle().Take(count); + } + + /// + /// Randomize a list of T objects in list + /// + public static IEnumerable Shuffle(this IEnumerable source) + { + return source.OrderBy(x => Guid.NewGuid()); + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/StringExtensions.cs b/src/Ghosts.Domain/Code/Helpers/StringExtensions.cs new file mode 100644 index 00000000..2939a959 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/StringExtensions.cs @@ -0,0 +1,53 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class StringExtensions + { + public static IEnumerable Split(this string o, string splitString) + { + return o.Split(new[] { splitString }, StringSplitOptions.None); + } + + public static string GetTextBetweenQuotes(this string o) + { + var result = Regex.Match(o, "\"([^\"]*)\"").ToString(); + if (!string.IsNullOrEmpty(result)) + result = result.TrimStart('"').TrimEnd('"'); + return result; + } + + public static string ReplaceCaseInsensitive(this string input, string search, string replacement) + { + var result = Regex.Replace(input, Regex.Escape(search), replacement.Replace("$", "$$"), RegexOptions.IgnoreCase); + return result; + } + + public static string RemoveFirstLines(this string text, int linesCount) + { + var lines = Regex.Split(text, "\r\n|\r|\n").Skip(linesCount); + return string.Join(Environment.NewLine, lines.ToArray()); + } + + public static string RemoveWhitespace(this string input) + { + return new string(input.Where(c => !char.IsWhiteSpace(c)).ToArray()); + } + + public static string RemoveDuplicateSpaces(this string input) + { + var regex = new Regex("[ ]{2,}", RegexOptions.None); + return regex.Replace(input, " "); + } + + public static string ToFormValueString(this IDictionary dictionary) + { + return string.Join("&", dictionary.Select(x => x.Key + "=" + x.Value).ToArray()); + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs b/src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs new file mode 100644 index 00000000..2d3d6b52 --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/StylingExtensions.cs @@ -0,0 +1,16 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Drawing; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class StylingExtensions + { + public static Color GetRandomColor() + { + var random = new Random(); + return Color.FromArgb((byte)random.Next(0, 255), (byte)random.Next(0, 255), (byte)random.Next(0, 255)); + } + } +} diff --git a/src/Ghosts.Domain/Code/Helpers/UriExtensions.cs b/src/Ghosts.Domain/Code/Helpers/UriExtensions.cs new file mode 100644 index 00000000..da5c925d --- /dev/null +++ b/src/Ghosts.Domain/Code/Helpers/UriExtensions.cs @@ -0,0 +1,13 @@ +using System; + +namespace Ghosts.Domain.Code.Helpers +{ + public static class UriExtensions + { + public static string GetDomain(this Uri uri) + { + var a = uri.Host.Split('.'); + return a.GetUpperBound(0) < 2 ? uri.Host : $"{a[a.GetUpperBound(0) - 1]}.{a[a.GetUpperBound(0)]}"; + } + } +} diff --git a/src/Ghosts.Domain/Code/Jitter.cs b/src/Ghosts.Domain/Code/Jitter.cs index bb38e4d1..0a2dc035 100755 --- a/src/Ghosts.Domain/Code/Jitter.cs +++ b/src/Ghosts.Domain/Code/Jitter.cs @@ -41,5 +41,16 @@ public static int Randomize(int baseSleepValue, int lowJitter, int highJitter) return newSleepValue; } + + public static int Basic(int baseSleep) + { + //sleep with jitter + var sleep = baseSleep; + var r = new Random().Next(-999, 1999); + sleep += r; + if (sleep < 0) + sleep = 1; + return sleep; + } } } \ No newline at end of file diff --git a/src/Ghosts.Domain/Code/LinkManager.cs b/src/Ghosts.Domain/Code/LinkManager.cs index ca52d8f2..2c0e0ed3 100644 --- a/src/Ghosts.Domain/Code/LinkManager.cs +++ b/src/Ghosts.Domain/Code/LinkManager.cs @@ -1,8 +1,10 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. +using Ghosts.Domain.Code.Helpers; using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; namespace Ghosts.Domain.Code { @@ -14,69 +16,126 @@ public Link() } public Uri Url { get; set; } + /// + /// Higher priority is more important + /// public int Priority { get; set; } + + public bool WasBrowsed { get; set; } } public class LinkManager { - private readonly string _baseUrl; + public List Links { private set; get; } + + private readonly Uri _baseUri; private readonly Random _random = new Random(); - public LinkManager(string baseUrl) + public LinkManager(Uri baseUri) { Links = new List(); - _baseUrl = baseUrl; + _baseUri = baseUri; } - public List Links { private set; get; } + public void AddLink(string url, int priority) + { + if (!Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out var uri)) + { + return; + } + this.AddLink(uri, priority); + } - /// - /// Adds proper links — invalid links get quickly discarded - /// - /// http|s://some.link/path/etc - public void AddLink(string url) + public void AddLink(Uri uri, int priority) { + string[] validSchemes = {"http", "https"}; + if (!validSchemes.Contains(uri.Scheme)) + { + return; + } + + foreach (var link in Links) + { + if (Uri.Compare(uri, link.Url, UriComponents.Host | UriComponents.PathAndQuery, UriFormat.SafeUnescaped, StringComparison.OrdinalIgnoreCase) == 0) + { + return; + } + } + + //truly a new link, add it try { - Links.Add(new Link {Url = new Uri(url)}); + Links.Add(new Link { Url = uri, Priority = priority }); } - catch + catch (Exception e) { + Console.WriteLine($"{uri} {e}"); } } public Link Choose() { - var baseUri = new Uri(_baseUrl); + var pickList = new List(); + foreach (var link in Links) + { try { - if (!link.Url.Host.Replace("www.", "").Contains(baseUri.Host.Replace("www.", ""))) - link.Priority += 10; + // give relative links priority + if ((link.Url.Scheme + link.Url.Host).Replace("www.", "").Equals((_baseUri.Scheme + _baseUri.Host).Replace("www.", ""), StringComparison.InvariantCultureIgnoreCase)) + { + link.Priority += 1; + } + else if (link.Url.Scheme.Equals("file", StringComparison.InvariantCultureIgnoreCase)) + { + link.Priority += 1; + } + + pickList.Add(link); } catch (Exception e) { Console.WriteLine($"{link.Url} : {e}"); } + } - Links = Links.OrderByDescending(o => o.Priority).ToList(); + Links = pickList.OrderByDescending(o => o.Priority).ToList(); if (Links.Count < 1) + { return null; + } - var totalWeight = Convert.ToInt32(Links.Sum(o => o.Priority)); - - // totalWeight is the sum of all weights - var r = _random.Next(0, totalWeight); + var priority = Links.First().Priority; + var chosen = Links.Where(x => x.Priority == priority).PickRandom(); - foreach (var link in Links) + if (chosen.Url.Scheme.ToLower().StartsWith("file")) { - if (r < link.Priority) return link; + try + { + var bUrl = _baseUri.ToString(); + if (bUrl.EndsWith("/")) + { + bUrl = bUrl.Substring(0, bUrl.Length - 1); + } + + var thisUrl = chosen.Url.ToString().Replace("file://", ""); + + thisUrl = Regex.Replace(thisUrl, "////", "//"); + if (thisUrl.StartsWith("/")) + { + thisUrl = thisUrl.Substring(1, thisUrl.Length - 1); + } - r -= link.Priority; + chosen.Url = new Uri($"{bUrl}/{thisUrl}"); + } + catch (Exception e) + { + Console.WriteLine($"{chosen.Url} : {e}"); + } } - return Links.PickRandom(); + return chosen; } } } \ No newline at end of file diff --git a/src/Ghosts.Domain/Code/TimelineBuilder.cs b/src/Ghosts.Domain/Code/TimelineBuilder.cs new file mode 100644 index 00000000..04e23d36 --- /dev/null +++ b/src/Ghosts.Domain/Code/TimelineBuilder.cs @@ -0,0 +1,131 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.IO; +using Newtonsoft.Json; +using NLog; + +namespace Ghosts.Domain.Code +{ + /// + /// Helper class that loads timeline and watches it for future changes + /// + public class TimelineBuilder + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + public static string TimelineFile = ApplicationDetails.ConfigurationFiles.Timeline; + + public static FileInfo TimelineFilePath() + { + return new FileInfo(TimelineFile); + } + + /// + /// Get from local disk + /// + /// The local timeline to be executed + public static Timeline GetLocalTimeline() + { + _log.Trace($"Loading timeline config {TimelineFile }"); + + var raw = File.ReadAllText(TimelineFile); + + var timeline = JsonConvert.DeserializeObject(raw); + if (timeline.Id == Guid.Empty) + { + timeline.Id = Guid.NewGuid(); + SetLocalTimeline(TimelineFile, timeline); + } + + _log.Debug($"Timeline {timeline.Id} loaded successfully"); + + return timeline; + } + + public static Timeline GetLocalTimeline(string path) + { + try + { + var raw = File.ReadAllText(path); + + var timeline = JsonConvert.DeserializeObject(raw); + if (timeline.Id == Guid.Empty) + { + timeline.Id = Guid.NewGuid(); + SetLocalTimeline(path, timeline); + } + + return timeline; + } + catch + { + return null; + } + } + + public static Timeline StringToTimeline(string raw) + { + try + { + var timeline = JsonConvert.DeserializeObject(raw); + return timeline; + } + catch + { + _log.Debug($"String is not a timeline: {raw}"); + return null; + } + } + + public static string TimelineToString(Timeline timeline) + { + try + { + return JsonConvert.SerializeObject(timeline); + } + catch + { + // not a timeline? + return null; + } + } + + /// + /// Save to local disk + /// + /// Raw timeline string (to be converted to `Timeline` type) + public static void SetLocalTimeline(string timelineString) + { + var timelineObject = JsonConvert.DeserializeObject(timelineString); + SetLocalTimeline(timelineObject); + } + + /// + /// Save to local disk + /// + /// `Timeline` type + public static void SetLocalTimeline(Timeline timeline) + { + using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Timeline)) + { + var serializer = new JsonSerializer + { + Formatting = Formatting.Indented + }; + serializer.Serialize(file, timeline); + } + } + + public static void SetLocalTimeline(string path, Timeline timeline) + { + using (var file = File.CreateText(path)) + { + var serializer = new JsonSerializer + { + Formatting = Formatting.Indented + }; + serializer.Serialize(file, timeline); + } + } + } +} diff --git a/src/Ghosts.Domain/Code/TimelineManager.cs b/src/Ghosts.Domain/Code/TimelineManager.cs new file mode 100644 index 00000000..0924f65c --- /dev/null +++ b/src/Ghosts.Domain/Code/TimelineManager.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Newtonsoft.Json.Linq; + +namespace Ghosts.Domain.Code +{ + public static class TimelineManager + { + public static IEnumerable GetLocalTimelines() + { + var timelines = new List + { + // get default timeline + TimelineBuilder.GetLocalTimeline() + }; + + var placesToLook = new List + { + // look for instance timelines + ApplicationDetails.InstanceDirectories.TimelineIn, + ApplicationDetails.InstanceDirectories.TimelineOut + }; + + foreach (var placeToLook in placesToLook) + { + var d = new DirectoryInfo(placeToLook); + + foreach (var file in d.GetFiles("*.*")) + { + // is this a timeline file? + try + { + var t = TimelineBuilder.GetLocalTimeline(file.FullName); + if (t != null) + { + timelines.Add(t); + } + } + catch + { + // ignored: not a timeline + } + } + } + + return timelines; + } + } + + public static class TimelineUpdateClientConfigManager + { + public static Guid GetConfigUpdateTimelineId(UpdateClientConfig config) + { + var result = new Guid(); + + try + { + if (config.Update != null) + { + var id = JObject.Parse(config.Update.ToString().ToLower())["timelineid"].ToString(); + Guid.TryParse(id, out result); + } + } + catch + { + // ignored: not valid update config + } + + return result; + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Domain/Code/TimelineTranslator.cs b/src/Ghosts.Domain/Code/TimelineTranslator.cs new file mode 100644 index 00000000..fcb35685 --- /dev/null +++ b/src/Ghosts.Domain/Code/TimelineTranslator.cs @@ -0,0 +1,101 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Linq; +using Ghosts.Domain.Code.Helpers; +using NLog; + +namespace Ghosts.Domain.Code +{ + public static class TimelineTranslator + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public static TimelineHandler FromBrowserUnitTests(IEnumerable commands) + { + var timelineHandler = new TimelineHandler(); + + timelineHandler.HandlerType = HandlerType.BrowserFirefox; + timelineHandler.Initial = "about:blank"; + timelineHandler.Loop = false; + + foreach (var command in commands) + { + TimelineEvent timelineEvent = null; + try + { + timelineEvent = GetEvent(command.Trim()); + } + catch (Exception exc) + { + _log.Error(exc); + } + + if (timelineEvent != null && !string.IsNullOrEmpty(timelineEvent.Command) && timelineEvent.CommandArgs.Count > 0) + timelineHandler.TimeLineEvents.Add(timelineEvent); + } + + return timelineHandler; + } + + private static TimelineEvent GetEvent(string command) + { + var timelineEvent = new TimelineEvent(); + timelineEvent.DelayBefore = 0; + timelineEvent.DelayAfter = 3000; + + if (command.StartsWith("driver.Quit", StringComparison.InvariantCultureIgnoreCase)) return timelineEvent; + + // determine command and commandArgs + if (command.StartsWith("driver.Navigate()", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "browse"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.Manage().Window.Size", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "manage.window.size"; + var size = command.Split("Size(").Last().Replace(");", "").Replace(" ", "").Split(Convert.ToChar(",")); + var width = Convert.ToInt32(size[0]); + var height = Convert.ToInt32(size[1]); + timelineEvent.CommandArgs.Add(width); + timelineEvent.CommandArgs.Add(height); + } + else if (command.StartsWith("js.", StringComparison.InvariantCultureIgnoreCase)) + { + if (command.StartsWith("js.ExecuteScript(", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "js.executescript"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + } + else if (command.StartsWith("driver.FindElement(", StringComparison.InvariantCultureIgnoreCase) && + command.EndsWith(").Click();", StringComparison.InvariantCultureIgnoreCase)) + { + if (command.StartsWith("driver.FindElement(By.LinkText(", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.linktext"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.Id", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.id"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.Name", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.name"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + else if (command.StartsWith("driver.FindElement(By.CssSelector", StringComparison.InvariantCultureIgnoreCase)) + { + timelineEvent.Command = "click.by.cssselector"; + timelineEvent.CommandArgs.Add(command.GetTextBetweenQuotes()); + } + } + + return timelineEvent; + } + } +} \ No newline at end of file diff --git a/src/Ghosts.Domain/Code/UserAgentManager.cs b/src/Ghosts.Domain/Code/UserAgentManager.cs index 971b9237..66242244 100755 --- a/src/Ghosts.Domain/Code/UserAgentManager.cs +++ b/src/Ghosts.Domain/Code/UserAgentManager.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using Ghosts.Domain.Code.Helpers; using NLog; namespace Ghosts.Domain.Code diff --git a/src/Ghosts.Domain/Messages/Timeline.cs b/src/Ghosts.Domain/Messages/Timeline.cs index 7755291b..884f4ae6 100755 --- a/src/Ghosts.Domain/Messages/Timeline.cs +++ b/src/Ghosts.Domain/Messages/Timeline.cs @@ -8,10 +8,15 @@ namespace Ghosts.Domain { /// - /// an array of events that a client should perform in order to best mimic some persona x + /// an array of events that a client should perform in order to best mimic some persona x /// public class Timeline { + /// + /// Useful for tracking where activity on a client originated + /// + public Guid Id { get; set; } + [JsonConverter(typeof(StringEnumConverter))] public enum TimelineStatus { @@ -25,7 +30,7 @@ public Timeline() } /// - /// Run or Stop + /// Run or Stop /// [JsonConverter(typeof(StringEnumConverter))] public TimelineStatus Status { get; set; } @@ -34,7 +39,7 @@ public Timeline() } /// - /// an array of application events that a client will execute - aka "randomly browse 6 different web pages for new shoes at 0900" + /// an array of application events that a client will execute - aka "randomly browse 6 different web pages for new shoes at 0900" /// public class TimelineHandler { @@ -48,7 +53,7 @@ public TimelineHandler() public HandlerType HandlerType { get; set; } /// - /// Used to instantiate browser object + /// Used to instantiate browser object /// public string Initial { get; set; } @@ -64,7 +69,7 @@ public TimelineHandler() } /// - /// handlers map to applications + /// handlers map to applications /// public enum HandlerType { @@ -90,7 +95,7 @@ public enum HandlerType } /// - /// The specific events that a handler will execute + /// The specific events that a handler will execute /// public class TimelineEvent { @@ -100,7 +105,7 @@ public TimelineEvent() } /// - /// AlertIds trace back to an alert that monitors specific activity executed within a timeline + /// AlertIds trace back to an alert that monitors specific activity executed within a timeline /// public string TrackableId { get; set; } @@ -108,18 +113,18 @@ public TimelineEvent() public List CommandArgs { get; set; } /// - /// Milliseconds + /// In milliseconds /// public int DelayAfter { get; set; } /// - /// Milliseconds + /// In milliseconds /// public int DelayBefore { get; set; } } /// - /// Gets passed back to api server 'Chrome, Browse, http://cnn.com' + /// Gets passed back to api server 'Chrome, Browse, https://cmu.edu' /// public class TimeLineRecord { diff --git a/src/Ghosts.Domain/Models/ThreadJob.cs b/src/Ghosts.Domain/Models/ThreadJob.cs new file mode 100644 index 00000000..18d27eb4 --- /dev/null +++ b/src/Ghosts.Domain/Models/ThreadJob.cs @@ -0,0 +1,13 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Threading; + +namespace Ghosts.Domain.Models +{ + public class ThreadJob + { + public Guid TimelineId { get; set; } + public Thread Thread { get; set; } + } +} \ No newline at end of file diff --git a/src/Ghosts.Domain/ghosts.domain.csproj b/src/Ghosts.Domain/ghosts.domain.csproj index df540d24..def1dde1 100755 --- a/src/Ghosts.Domain/ghosts.domain.csproj +++ b/src/Ghosts.Domain/ghosts.domain.csproj @@ -3,15 +3,14 @@ netstandard2.0 Ghosts.Domain - 4.0.0.0 - 4.0.0.0 + 6.0.0.0 + 6.0.0.0 7.3 + AnyCPU;x64 - - Always - + diff --git a/src/ghosts.api.sln b/src/ghosts.api.sln index 8f26b957..99b561ac 100644 --- a/src/ghosts.api.sln +++ b/src/ghosts.api.sln @@ -9,6 +9,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution LICENSE.md = LICENSE.md README.md = README.md compose-api.yml = compose-api.yml + Dockerfile-api = Dockerfile-api EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ghosts.domain", "Ghosts.Domain\ghosts.domain.csproj", "{3BCAE63E-7914-4BD3-A751-BF69820FF1BE}" diff --git a/src/ghosts.client.linux/Comms/CheckId.cs b/src/ghosts.client.linux/Communications/CheckId.cs similarity index 78% rename from src/ghosts.client.linux/Comms/CheckId.cs rename to src/ghosts.client.linux/Communications/CheckId.cs index 2bd52214..ae053c3f 100644 --- a/src/ghosts.client.linux/Comms/CheckId.cs +++ b/src/ghosts.client.linux/Communications/CheckId.cs @@ -8,10 +8,10 @@ using Ghosts.Domain.Code; using NLog; -namespace ghosts.client.linux.Comms +namespace ghosts.client.linux.Communications { /// - /// The client ID is used in the header to save having to send hostname/user/fqdn/etc. inforamtion with every request + /// The client ID is used in the header to save having to send hostname/user/fqdn/etc. information with every request /// public static class CheckId { @@ -20,7 +20,7 @@ public static class CheckId /// /// The actual path to the client id file, specified in application config /// - public static string ConfigFile = ApplicationDetails.InstanceFiles.Id; + private static readonly string ConfigFile = ApplicationDetails.InstanceFiles.Id; /// /// Gets the agent's current id from local instance, and if it does not exist, gets an id from the server and saves it locally @@ -31,11 +31,7 @@ public static string Id { try { - if (!File.Exists(ConfigFile)) - { - return Run(); - } - return File.ReadAllText(ConfigFile); + return !File.Exists(ConfigFile) ? Run() : File.ReadAllText(ConfigFile); } catch { @@ -63,18 +59,16 @@ private static string Run() { try { - using (var reader = - new StreamReader(client.OpenRead(Program.Configuration.IdUrl))) - { - s = reader.ReadToEnd(); - _log.Debug($"{DateTime.Now} - Received client ID"); - } + using var reader = + new StreamReader(client.OpenRead(Program.Configuration.IdUrl) ?? throw new Exception("Application has invalid ID url")); + s = reader.ReadToEnd(); + _log.Debug($"{DateTime.Now} - Received client ID"); } catch (WebException wex) { if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) { - _log.Debug("No ID returned!", wex); + _log.Debug($"No ID returned from API! {wex}"); } } catch (Exception e) diff --git a/src/ghosts.client.linux/Comms/Updates.cs b/src/ghosts.client.linux/Communications/Updates.cs similarity index 65% rename from src/ghosts.client.linux/Comms/Updates.cs rename to src/ghosts.client.linux/Communications/Updates.cs index 458bdfd3..2067bd9d 100644 --- a/src/ghosts.client.linux/Comms/Updates.cs +++ b/src/ghosts.client.linux/Communications/Updates.cs @@ -1,7 +1,9 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Text; using System.Threading; @@ -10,8 +12,8 @@ using Ghosts.Domain; using Ghosts.Domain.Code; using Ghosts.Domain.Messages.MesssagesForServer; -using Newtonsoft.Json; using NLog; +using Newtonsoft.Json; namespace ghosts.client.linux.Comms { @@ -45,9 +47,13 @@ private static void GetServerUpdates() if (!Program.Configuration.ClientUpdates.IsEnabled) return; + // ignore all certs + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); - Thread.Sleep(Program.Configuration.ClientUpdates.CycleSleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientUpdates.CycleSleep)); while (true) { @@ -67,14 +73,18 @@ private static void GetServerUpdates() } catch (WebException wex) { - if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) + if (wex?.Response == null) + { + _log.Debug($"{DateTime.Now} - API Server appears to be not responding"); + } + else if (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) { _log.Debug($"{DateTime.Now} - No new configuration found"); } } catch (Exception e) { - _log.Error(e); + _log.Error($"Exception in connecting to server: {e.Message}"); } } @@ -84,6 +94,9 @@ private static void GetServerUpdates() switch (update.Type) { + case UpdateClientConfig.UpdateType.RequestForTimeline: + PostCurrentTimeline(update); + break; case UpdateClientConfig.UpdateType.Timeline: TimelineBuilder.SetLocalTimeline(update.Update.ToString()); break; @@ -104,8 +117,7 @@ private static void GetServerUpdates() } } - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + Orchestrator.RunCommand(timeline, timelineHandler); } } catch (Exception exc) @@ -116,7 +128,7 @@ private static void GetServerUpdates() break; case UpdateClientConfig.UpdateType.Health: { - var newTimeline = JsonConvert.DeserializeObject(update.Update.ToString()); + var newTimeline = JsonConvert.DeserializeObject(update.Update.ToString()); //save to local disk using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Health)) { @@ -128,10 +140,8 @@ private static void GetServerUpdates() break; } default: - { _log.Debug($"Update {update.Type} has no handler, ignoring..."); break; - } } } } @@ -141,7 +151,72 @@ private static void GetServerUpdates() _log.Error(e); } - Thread.Sleep(Program.Configuration.ClientUpdates.CycleSleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientUpdates.CycleSleep)); + } + } + + private static void PostCurrentTimeline(UpdateClientConfig update) + { + // is the config for a specific timeline id? + var timelineId = TimelineUpdateClientConfigManager.GetConfigUpdateTimelineId(update); + + // get all timelines + var localTimelines = TimelineManager.GetLocalTimelines(); + + var timelines = localTimelines as Timeline[] ?? localTimelines.ToArray(); + if (timelineId != Guid.Empty) + { + foreach (var timeline in timelines) + { + if (timeline.Id == timelineId) + { + timelines = new List() + { + timeline + }.ToArray(); + break; + } + } + } + + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + + var posturl = string.Empty; + + try + { + posturl = Program.Configuration.IdUrl.Replace("clientid", "clienttimeline"); + } + catch + { + _log.Error("Can't get timeline posturl!"); + return; + } + + foreach (var timeline in timelines) + { + try + { + _log.Trace("posting timeline"); + + var payload = TimelineBuilder.TimelineToString(timeline); + var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); + + using (var client = WebClientBuilder.Build(machine)) + { + client.Headers[HttpRequestHeader.ContentType] = "application/json"; + client.UploadString(posturl, JsonConvert.SerializeObject(payload)); + } + + _log.Trace($"{DateTime.Now} - timeline posted to server successfully"); + } + catch (Exception e) + { + _log.Debug( + $"Problem posting timeline to server from {ApplicationDetails.ConfigurationFiles.Timeline} to {posturl}"); + _log.Error(e); + } } } @@ -150,19 +225,22 @@ private static void PostClientResults() if (!Program.Configuration.ClientResults.IsEnabled) return; + // ignore all certs + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + var fileName = ApplicationDetails.LogFiles.ClientUpdates; - var cyclesleep = Program.Configuration.ClientResults.CycleSleep; var posturl = Program.Configuration.ClientResults.PostUrl; var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); - Thread.Sleep(cyclesleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientResults.CycleSleep)); while (true) { try { - if(File.Exists(fileName)) + if (File.Exists(fileName)) { PostResults(fileName, machine, posturl); } @@ -173,13 +251,18 @@ private static void PostClientResults() } catch (Exception e) { - _log.Error($"Problem posting logs to server {e}"); + _log.Error($"Problem posting logs to server: {e.Message}"); + } + finally + { + GC.Collect(); + GC.WaitForPendingFinalizers(); } // look for other result files that have not been posted try { - foreach(var file in Directory.GetFiles(Path.GetDirectoryName(fileName))) + foreach (var file in Directory.GetFiles(Path.GetDirectoryName(fileName))) { if (!file.EndsWith("app.log") && file != fileName) { @@ -189,10 +272,15 @@ private static void PostClientResults() } catch (Exception e) { - _log.Debug($"Problem posting overflow logs from {fileName} to server {posturl}: {e}"); + _log.Debug($"Problem posting overflow logs from {fileName} to server {posturl} : {e}"); + } + finally + { + GC.Collect(); + GC.WaitForPendingFinalizers(); } - Thread.Sleep(cyclesleep); + Thread.Sleep(Jitter.Basic(Program.Configuration.ClientResults.CycleSleep)); } } @@ -241,11 +329,26 @@ private static void PostResults(string fileName, ResultMachine machine, string p internal static void PostSurvey() { + // ignore all certs + ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; + + var posturl = string.Empty; + + try + { + posturl = Program.Configuration.Survey.PostUrl; + } + catch + { + _log.Error("Can't get survey posturl!"); + return; + } + try { _log.Trace("posting survey"); - var posturl = Program.Configuration.Survey.PostUrl; + Thread.Sleep(Jitter.Basic(100)); if (!File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) return; @@ -255,6 +358,7 @@ internal static void PostSurvey() var payload = JsonConvert.SerializeObject(survey); var machine = new ResultMachine(); + // GuestInfoVars.Load(machine); if (Program.Configuration.Survey.IsSecure) { @@ -279,7 +383,7 @@ internal static void PostSurvey() } catch (Exception e) { - _log.Trace("Problem posting logs to server"); + _log.Debug($"Problem posting logs to server from { ApplicationDetails.InstanceFiles.SurveyResults } to { Program.Configuration.Survey.PostUrl }"); _log.Error(e); } } diff --git a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs index 7e4697c9..8e5a5974 100755 --- a/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseBrowserHandler.cs @@ -2,12 +2,15 @@ using System; using System.Threading; +using System.Threading.Tasks; using ghosts.client.linux.Infrastructure.Browser; using Ghosts.Domain; using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; using NLog; using OpenQA.Selenium; using OpenQA.Selenium.Interactions; +// ReSharper disable StringLiteralTypo namespace ghosts.client.linux.handlers { @@ -15,11 +18,19 @@ public abstract class BaseBrowserHandler : BaseHandler { public static readonly Logger _log = LogManager.GetCurrentClassLogger(); public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } public HandlerType BrowserType { get; set; } private int _stickiness = 0; private int _depthMin = 1; private int _depthMax = 10; + private LinkManager _linkManager; + private Task LaunchThread(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + var o = new BrowserCrawl(); + return o.Crawl(handler, timelineEvent, site); + } + public void ExecuteEvents(TimelineHandler handler) { try @@ -40,6 +51,29 @@ public void ExecuteEvents(TimelineHandler handler) switch (timelineEvent.Command) { + case "crawl": + var _taskMax = 1; + if (handler.HandlerArgs.ContainsKey("crawl-tasks-maximum")) + { + int.TryParse(handler.HandlerArgs["crawl-tasks-maximum"], out _taskMax); + } + + var i = 0; + foreach (var site in timelineEvent.CommandArgs) + { + //LaunchThread(handler, timelineEvent, site.ToString()); + + Task.Factory.StartNew(() => LaunchThread(handler, timelineEvent, site.ToString())); + Thread.Sleep(5000); + i++; + + if (i >= _taskMax) + { + Task.WaitAll(); + i = 0; + } + } + break; case "random": // setup @@ -80,32 +114,9 @@ public void ExecuteEvents(TimelineHandler handler) { try { - var linkManager = new LinkManager(config.GetHost()); - - - //get all links - var links = Driver.FindElements(By.TagName("a")); - foreach (var l in links) - { - var node = l.GetAttribute("href"); - if (string.IsNullOrEmpty(node) || - node.ToLower().StartsWith("//")) - { - //skip, these seem ugly - } - // http|s links - else if (node.ToLower().StartsWith("http")) - { - linkManager.AddLink(node.ToLower()); - } - // relative links - prefix the scheme and host - else - { - linkManager.AddLink($"{config.GetHost()}{node.ToLower()}"); - } - } - - var link = linkManager.Choose(); + this._linkManager = new LinkManager(config.Uri); + GetAllLinks(config, false); + var link = this._linkManager.Choose(); if (link == null) { return; @@ -157,15 +168,33 @@ public void ExecuteEvents(TimelineHandler handler) actions.SendKeys(element, timelineEvent.CommandArgs[1].ToString()).Build().Perform(); break; case "click": + case "click.by.name": element = Driver.FindElement(By.Name(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; case "clickbyid": + case "click.by.id": element = Driver.FindElement(By.Id(timelineEvent.CommandArgs[0].ToString())); actions = new Actions(Driver); actions.MoveToElement(element).Click().Perform(); break; + case "click.by.linktext": + element = Driver.FindElement(By.LinkText(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "click.by.cssselector": + element = Driver.FindElement(By.CssSelector(timelineEvent.CommandArgs[0].ToString())); + actions = new Actions(Driver); + actions.MoveToElement(element).Click().Perform(); + break; + case "js.executescript": + JS.ExecuteScript(timelineEvent.CommandArgs[0].ToString()); + break; + case "manage.window.size": + Driver.Manage().Window.Size = new System.Drawing.Size(Convert.ToInt32(timelineEvent.CommandArgs[0]), Convert.ToInt32(timelineEvent.CommandArgs[1])); + break; } if (timelineEvent.DelayAfter > 0) @@ -180,6 +209,38 @@ public void ExecuteEvents(TimelineHandler handler) } } + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + try + { + var links = Driver.FindElements(By.TagName("a")); + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + this._linkManager.AddLink(uri, 1); + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + } + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + } + private void MakeRequest(RequestConfiguration config) { // Added try here because some versions of FF (v56) throw an exception for an unresolved site, diff --git a/src/ghosts.client.linux/Handlers/BaseHandler.cs b/src/ghosts.client.linux/Handlers/BaseHandler.cs index 3ac68538..311dde60 100644 --- a/src/ghosts.client.linux/Handlers/BaseHandler.cs +++ b/src/ghosts.client.linux/Handlers/BaseHandler.cs @@ -11,17 +11,14 @@ public abstract class BaseHandler { private static readonly Logger _timelineLog = LogManager.GetLogger("TIMELINE"); - public void Report(string handler, string command, string arg) + protected static void Report(string handler, string command, string arg, string trackable = null) { - Report(handler, command, arg, null); - } - - public void Report(string handler, string command, string arg, string trackable) - { - var result = new TimeLineRecord(); - result.Handler = handler; - result.Command = command; - result.CommandArg = arg; + var result = new TimeLineRecord + { + Handler = handler, + Command = command, + CommandArg = arg + }; if (!string.IsNullOrEmpty(trackable)) { diff --git a/src/ghosts.client.linux/Handlers/Bash.cs b/src/ghosts.client.linux/Handlers/Bash.cs index 8449a9de..58e459c8 100644 --- a/src/ghosts.client.linux/Handlers/Bash.cs +++ b/src/ghosts.client.linux/Handlers/Bash.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; +using System.Linq; using System.Threading; using Ghosts.Domain; using Ghosts.Domain.Code; @@ -12,7 +13,7 @@ namespace ghosts.client.linux.handlers public class Bash : BaseHandler { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public string Result { get; private set; } + private string Result { get; set; } public Bash(TimelineHandler handler) { @@ -36,7 +37,7 @@ public Bash(TimelineHandler handler) } } - public void Ex(TimelineHandler handler) + private void Ex(TimelineHandler handler) { foreach (var timelineEvent in handler.TimeLineEvents) { @@ -53,9 +54,10 @@ public void Ex(TimelineHandler handler) this.Command(handler.Initial, timelineEvent.Command); - foreach (var cmd in timelineEvent.CommandArgs) - if (!string.IsNullOrEmpty(cmd.ToString())) - this.Command(handler.Initial, cmd.ToString()); + foreach (var cmd in timelineEvent.CommandArgs.Where(cmd => !string.IsNullOrEmpty(cmd.ToString()))) + { + this.Command(handler.Initial, cmd.ToString()); + } break; } @@ -88,15 +90,15 @@ private void Command(string initial, string command) p.WaitForExit(); - this.Report(HandlerType.Command.ToString(), escapedArgs, this.Result); + Report(HandlerType.Command.ToString(), escapedArgs, this.Result); } - void OutputHandler(object sendingProcess, DataReceivedEventArgs outLine) + private void OutputHandler(object sendingProcess, DataReceivedEventArgs outLine) { this.Result += outLine.Data; } - void ErrorHandler(object sendingProcess, DataReceivedEventArgs outLine) + private static void ErrorHandler(object sendingProcess, DataReceivedEventArgs outLine) { //* Do your stuff with the output (write to console/log/StringBuilder) Console.WriteLine(outLine.Data); diff --git a/src/ghosts.client.linux/Handlers/BrowserChrome.cs b/src/ghosts.client.linux/Handlers/BrowserChrome.cs new file mode 100644 index 00000000..8147a8a6 --- /dev/null +++ b/src/ghosts.client.linux/Handlers/BrowserChrome.cs @@ -0,0 +1,134 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using Ghosts.Domain; +using OpenQA.Selenium.Chrome; +using System; +using Ghosts.Domain.Code.Helpers; +using OpenQA.Selenium; +// ReSharper disable StringLiteralTypo + +namespace ghosts.client.linux.handlers +{ + public class BrowserChrome : BaseBrowserHandler + { + public new IJavaScriptExecutor JS { get; private set; } + + public BrowserChrome(TimelineHandler handler) + { + BrowserType = HandlerType.BrowserChrome; + try + { + Driver = GetDriver(handler); + base.Driver = Driver; + + if (handler.HandlerArgs.ContainsKey("javascript-enable")) + { + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; + } + + Driver.Navigate().GoToUrl(handler.Initial); + + if (handler.Loop) + { + while (true) + { + ExecuteEvents(handler); + } + } + else + { + ExecuteEvents(handler); + } + } + catch (Exception e) + { + _log.Error(e); + } + } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var options = new ChromeOptions(); + options.AddArguments("disable-infobars"); + options.AddArguments("disable-logging"); + options.AddArguments("--disable-logging"); + options.AddArgument("--log-level=3"); + options.AddArgument("--silent"); + + options.AddUserProfilePreference("download.default_directory", @"%homedrive%%homepath%\\Downloads"); + options.AddUserProfilePreference("disable-popup-blocking", "true"); + + if (handler.HandlerArgs != null) + { + if (handler.HandlerArgs.ContainsKey("executable-location") && + !string.IsNullOrEmpty(handler.HandlerArgs["executable-location"])) + { + options.BinaryLocation = handler.HandlerArgs["executable-location"]; + } + + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("headless"); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.stylesheets", 2); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + // ? + } + + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.javascript", 1); + } + } + + options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2); + options.AddUserProfilePreference("profile.default_content_setting_values.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.cookies", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.plugins", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.popups", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.geolocation", 2); + options.AddUserProfilePreference("profile.managed_default_content_settings.media_stream", 2); + + if (!string.IsNullOrEmpty(Program.Configuration.ChromeExtensions)) + { + options.AddArguments($"--load-extension={Program.Configuration.ChromeExtensions}"); + } + + _log.Trace("Browser preferences set successfully, getting driver..."); + + ChromeDriver driver; + + try + { + driver = new ChromeDriver(options); + } + catch + { + _log.Trace("Driver could not be instantiated. Does the proper driver exist? Are you running as a user and not root? Sometimes running the driver directly will uncover the underlaying issue."); + throw; + } + + _log.Trace("Driver instantiated successfully, setting timeouts..."); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + _log.Trace("Driver timeouts set successfully, continuing..."); + return driver; + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserCrawl.cs b/src/ghosts.client.linux/Handlers/BrowserCrawl.cs new file mode 100644 index 00000000..6f51e1e7 --- /dev/null +++ b/src/ghosts.client.linux/Handlers/BrowserCrawl.cs @@ -0,0 +1,219 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Linq; +using System.Threading.Tasks; +using ghosts.client.linux.Infrastructure.Browser; +using Ghosts.Domain; +using Ghosts.Domain.Code; +using Ghosts.Domain.Code.Helpers; +using NLog; +using OpenQA.Selenium; + +namespace ghosts.client.linux.handlers +{ + class BrowserCrawl : BaseHandler + { + public static readonly Logger _log = LogManager.GetCurrentClassLogger(); + public IWebDriver Driver { get; set; } + public IJavaScriptExecutor JS { get; set; } + private int _stickiness = 0; + private LinkManager _linkManager; + private int _pageBrowseCount = 0; + private string _proxyLocalUrl = string.Empty; + private int _siteDepthMax = 1; + private int _siteDepthCurrent = 0; + + internal Task Crawl(TimelineHandler handler, TimelineEvent timelineEvent, string site) + { + switch (handler.HandlerType) + { + case HandlerType.BrowserChrome: + this.Driver = BrowserChrome.GetDriver(handler); + break; + case HandlerType.BrowserFirefox: + this.Driver = BrowserFirefox.GetDriver(handler); + break; + } + + Console.WriteLine($"{Environment.CurrentManagedThreadId} handle: {Driver.CurrentWindowHandle}"); + + if (handler.HandlerArgs.ContainsKey("stickiness")) + { + int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness); + } + + if (handler.HandlerArgs.ContainsKey("crawl-site-depth")) + { + int.TryParse(handler.HandlerArgs["crawl-site-depth"], out _siteDepthMax); + } + + if (handler.HandlerArgs.ContainsKey("crawl-proxy-local-url")) + { + _proxyLocalUrl = handler.HandlerArgs["crawl-proxy-local-url"]; + } + + this._pageBrowseCount = 0; + var config = RequestConfiguration.Load(site); + this._linkManager = new LinkManager(config.Uri); + if (config.Uri.IsWellFormedOriginalString()) + { + MakeRequest(config); + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + this._siteDepthCurrent += 1; + + if (this._siteDepthCurrent >= this._siteDepthMax) + return Task.CompletedTask; + + GetAllLinks(config, true); + CrawlAllLinks(config, handler, timelineEvent, true); + } + + Driver.Close(); + Driver.Quit(); + _log.Trace($"Run complete for {site}"); + return Task.CompletedTask; + } + + private void GetAllLinks(RequestConfiguration config, bool sameSite) + { + _log.Trace($"Getting links for {config.Uri}..."); + var linksAdded = 0; + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness}..."); + return; + } + + try + { + var isInIframe = false; + // for use with pywb and proxy scraping + var iframes = Driver.FindElements(By.TagName("iframe")); + foreach(var iframe in iframes) + { + if (iframe.GetAttribute("id") == "replay_iframe") + { + Driver.SwitchTo().Frame(iframe); + isInIframe = true; + _log.Trace("replay_iframe found. Made that the focus..."); + } + _log.Trace($"Iframe found: {iframe.GetAttribute("id")}"); + } + + var links = Driver.FindElements(By.TagName("a")); + + foreach (var l in links) + { + var node = l.GetAttribute("href"); + if (string.IsNullOrEmpty(node)) + continue; + node = node.ToLower(); + if (isInIframe && !string.IsNullOrEmpty(this._proxyLocalUrl)) + node = this._proxyLocalUrl + node; + if (Uri.TryCreate(node, UriKind.RelativeOrAbsolute, out var uri)) + { + if (uri.GetDomain() != config.Uri.GetDomain()) + { + if (!sameSite) + { + this._linkManager.AddLink(uri, 1); + linksAdded += 1; + } + } + // relative links - prefix the scheme and host + else + { + this._linkManager.AddLink(uri, 2); + linksAdded += 1; + } + } + } + + if (isInIframe) + { + Driver.SwitchTo().DefaultContent(); + _log.Trace("Switched back to main window focus"); + } + + _log.Trace($"Added {linksAdded} links for {config.Uri}"); + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private void CrawlAllLinks(RequestConfiguration config, TimelineHandler handler, + TimelineEvent timelineEvent, bool sameSite) + { + _log.Trace($"Crawling links for {config.Uri}"); + if (this._linkManager?.Links == null) + { + return; + } + if (this._pageBrowseCount > this._stickiness) + { + return; + } + + foreach (var link in this._linkManager.Links.Where(x => x.WasBrowsed == false) + .OrderByDescending(x => x.Priority)) + { + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (2)..."); + return; + } + + if (this._linkManager.Links.Any(x => x.Url.ToString() == link.Url.ToString() && x.WasBrowsed)) + { + continue; + } + + config.Method = "GET"; + config.Uri = link.Url; + + MakeRequest(config); + + foreach (var l in this._linkManager.Links.Where(x => x.Url.ToString() == link.Url.ToString())) + { + l.WasBrowsed = true; + _log.Trace($"Skipping {config.Uri} (already browsed)"); + } + this._pageBrowseCount += 1; + if (this._pageBrowseCount > this._stickiness) + { + _log.Trace($"Exceeded stickiness for {config.Uri} {this._stickiness} (3)..."); + return; + } + + Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), + timelineEvent.TrackableId); + + // if this is the last step down, there is no reason to keep digging, + // but we don't increase the current depth count so as to allow peer + // pages at this level to still be scraped + if (this._siteDepthCurrent + 1 < this._siteDepthMax) + { + _log.Trace($"Drilling into {config.Uri}..."); + GetAllLinks(config, sameSite); + CrawlAllLinks(config, handler, timelineEvent, sameSite); + } + } + } + + private void MakeRequest(RequestConfiguration config) + { + try + { + Driver.Navigate().GoToUrl(config.Uri); + } + catch (Exception e) + { + _log.Trace($"Requst error for {config.Uri}: {e.Message}"); + } + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs index 6d9f9506..5d5e54af 100755 --- a/src/ghosts.client.linux/Handlers/BrowserFirefox.cs +++ b/src/ghosts.client.linux/Handlers/BrowserFirefox.cs @@ -3,18 +3,23 @@ using Ghosts.Domain; using NLog; using System; +using System.Diagnostics; using System.IO; +using System.Runtime.CompilerServices; using System.Text; +using Ghosts.Domain.Code.Helpers; using OpenQA.Selenium; using OpenQA.Selenium.Firefox; +// ReSharper disable StringLiteralTypo namespace ghosts.client.linux.handlers { public class BrowserFirefox : BaseBrowserHandler { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private new static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public IWebDriver Driver { get; private set; } + public new IWebDriver Driver { get; private set; } + public new IJavaScriptExecutor JS { get; private set; } public BrowserFirefox(TimelineHandler handler) { @@ -26,7 +31,25 @@ public BrowserFirefox(TimelineHandler handler) } } - private string GetInstallLocation() + private static int GetFirefoxVersion(string path) + { + var versionInfo = FileVersionInfo.GetVersionInfo(path); + return versionInfo.FileMajorPart; + } + + private static bool IsSufficientVersion(string path) + { + int currentVersion = GetFirefoxVersion(path); + int minimumVersion = Program.Configuration.FirefoxMajorVersionMinimum; + if (currentVersion < minimumVersion) + { + _log.Debug($"Firefox version ({currentVersion}) is incompatible - requires at least {minimumVersion}"); + return false; + } + return true; + } + + internal static string GetInstallLocation() { var path = "/bin/firefox"; if (File.Exists(path)) @@ -35,61 +58,24 @@ private string GetInstallLocation() } path = "/usr/bin/firefox"; - return File.Exists(path) ? path : Program.Configuration.FirefoxInstallLocation; + var retVal = File.Exists(path) ? path : Program.Configuration.FirefoxInstallLocation; + _log.Trace($"Using install location of [{retVal}]"); + return retVal; } private bool FirefoxEx(TimelineHandler handler) { try { - var path = GetInstallLocation(); - - FirefoxOptions options = new FirefoxOptions(); - options.AddArguments("--disable-infobars"); - options.AddArguments("--disable-extensions"); - options.AddArguments("--disable-notifications"); - - //options.BrowserExecutableLocation = path; - options.Profile = new FirefoxProfile(); + Driver = GetDriver(handler); + base.Driver = Driver; - if (handler.HandlerArgs != null) + if (handler.HandlerArgs.ContainsKey("javascript-enable")) { - if (handler.HandlerArgs.ContainsKey("isheadless") && handler.HandlerArgs["isheadless"] == "true") - { - options.AddArguments("--headless"); - } - if (handler.HandlerArgs.ContainsKey("incognito") && handler.HandlerArgs["incognito"] == "true") - { - options.AddArguments("--incognito"); - } - if (handler.HandlerArgs.ContainsKey("blockstyles") && handler.HandlerArgs["blockstyles"] == "true") - { - options.Profile.SetPreference("permissions.default.stylesheet", 2); - } - if (handler.HandlerArgs.ContainsKey("blockimages") && handler.HandlerArgs["blockimages"] == "true") - { - options.Profile.SetPreference("permissions.default.image", 2); - } - if (handler.HandlerArgs.ContainsKey("blockflash") && handler.HandlerArgs["blockflash"] == "true") - { - options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); - } - if (handler.HandlerArgs.ContainsKey("blockscripts") && handler.HandlerArgs["blockscripts"] == "true") - { - options.Profile.SetPreference("permissions.default.script", 2); - } + JS = (IJavaScriptExecutor)Driver; + base.JS = JS; } - options.Profile.SetPreference("permissions.default.cookies", 2); - options.Profile.SetPreference("permissions.default.popups", 2); - options.Profile.SetPreference("permissions.default.geolocation", 2); - options.Profile.SetPreference("permissions.default.media_stream", 2); - - CodePagesEncodingProvider.Instance.GetEncoding(437); - Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); - Driver = new FirefoxDriver(options); - base.Driver = Driver; - //hack: bad urls used in the past... if (handler.Initial.Equals("") || handler.Initial.Equals("about:internal", StringComparison.InvariantCultureIgnoreCase) || @@ -122,12 +108,79 @@ private bool FirefoxEx(TimelineHandler handler) _log.Debug(e); return false; } - finally + + return true; + } + + internal static IWebDriver GetDriver(TimelineHandler handler) + { + var path = GetInstallLocation(); + + var options = new FirefoxOptions(); + options.BrowserExecutableLocation = path; + options.AddArguments("--disable-infobars"); + options.AddArguments("--disable-extensions"); + options.AddArguments("--disable-notifications"); + + options.Profile = new FirefoxProfile(); + + if (handler.HandlerArgs != null) { - + if (handler.HandlerArgs.ContainsKeyWithOption("isheadless", "true")) + { + options.AddArguments("--headless"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("incognito", "true")) + { + options.AddArguments("--incognito"); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockstyles", "true")) + { + options.Profile.SetPreference("permissions.default.stylesheet", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockimages", "true")) + { + options.Profile.SetPreference("permissions.default.image", 2); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockflash", "true")) + { + options.Profile.SetPreference("dom.ipc.plugins.enabled.libflashplayer.so", false); + } + if (handler.HandlerArgs.ContainsKeyWithOption("blockscripts", "true")) + { + options.Profile.SetPreference("permissions.default.script", 2); + } } - return true; + options.Profile.SetPreference("permissions.default.cookies", 2); + options.Profile.SetPreference("permissions.default.popups", 2); + options.Profile.SetPreference("permissions.default.geolocation", 2); + options.Profile.SetPreference("permissions.default.media_stream", 2); + + options.Profile.SetPreference("geo.enabled", false); + options.Profile.SetPreference("geo.prompt.testing", false); + options.Profile.SetPreference("geo.prompt.testing.allow", false); + + _log.Trace("Browser preferences set successfully, getting driver..."); + + CodePagesEncodingProvider.Instance.GetEncoding(437); + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + FirefoxDriver driver; + + try + { + driver = new FirefoxDriver(options); + } + catch + { + _log.Trace("Driver could not be instantiated. Does the proper driver exist? Are you running as a user and not root? Sometimes running the driver directly will uncover the underlaying issue."); + throw; + } + + _log.Trace("Driver instantiated successfully, setting timeouts..."); + driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); + _log.Trace("Driver timeouts set successfully, continuing..."); + return driver; } } -} +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Handlers/Curl.cs b/src/ghosts.client.linux/Handlers/Curl.cs index e7aa3cb2..6fc93fa1 100755 --- a/src/ghosts.client.linux/Handlers/Curl.cs +++ b/src/ghosts.client.linux/Handlers/Curl.cs @@ -14,9 +14,9 @@ namespace ghosts.client.linux.handlers public class Curl : BaseHandler { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - public string Result { get; private set; } + private string Result { get; set; } private readonly TimelineHandler _handler; - private readonly int _stickiness = 0; + private readonly int _stickiness; private readonly int _depthMin = 1; private readonly int _depthMax = 10; private int _wait = 500; @@ -55,10 +55,8 @@ public Curl(TimelineHandler handler) Ex(); } } - else - { - Ex(); - } + + Ex(); } catch (Exception e) { @@ -80,22 +78,18 @@ private void Ex() default: this.Command(timelineEvent.Command); - foreach (var cmd in timelineEvent.CommandArgs) + foreach (var cmd in timelineEvent.CommandArgs.Where(cmd => !string.IsNullOrEmpty(cmd.ToString()))) { - if (!string.IsNullOrEmpty(cmd.ToString())) - { - this.Command(cmd.ToString()); - } + this.Command(cmd.ToString()); } break; } - if (timelineEvent.DelayAfter > 0) - { - _wait = timelineEvent.DelayAfter; - Thread.Sleep(timelineEvent.DelayAfter); - } + if (timelineEvent.DelayAfter <= 0) continue; + + _wait = timelineEvent.DelayAfter; + Thread.Sleep(timelineEvent.DelayAfter); } } @@ -141,7 +135,7 @@ private void Command(string command) this.Result += p.StandardOutput.ReadToEnd(); } - this.Report(HandlerType.Curl.ToString(), escapedArgs, this.Result); + Report(HandlerType.Curl.ToString(), escapedArgs, this.Result); this.DeepBrowse(); } catch (Exception exc) @@ -155,72 +149,69 @@ private void Command(string command) /// private void DeepBrowse() { - if (_stickiness > 0) + if (_stickiness <= 0) return; + var random = new Random(); + if (random.Next(100) >= _stickiness) return; + + // some percentage of the time, we should stay on this site + var loops = random.Next(_depthMin, _depthMax); + for (var loopNumber = 0; loopNumber < loops; loopNumber++) { - var random = new Random(); - // some percentage of the time, we should stay on this site - if (random.Next(100) < _stickiness) + try { - var loops = random.Next(_depthMin, _depthMax); - for (var loopNumber = 0; loopNumber < loops; loopNumber++) + //get all links from results + var doc = new HtmlDocument(); + doc.LoadHtml(this.Result.ToLower()); + + var nodes = doc.DocumentNode.SelectNodes("//a"); + if (nodes == null || !nodes.Any()) { - try + return; + } + + var linkManager = new LinkManager(new Uri(this._currentHost)); + foreach (var node in nodes) + { + if (!node.HasAttributes + || node.Attributes["href"] == null + || string.IsNullOrEmpty(node.Attributes["href"].Value) + || node.Attributes["href"].Value.ToLower().StartsWith("//")) { - //get all links from results - var doc = new HtmlDocument(); - doc.LoadHtml(this.Result.ToLower()); - - var nodes = doc.DocumentNode.SelectNodes("//a"); - if (nodes == null || !nodes.Any()) - { - return; - } - - var linkManager = new LinkManager(this._currentHost); - foreach (var node in nodes) - { - if (!node.HasAttributes - || node.Attributes["href"] == null - || string.IsNullOrEmpty(node.Attributes["href"].Value) - || node.Attributes["href"].Value.ToLower().StartsWith("//")) - { - //skip, these seem ugly - } - // http|s links - else if (node.Attributes["href"].Value.ToLower().StartsWith("http")) - { - linkManager.AddLink(node.Attributes["href"].Value.ToLower()); - } - // relative links - prefix the scheme and host - else - { - linkManager.AddLink($"{this._currentHost}{node.Attributes["href"].Value.ToLower()}"); - } - } - - var link = linkManager.Choose(); - if (link == null) - { - return; - } - var href = link.Url.ToString(); - - if (!string.IsNullOrEmpty(href)) - { - this.Result = ""; - Command(href); - } + //skip, these seem ugly } - catch (Exception e) + // http|s links + else if (node.Attributes["href"].Value.ToLower().StartsWith("http")) { - _log.Error(e); + linkManager.AddLink(node.Attributes["href"].Value.ToLower(), 1); } - - if (_wait > 0) + // relative links - prefix the scheme and host + else { - Thread.Sleep(_wait); + linkManager.AddLink($"{this._currentHost}{node.Attributes["href"].Value.ToLower()}", 2); } } + + var link = linkManager.Choose(); + if (link == null) + { + return; + } + var href = link.Url.ToString(); + + if (!string.IsNullOrEmpty(href)) + { + this.Result = ""; + Command(href); + } + } + catch (Exception e) + { + _log.Error(e); + } + + if (_wait > 0) + { + Thread.Sleep(_wait); } } } diff --git a/src/ghosts.client.linux/Handlers/NpcSystem.cs b/src/ghosts.client.linux/Handlers/NpcSystem.cs new file mode 100644 index 00000000..3b8b47d6 --- /dev/null +++ b/src/ghosts.client.linux/Handlers/NpcSystem.cs @@ -0,0 +1,51 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Linq; +using ghosts.client.linux.Infrastructure; +using ghosts.client.linux.timelineManager; +using Ghosts.Domain; +using Ghosts.Domain.Code; +using NLog; + +namespace ghosts.client.linux.handlers +{ + public class NpcSystem : BaseHandler + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public NpcSystem(Timeline timeline, TimelineHandler handler) + { + _log.Trace($"Handling NpcSystem call: {handler}"); + + foreach (var timelineEvent in handler.TimeLineEvents.Where(timelineEvent => !string.IsNullOrEmpty(timelineEvent.Command))) + { + Timeline t; + + switch (timelineEvent.Command.ToLower()) + { + case "start": + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Run; + TimelineBuilder.SetLocalTimeline(t); + break; + case "stop": + if (timeline.Id != Guid.Empty) + { + Orchestrator.StopTimeline(timeline.Id); + } + else + { + t = TimelineBuilder.GetLocalTimeline(); + t.Status = Timeline.TimelineStatus.Stop; + StartupTasks.CleanupProcesses(); + TimelineBuilder.SetLocalTimeline(t); + } + + break; + } + } + } + + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/Health/Check.cs b/src/ghosts.client.linux/Health/Check.cs index f354d462..3b6f029b 100644 --- a/src/ghosts.client.linux/Health/Check.cs +++ b/src/ghosts.client.linux/Health/Check.cs @@ -18,7 +18,7 @@ public class Check private static readonly Logger _healthLog = LogManager.GetLogger("HEALTH"); private static DateTime _lastRead = DateTime.MinValue; - private List _threads { get; set; } + private List _threads { get; } public Check() { @@ -35,7 +35,7 @@ public void Run() _log.Trace($"watching {watcher.Path}"); watcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size; watcher.EnableRaisingEvents = true; - watcher.Changed += new FileSystemEventHandler(OnChanged); + watcher.Changed += OnChanged; Thread t = null; new Thread(() => @@ -43,15 +43,13 @@ public void Run() Thread.CurrentThread.IsBackground = true; t = Thread.CurrentThread; t.Name = Guid.NewGuid().ToString(); - this.RunEx(); + RunEx(); }).Start(); - if (t != null) - { - _log.Trace($"HEALTH THREAD: {t.Name}"); - this._threads.Add(t); - } + if (t == null) return; + _log.Trace($"HEALTH THREAD: {t.Name}"); + this._threads.Add(t); } catch (Exception exc) { @@ -59,18 +57,16 @@ public void Run() } } - public void Shutdown() + private void Shutdown() { - if (this._threads != null) + if (this._threads == null) return; + foreach (var thread in this._threads) { - foreach (var thread in this._threads) - { - thread.Abort(null); - } + thread.Interrupt(); } } - private void RunEx() + private static void RunEx() { var c = new ConfigHealth(ApplicationDetails.ConfigurationFiles.Health); var config = c.Load(); @@ -97,23 +93,22 @@ private void RunEx() _log.Debug(e); } } + // ReSharper disable once FunctionNeverReturns } private void OnChanged(object source, FileSystemEventArgs e) { - // filewatcher throws two events, we only need 1 - DateTime lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime > _lastRead.AddSeconds(1)) - { - _lastRead = lastWriteTime; - _log.Trace("File: " + e.FullPath + " " + e.ChangeType); - _log.Trace($"Reloading {System.Reflection.MethodBase.GetCurrentMethod().DeclaringType}"); - - // now terminate existing tasks and rerun - this.Shutdown(); - StartupTasks.CleanupProcesses(); - this.RunEx(); - } + // file watcher throws two events, we only need 1 + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime <= _lastRead.AddSeconds(1)) return; + _lastRead = lastWriteTime; + _log.Trace("File: " + e.FullPath + " " + e.ChangeType); + _log.Trace($"Reloading {System.Reflection.MethodBase.GetCurrentMethod()?.DeclaringType}"); + + // now terminate existing tasks and rerun + this.Shutdown(); + StartupTasks.CleanupProcesses(); + RunEx(); } } } diff --git a/src/ghosts.client.linux/Health/HealthRecord.cs b/src/ghosts.client.linux/Health/HealthRecord.cs index d92a316c..3613ef3c 100644 --- a/src/ghosts.client.linux/Health/HealthRecord.cs +++ b/src/ghosts.client.linux/Health/HealthRecord.cs @@ -1,7 +1,6 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.IO; using System.Net; using Ghosts.Domain; @@ -26,15 +25,15 @@ public static ResultHealth Check(ConfigHealth config) try { - using (var response = (HttpWebResponse)request.GetResponse()) + using var response = (HttpWebResponse)request.GetResponse(); + using var stream = response.GetResponseStream(); + if (stream != null) { - using (var stream = response.GetResponseStream()) - { - using (var reader = new StreamReader(stream)) - { - //html = reader.ReadToEnd(); //if we wanted to read html - } - } + // ignore + // using (var reader = new StreamReader(stream)) + // { + // //html = reader.ReadToEnd(); //if we wanted to read html + // } } } catch (WebException e) @@ -49,7 +48,7 @@ public static ResultHealth Check(ConfigHealth config) watch.Stop(); r.ExecutionTime = watch.ElapsedMilliseconds; - r.Internet = (r.Errors.Count == 0); + r.Internet = r.Errors.Count == 0; r.Stats = MachineHealth.Run(); } diff --git a/src/ghosts.client.linux/Health/MachineHealth.cs b/src/ghosts.client.linux/Health/MachineHealth.cs index 89dde2f8..849ae355 100644 --- a/src/ghosts.client.linux/Health/MachineHealth.cs +++ b/src/ghosts.client.linux/Health/MachineHealth.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; using System.IO; +using System.Linq; using Ghosts.Domain; using Ghosts.Domain.Code; using NLog; @@ -31,30 +32,24 @@ public static ResultHealth.MachineStats Run() return stats; } - public static float GetMemory() + private static float GetMemory() { - float totalMemory = 0; - foreach (var process in Process.GetProcesses()) - { - totalMemory += process.PrivateMemorySize64; - } - - return totalMemory; + return Process.GetProcesses().Aggregate(0, (current, process) => current + process.PrivateMemorySize64); } - public static float GetCpu() + private static float GetCpu() { var proc = Process.GetCurrentProcess(); var cpu = proc.TotalProcessorTime; - foreach (var process in Process.GetProcesses()) - { - //Console.WriteLine("Proc {0,30} CPU {1,-20:n} msec", process.ProcessName, cpu.TotalMilliseconds); - } + // foreach (var process in Process.GetProcesses()) + // { + // //Console.WriteLine("Proc {0,30} CPU {1,-20:n} m sec", process.ProcessName, cpu.TotalMilliseconds); + // } return cpu.Ticks; } - public static float GetDiskSpace() + private static float GetDiskSpace() { foreach (var drive in DriveInfo.GetDrives()) { diff --git a/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs b/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs index a4cdcc11..8685f1fa 100755 --- a/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs +++ b/src/ghosts.client.linux/Infrastructure/Browser/ExtendedConfiguration.cs @@ -1,6 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using Newtonsoft.Json; +// ReSharper disable UnusedMember.Global namespace ghosts.client.linux.Infrastructure.Browser { @@ -20,13 +21,10 @@ public static ExtendedConfiguration Load(object o) { var commandArg = o.ToString(); var result = new ExtendedConfiguration(); - if (commandArg.StartsWith("{")) - { - result = JsonConvert.DeserializeObject(commandArg); - return result; - } - + if (commandArg == null || !commandArg.StartsWith("{")) return result; + result = JsonConvert.DeserializeObject(commandArg); return result; + } } } diff --git a/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs b/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs index 6a0f7d36..ec33256a 100755 --- a/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs +++ b/src/ghosts.client.linux/Infrastructure/Browser/RequestConfiguration.cs @@ -20,13 +20,16 @@ public string GetHost() /// /// For categorizing browsing, e.g. I want to simulate someone shopping for shoes on x, y and z sites /// + // ReSharper disable once UnusedMember.Global public string Category { get; set; } /// /// GET, POST, PUT, DELETE /// public string Method { get; set; } + // ReSharper disable once UnusedMember.Global public IDictionary Headers { get; set; } public IDictionary FormValues { get; set; } + // ReSharper disable once UnusedMember.Global public string Body { get; set; } public override string ToString() @@ -38,7 +41,7 @@ public static RequestConfiguration Load(object o) { var commandArg = o.ToString(); var result = new RequestConfiguration(); - if (commandArg.StartsWith("{")) + if (commandArg != null && commandArg.StartsWith("{")) { result = JsonConvert.DeserializeObject(commandArg); return result; diff --git a/src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs b/src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs deleted file mode 100644 index d82ed28f..00000000 --- a/src/ghosts.client.linux/Infrastructure/ClientConfigurationResolver.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Ghosts.Domain.Code; - -namespace ghosts.client.linux.Infrastructure -{ - public static class ClientConfigurationResolver - { - public static string Dictionary => ApplicationDetails.ConfigurationFiles.Dictionary(Program.Configuration.Content.Dictionary); - public static string EmailContent => ApplicationDetails.ConfigurationFiles.EmailContent(Program.Configuration.Content.EmailContent); - public static string EmailReply => ApplicationDetails.ConfigurationFiles.EmailReply(Program.Configuration.Content.EmailReply); - public static string EmailDomain => ApplicationDetails.ConfigurationFiles.EmailDomain(Program.Configuration.Content.EmailDomain); - public static string EmailOutside => ApplicationDetails.ConfigurationFiles.EmailOutside(Program.Configuration.Content.EmailOutside); - public static string FileNames => ApplicationDetails.ConfigurationFiles.FileNames(Program.Configuration.Content.FileNames); - } -} \ No newline at end of file diff --git a/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs b/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs index 2cde613e..e5484c4c 100644 --- a/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs +++ b/src/ghosts.client.linux/Infrastructure/CommandLineFlagManager.cs @@ -1,7 +1,8 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + using System; -using System.Drawing; +using System.Collections.Generic; using System.IO; -using System.Linq; using System.Reflection; using System.Runtime.Versioning; using CommandLine; @@ -14,7 +15,7 @@ namespace ghosts.client.linux.Infrastructure { internal static class CommandLineFlagManager { - internal static bool Parse(string[] args) + internal static bool Parse(IEnumerable args) { Console.WriteLine(ApplicationDetails.Header); @@ -32,8 +33,6 @@ internal static bool Parse(string[] args) .ParseArguments(args) .WithParsed(o => options = o); - Program.OptionFlags = options; - // start handling flags that result in program exit if (options.Help) { diff --git a/src/ghosts.client.linux/Infrastructure/StartupTasks.cs b/src/ghosts.client.linux/Infrastructure/StartupTasks.cs index d1cc234c..4c77c627 100644 --- a/src/ghosts.client.linux/Infrastructure/StartupTasks.cs +++ b/src/ghosts.client.linux/Infrastructure/StartupTasks.cs @@ -41,13 +41,14 @@ public static void CleanupProcesses() Thread.CurrentThread.IsBackground = true; foreach (var process in Process.GetProcessesByName(cleanupItem)) { - if (process.Id != ghosts.Id) //don't kill thyself + if (process.Id == ghosts.Id) continue; + try { - try - { - process.Kill(); - } - catch { } + process.Kill(); + } + catch + { + // ignored } } }).Start(); @@ -70,31 +71,7 @@ public static void CleanupProcesses() /// public static void SetStartup() { - return; - /* - try - { - throw new NotImplementedException(); - - [Unit] - Description=GHOSTS NPC Orchestration - After=multi-user.target - - [Service] - Type=simple - ExecStart=/usr/bin/ghosts - - [Install] - WantedBy=multi-user.target - - - //_log.Trace("Startup set successfully"); - } - catch (Exception e) - { - //_log.Debug($"Set startup: {e}"); - } - */ + // ignored } } } diff --git a/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs b/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs index 2337e0a0..9ecffef7 100644 --- a/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs +++ b/src/ghosts.client.linux/Infrastructure/WebClientHeaders.cs @@ -1,7 +1,7 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System.Net; -using ghosts.client.linux.Comms; +using ghosts.client.linux.Communications; using Ghosts.Domain; using Ghosts.Domain.Code; diff --git a/src/ghosts.client.linux/Program.cs b/src/ghosts.client.linux/Program.cs index 2865bc08..a0a3afea 100755 --- a/src/ghosts.client.linux/Program.cs +++ b/src/ghosts.client.linux/Program.cs @@ -1,29 +1,32 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; +using System.Collections.Generic; using System.Drawing; using System.IO; using System.Net; using System.Reflection; using System.Threading; +using ghosts.client.linux.Comms; +using ghosts.client.linux.Communications; using ghosts.client.linux.Infrastructure; using ghosts.client.linux.timelineManager; using Ghosts.Domain.Code; +using Ghosts.Domain.Models; using NLog; namespace ghosts.client.linux { - class Program + internal static class Program { - - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - internal static ClientConfiguration Configuration { get; set; } - internal static Options OptionFlags; + internal static ClientConfiguration Configuration { get; private set; } internal static bool IsDebug; - private static ListenerManager _listenerManager { get; set; } + internal static List ThreadJobs { get; private set; } + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - static void Main(string[] args) + private static void Main(string[] args) { + ThreadJobs = new List(); ClientConfigurationLoader.UpdateConfigurationWithEnvVars(); try @@ -38,7 +41,7 @@ static void Main(string[] args) } } - private static void Run(string[] args) + private static void Run(IEnumerable args) { // ignore all certs ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; @@ -56,7 +59,7 @@ private static void Run(string[] args) //load configuration try { - Program.Configuration = ClientConfigurationLoader.Config; + Configuration = ClientConfigurationLoader.Config; } catch (Exception e) { @@ -72,27 +75,53 @@ private static void Run(string[] args) StartupTasks.SetStartup(); - _listenerManager = new ListenerManager(); + ListenerManager.Run(); //check id - _log.Trace(Comms.CheckId.Id); + _log.Trace(CheckId.Id); //connect to command server for updates and sending logs - Comms.Updates.Run(); + Updates.Run(); - //linux clients do not perform local survey + //local survey gathers information such as drives, accounts, logs, etc. + if (Configuration.Survey.IsEnabled) + { + _log.Trace("Survey enabled, initalizing..."); + try + { + Survey.SurveyManager.Run(); + } + catch (Exception exc) + { + _log.Error(exc); + } + } + else + { + _log.Trace("Survey disabled, continuing."); + } if (Configuration.HealthIsEnabled) { + _log.Trace("Health checks enabled, initalizing..."); var h = new Health.Check(); h.Run(); } + else + { + _log.Trace("Health checks disabled, continuing."); + } if (Configuration.HandlersIsEnabled) { + _log.Trace("Handlers enabled, initalizing..."); var o = new Orchestrator(); o.Run(); } + else + { + _log.Trace("Handling disabed, continuing."); + } new ManualResetEvent(false).WaitOne(); } diff --git a/src/ghosts.client.linux/Survey/SurveyManager.cs b/src/ghosts.client.linux/Survey/SurveyManager.cs new file mode 100644 index 00000000..26a83330 --- /dev/null +++ b/src/ghosts.client.linux/Survey/SurveyManager.cs @@ -0,0 +1,392 @@ +// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net.NetworkInformation; +using System.Text.RegularExpressions; +using System.Threading; +using ghosts.client.linux.Comms; +using ghosts.client.linux.Communications; +using Ghosts.Domain.Code; +using Newtonsoft.Json; +using NLog; + +namespace ghosts.client.linux.Survey +{ + public static class SurveyManager + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public static void Run() + { + try + { + new Thread(() => + { + Thread.CurrentThread.IsBackground = true; + RunEx(); + + }).Start(); + } + catch (Exception e) + { + _log.Trace(e); + } + } + + private static void RunEx() + { + while (true) + { + //has file already been generated + if (Program.Configuration.Survey.Frequency.Equals("once", + StringComparison.InvariantCultureIgnoreCase) && + File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) + break; + + + _log.Trace("Running new survey..."); + try + { + var s = new SurveyResult + { + Survey = + { + Created = DateTime.UtcNow + } + }; + + if (Guid.TryParse(CheckId.Id, out var g)) + s.Survey.MachineId = g; + + s.LoadAll(); + + var f = new FileInfo(ApplicationDetails.InstanceFiles.SurveyResults); + if (f.Directory is { Exists: false }) + { + Directory.CreateDirectory(f.DirectoryName); + } + + try + { + if (File.Exists(ApplicationDetails.InstanceFiles.SurveyResults)) + File.Delete(ApplicationDetails.InstanceFiles.SurveyResults); + } + catch (Exception e) + { + _log.Trace(e); + } + + var formatting = Formatting.Indented; + if (Program.Configuration.Survey.OutputFormat.Equals("none", + StringComparison.InvariantCultureIgnoreCase)) + formatting = Formatting.None; + + using (var file = File.CreateText(ApplicationDetails.InstanceFiles.SurveyResults)) + { + var serializer = new JsonSerializer + { + Formatting = formatting, + NullValueHandling = NullValueHandling.Ignore + }; + serializer.Serialize(file, s.Survey); + } + Updates.PostSurvey(); + } + catch (Exception e) + { + _log.Trace(e); + } + + Thread.Sleep(Program.Configuration.Survey.CycleSleepMinutes * 60000); + } + } + } + + public class SurveyResult + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + + public Ghosts.Domain.Messages.MesssagesForServer.Survey Survey { get; set; } + + public SurveyResult() + { + this.Survey = new Ghosts.Domain.Messages.MesssagesForServer.Survey + { + Uptime = GetUptime() + }; + } + + public void LoadAll() + { + var random = new Random(); + + this.Survey.Ports = GetNetStatPorts(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.Interfaces = GetInterfaces(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.LocalUsers = GetLocalAccounts(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.Drives = this.GetDriveInfo(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.Processes = this.GetProcesses(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + this.Survey.EventLogs = GetEventLogs(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 900000)); + + foreach (var item in this.Survey.EventLogs) + { + item.Entries = new List(); + if (!Program.IsDebug) + Thread.Sleep(random.Next(500, 5000)); + foreach (var o in this.GetEventLogEntries(item.Name)) + item.Entries.Add(o); + } + } + + public static TimeSpan GetUptime() + { + try + { + return TimeSpan.FromMilliseconds(Stopwatch.GetTimestamp()); + } + catch (Exception e) + { + _log.Trace(e); + } + return new TimeSpan(); + } + + public static List GetNetStatPorts() + { + var ports = new List(); + + try + { + using var p = new Process(); + var ps = new ProcessStartInfo + { + Arguments = "-a -n -o", + FileName = "netstat.exe", + UseShellExecute = false, + WindowStyle = ProcessWindowStyle.Hidden, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + p.StartInfo = ps; + p.Start(); + + var stdOutput = p.StandardOutput; + var stdError = p.StandardError; + + var content = stdOutput.ReadToEnd() + stdError.ReadToEnd(); + var exitStatus = p.ExitCode.ToString(); + + if (exitStatus != "0") + { + // Command Errored. Handle Here If Need Be + } + + //Get The Rows + var rows = Regex.Split(content, "\r\n"); + foreach (var row in rows) + { + //Split it + var tokens = Regex.Split(row, "\\s+"); + if (tokens.Length > 4 && (tokens[1].Equals("UDP") || tokens[1].Equals("TCP"))) + { + var localAddress = Regex.Replace(tokens[2], @"\[(.*?)\]", "1.1.1.1"); + var foreignAddress = Regex.Replace(tokens[3], @"\[(.*?)\]", "1.1.1.1"); + ports.Add(new Ghosts.Domain.Messages.MesssagesForServer.Survey.Port + { + LocalAddress = localAddress.Split(':')[0], + LocalPort = localAddress.Split(':')[1], + ForeignAddress = foreignAddress.Split(':')[0], + ForeignPort = foreignAddress.Split(':')[1], + State = tokens[1] == "UDP" ? null : tokens[4], + PID = tokens[1] == "UDP" ? Convert.ToInt16(tokens[4]) : Convert.ToInt16(tokens[5]), + Protocol = localAddress.Contains("1.1.1.1") ? String.Format("{0}v6", tokens[1]) : String.Format("{0}v4", tokens[1]), + Process = tokens[1] == "UDP" ? LookupProcess(Convert.ToInt16(tokens[4])) : LookupProcess(Convert.ToInt16(tokens[5])) + }); + } + } + } + catch (Exception e) + { + _log.Trace(e); + } + + return ports; + } + + public static string LookupProcess(int pid) + { + string procName; + try { procName = Process.GetProcessById(pid).ProcessName; } + catch (Exception) { procName = "-"; } + return procName; + } + + public List GetInterfaces() + { + var results = new List(); + + try + { + var adapters = NetworkInterface.GetAllNetworkInterfaces(); + var adapterCount = 0; + foreach (var adapter in adapters) + { + adapterCount += 1; + var iFace = new Ghosts.Domain.Messages.MesssagesForServer.Survey.Interface(); + iFace.Name = adapter.Name; + iFace.Id = adapterCount; + + var physicalAddress = adapter.GetPhysicalAddress(); + var ipProperties = adapter.GetIPProperties(); + if (ipProperties.UnicastAddresses != null) + { + foreach (var address in ipProperties.UnicastAddresses) + { + var bind = new Ghosts.Domain.Messages.MesssagesForServer.Survey.Interface.InterfaceBinding(); + bind.Type = adapter.NetworkInterfaceType.ToString(); + bind.InternetAddress = address.Address.ToString(); + bind.PhysicalAddress = physicalAddress.ToString(); + iFace.Bindings.Add(bind); + } + } + + results.Add(iFace); + } + } + catch (Exception e) + { + _log.Trace(e); + } + + return results; + } + + public List GetLocalAccounts() + { + var users = new List(); + try + { + + } + catch (Exception e) + { + _log.Trace(e); + } + return users; + } + + public List GetDriveInfo() + { + var results = new List(); + try + { + var allDrives = System.IO.DriveInfo.GetDrives(); + foreach (var drive in allDrives) + { + var result = new Ghosts.Domain.Messages.MesssagesForServer.Survey.DriveInfo(); + result.AvailableFreeSpace = drive.AvailableFreeSpace; + result.DriveFormat = drive.DriveFormat; + result.DriveType = drive.DriveType.ToString(); + result.IsReady = drive.IsReady; + result.Name = drive.Name; + result.RootDirectory = drive.RootDirectory.ToString(); + result.TotalFreeSpace = drive.TotalFreeSpace; + result.TotalSize = drive.TotalSize; + result.VolumeLabel = drive.VolumeLabel; + results.Add(result); + } + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + + public List GetProcesses() + { + var results = new List(); + try + { + foreach (var item in Process.GetProcesses()) + { + var result = new Ghosts.Domain.Messages.MesssagesForServer.Survey.LocalProcess(); + + if (!string.IsNullOrEmpty(item.MainWindowTitle)) + result.MainWindowTitle = item.MainWindowTitle; + if (!string.IsNullOrEmpty(item.ProcessName)) + result.ProcessName = item.ProcessName; + try { result.StartTime = item.StartTime; } + catch + { + // ignore + } + if (!string.IsNullOrEmpty(item.StartInfo.FileName)) + result.FileName = item.StartInfo.FileName; + if (!string.IsNullOrEmpty(item.StartInfo.UserName)) + result.Owner = item.StartInfo.UserName; + if (string.IsNullOrEmpty(result.Owner)) + { + // how to get owners on linux? + } + if (!results.Exists(x => x.ProcessName == item.ProcessName)) + results.Add(result); + } + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + + public List GetEventLogs() + { + var results = new List(); + try + { + + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + + public List GetEventLogEntries(string logName) + { + var results = new List(); + try + { + + } + catch (Exception e) + { + _log.Trace(e); + } + return results; + } + } +} \ No newline at end of file diff --git a/src/ghosts.client.linux/TimelineManager/Listener.cs b/src/ghosts.client.linux/TimelineManager/Listener.cs index a279804c..2129bb12 100644 --- a/src/ghosts.client.linux/TimelineManager/Listener.cs +++ b/src/ghosts.client.linux/TimelineManager/Listener.cs @@ -2,22 +2,24 @@ using System; using System.IO; +using System.Linq; using System.Threading; using Ghosts.Domain; using Ghosts.Domain.Code; using Newtonsoft.Json; using NLog; using SimpleTCP; +// ReSharper disable ObjectCreationAsStatement namespace ghosts.client.linux.timelineManager { - public class ListenerManager + public static class ListenerManager { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private string In = ApplicationDetails.InstanceDirectories.TimelineIn; - private string Out = ApplicationDetails.InstanceDirectories.TimelineOut; + private static readonly string In = ApplicationDetails.InstanceDirectories.TimelineIn; + private static readonly string Out = ApplicationDetails.InstanceDirectories.TimelineOut; - public ListenerManager() + public static void Run() { try { @@ -59,6 +61,13 @@ public ListenerManager() }; t.Start(); + t = new Thread(() => { new InitialDirectoryListener(); }) + { + IsBackground = true, + Name = "ghosts-initialdirectorylistener" + }; + t.Start(); + EnsureWatch(); } else @@ -72,39 +81,88 @@ public ListenerManager() } } - private void EnsureWatch() + private static void EnsureWatch() { File.WriteAllText(In + "init.json", JsonConvert.SerializeObject(new Timeline(), Formatting.None)); } } + public class InitialDirectoryListener + { + private static readonly Logger _log = LogManager.GetCurrentClassLogger(); + private static DateTime _lastRead = DateTime.Now; + public InitialDirectoryListener() + { + var directoryName = TimelineBuilder.TimelineFilePath().DirectoryName; + if (directoryName == null) + { + throw new Exception("Timeline builder path cannot be determined"); + } + + var timelineWatcher = new FileSystemWatcher(directoryName); + timelineWatcher.Filter = Path.GetFileName(TimelineBuilder.TimelineFilePath().Name); + _log.Trace($"watching {timelineWatcher.Path}"); + timelineWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size | NotifyFilters.CreationTime | + NotifyFilters.LastWrite; + timelineWatcher.Changed += InitialOnChanged; + timelineWatcher.EnableRaisingEvents = true; + + + new ManualResetEvent(false).WaitOne(); + } + + private static void InitialOnChanged(object source, FileSystemEventArgs e) + { + // file watcher throws two events, we only need 1 + var lastWriteTime = File.GetLastWriteTime(e.FullPath); + if (lastWriteTime <= _lastRead.AddSeconds(1)) return; + + _lastRead = lastWriteTime; + _log.Trace("File: " + e.FullPath + " " + e.ChangeType); + + var method = string.Empty; + if (System.Reflection.MethodBase.GetCurrentMethod() != null) + { + var declaringType = System.Reflection.MethodBase.GetCurrentMethod()?.DeclaringType; + if (declaringType != null) + method = declaringType.ToString(); + } + _log.Trace($"Reloading {method}..."); + + // now terminate existing tasks and rerun + var o = new Orchestrator(); + Orchestrator.Stop(); + o.Run(); + } + } + /// - /// Watches a directory [ghosts install]\instance\timeline for dropped files, and processes them immediately + /// Watches a directory [ghosts install] \ instance \ timeline for dropped files, and processes them immediately /// public class DirectoryListener { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private string _in = ApplicationDetails.InstanceDirectories.TimelineIn; - private string _out = ApplicationDetails.InstanceDirectories.TimelineOut; + private readonly string _in = ApplicationDetails.InstanceDirectories.TimelineIn; + private readonly string _out = ApplicationDetails.InstanceDirectories.TimelineOut; private string _currentlyProcessing = string.Empty; - + public DirectoryListener() { var watcher = new FileSystemWatcher { Path = _in, NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.FileName, - Filter = "*.json" + Filter = "*.*" }; watcher.Changed += OnChanged; watcher.Created += OnChanged; watcher.EnableRaisingEvents = true; - Console.ReadLine(); - } - + new ManualResetEvent(false).WaitOne(); + } + private void OnChanged(object source, FileSystemEventArgs e) { - // filewatcher throws multiple events, we only need 1 + // file watcher throws multiple events, we only need 1 if (!string.IsNullOrEmpty(_currentlyProcessing) && _currentlyProcessing == e.FullPath) return; _currentlyProcessing = e.FullPath; @@ -112,37 +170,63 @@ private void OnChanged(object source, FileSystemEventArgs e) if (!File.Exists(e.FullPath)) return; - - try - { - var raw = File.ReadAllText(e.FullPath); - var timeline = JsonConvert.DeserializeObject(raw); - - foreach (var timelineHandler in timeline.TimeLineHandlers) + if (e.FullPath.EndsWith(".json")) + { + try { - _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); - - foreach (var timelineEvent in timelineHandler.TimeLineEvents) + var timeline = TimelineBuilder.GetLocalTimeline(e.FullPath); + foreach (var timelineHandler in timeline.TimeLineHandlers) { - if (string.IsNullOrEmpty(timelineEvent.TrackableId)) + _log.Trace($"DirectoryListener command found: {timelineHandler.HandlerType}"); + + foreach (var timelineEvent in timelineHandler.TimeLineEvents.Where(timelineEvent => string.IsNullOrEmpty(timelineEvent.TrackableId))) { timelineEvent.TrackableId = Guid.NewGuid().ToString(); } - } - var orchestrator = new Orchestrator(); - orchestrator.RunCommand(timelineHandler); + Orchestrator.RunCommand(timeline, timelineHandler); + } } + catch (Exception exc) + { + _log.Debug(exc); + } + } + else if (e.FullPath.EndsWith(".cs")) + { + try + { + var commands = File.ReadAllText(e.FullPath).Split(Convert.ToChar("\n")).ToList(); + if (commands.Count > 0) + { + var constructedTimelineHandler = TimelineTranslator.FromBrowserUnitTests(commands); + + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(constructedTimelineHandler); + Orchestrator.RunCommand(t, constructedTimelineHandler); + } + } + catch (Exception exc) + { + _log.Debug(exc); + } + } + try + { var outfile = e.FullPath.Replace(_in, _out); outfile = outfile.Replace(e.Name, $"{DateTime.Now.ToString("G").Replace("/", "-").Replace(" ", "").Replace(":", "")}-{e.Name}"); File.Move(e.FullPath, outfile); } - catch (Exception exc) + catch (Exception exception) { - _log.Debug(exc); + _log.Debug(exception); } _currentlyProcessing = string.Empty; @@ -169,12 +253,6 @@ public Listener() message.ReplyLine($"{obj}{Environment.NewLine}"); Console.WriteLine(obj); }; - -// while (true) -// { -// Console.WriteLine("..."); -// Thread.Sleep(10000); -// } } catch (Exception e) { @@ -182,12 +260,12 @@ public Listener() } } - private string Handle(Message message) + private static string Handle(Message message) { var command = message.MessageString; var index = command.LastIndexOf("}", StringComparison.InvariantCultureIgnoreCase); if (index > 0) - command = command.Substring(0, index + 1); + command = command[..(index + 1)]; _log.Trace($"Received from {message.TcpClient.Client.RemoteEndPoint}: {command}"); @@ -195,14 +273,21 @@ private string Handle(Message message) { var timelineHandler = JsonConvert.DeserializeObject(command); - foreach (var evs in timelineHandler.TimeLineEvents) - if (string.IsNullOrEmpty(evs.TrackableId)) - evs.TrackableId = Guid.NewGuid().ToString(); + foreach (var evs in timelineHandler.TimeLineEvents.Where(evs => string.IsNullOrEmpty(evs.TrackableId))) + { + evs.TrackableId = Guid.NewGuid().ToString(); + } _log.Trace($"Command found: {timelineHandler.HandlerType}"); - var o = new Orchestrator(); - o.RunCommand(timelineHandler); + var t = new Timeline + { + Id = Guid.NewGuid(), + Status = Timeline.TimelineStatus.Run + }; + t.TimeLineHandlers.Add(timelineHandler); + + Orchestrator.RunCommand(t, timelineHandler); var obj = JsonConvert.SerializeObject(timelineHandler); diff --git a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs index 01a12e6d..8634aa37 100644 --- a/src/ghosts.client.linux/TimelineManager/Orchestrator.cs +++ b/src/ghosts.client.linux/TimelineManager/Orchestrator.cs @@ -1,11 +1,12 @@ // Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. using System; -using System.Collections.Generic; -using System.IO; +using System.Linq; using System.Threading; using ghosts.client.linux.handlers; using Ghosts.Domain; +using Ghosts.Domain.Code; +using Ghosts.Domain.Models; using NLog; namespace ghosts.client.linux.timelineManager @@ -16,9 +17,6 @@ namespace ghosts.client.linux.timelineManager public class Orchestrator { private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - private static DateTime _lastRead = DateTime.MinValue; - private List _threads { get; set; } - private List _threadJobs { get; set; } private Thread MonitorThread { get; set; } public void Run() @@ -27,30 +25,18 @@ public void Run() { var timeline = TimelineBuilder.GetLocalTimeline(); - // now watch that file for changes - var timelineWatcher = new FileSystemWatcher(TimelineBuilder.TimelineFilePath().DirectoryName); - timelineWatcher.Filter = Path.GetFileName(TimelineBuilder.TimelineFilePath().Name); - _log.Trace($"watching {timelineWatcher.Path}"); - timelineWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.FileName | NotifyFilters.Size | NotifyFilters.CreationTime | NotifyFilters.LastWrite; - timelineWatcher.EnableRaisingEvents = true; - timelineWatcher.Changed += new FileSystemEventHandler(OnChanged); - - this._threadJobs = new List(); - //load into an managing object //which passes the timeline commands to handlers //and creates a thread to execute instructions over that timeline if (timeline.Status == Timeline.TimelineStatus.Run) { - this.RunEx(timeline); + RunEx(timeline); } else { - if (this.MonitorThread != null) - { - this.MonitorThread.Abort(); - this.MonitorThread = null; - } + if (this.MonitorThread == null) return; + this.MonitorThread.Interrupt(); + this.MonitorThread = null; } } catch (Exception exc) @@ -59,149 +45,129 @@ public void Run() } } - public void Shutdown() + public static void StopTimeline(Guid timelineId) { - foreach (var thread in this._threads) + foreach (var threadJob in Program.ThreadJobs.Where(x=>x.TimelineId == timelineId)) { - thread.Abort(null); + try + { + threadJob.Thread.Interrupt(); + } + catch (Exception e) + { + _log.Debug(e); + } + + try + { + threadJob.Thread.Join(); + } + catch (Exception e) + { + _log.Debug(e); + } } } - private void RunEx(Timeline timeline) + public static void Stop() { - this._threads = new List(); - - this.WhatsInstalled(); - - foreach (var handler in timeline.TimeLineHandlers) + foreach (var threadJob in Program.ThreadJobs) { - ThreadLaunch(handler); - } + try + { + threadJob.Thread.Interrupt(); + } + catch (Exception e) + { + _log.Debug(e); + } - this.MonitorThread = new Thread(this.ThreadMonitor); - this.MonitorThread.IsBackground = true; - this.MonitorThread.Start(); + try + { + threadJob.Thread.Interrupt(); + } + catch (Exception e) + { + _log.Debug(e); + } + } } - public void RunCommand(TimelineHandler handler) + private static void RunEx(Timeline timeline) { - this.WhatsInstalled(); - ThreadLaunch(handler); + foreach (var handler in timeline.TimeLineHandlers) + { + ThreadLaunch(timeline, handler); + } } - private void WhatsInstalled() + public static void RunCommand(Timeline timeline, TimelineHandler handler) { - //TODO: check that used applications exist + ThreadLaunch(timeline, handler); } - private void ThreadLaunch(TimelineHandler handler) + private static void ThreadLaunch(Timeline timeline, TimelineHandler handler) { try { _log.Trace($"Attempting new thread for: {handler.HandlerType}"); Thread t = null; - var threadJob = new ThreadJob(); - threadJob.Id = Guid.NewGuid().ToString(); - threadJob.Handler = handler; - + // ReSharper disable once NotAccessedVariable + object o; + // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault switch (handler.HandlerType) { case HandlerType.NpcSystem: - //var npc = new NpcSystem(handler); - //break; + // ReSharper disable once RedundantAssignment + o = new NpcSystem(timeline, handler); + break; case HandlerType.Command: t = new Thread(() => { - var bash = new Bash(handler); - + o = new Bash(handler); }); - t.IsBackground = true; - t.Name = threadJob.Id; - t.Start(); - - //threadJob.ProcessName = ProcessManager.ProcessNames.Command; break; case HandlerType.Curl: t = new Thread(() => { - var curl = new Curl(handler); - + o = new Curl(handler); + }); + break; + case HandlerType.BrowserChrome: + t = new Thread(() => + { + o = new BrowserChrome(handler); }); - t.IsBackground = true; - t.Name = threadJob.Id; - t.Start(); - - //threadJob.ProcessName = ProcessManager.ProcessNames.Command; break; case HandlerType.BrowserFirefox: - BrowserFirefox o = new BrowserFirefox(handler); - // t = new Thread(() => - // { - // BrowserFirefox o = new BrowserFirefox(handler); - // }) - // { - // IsBackground = true, - // Name = threadJob.Id - // }; - // t.Start(); - + t = new Thread(() => + { + o = new BrowserFirefox(handler); + }); break; + default: + throw new ArgumentOutOfRangeException(); } - if (threadJob.ProcessName != null) + if (t == null) { - this._threadJobs.Add(threadJob); + _log.Debug($"HandlerType {handler.HandlerType} not supported on this platform"); + return; } - - if (t != null) + + t.IsBackground = true; + t.Start(); + Program.ThreadJobs.Add(new ThreadJob { - this._threads.Add(t); - } + TimelineId = timeline.Id, + Thread = t + }); } catch (Exception e) { _log.Error(e); } } - - private void ThreadMonitor() - { - //this should be the original list only - var jobs = this._threadJobs.ToArray(); - while (true) - { - Thread.Sleep(30000); - //first, get all jobs and if not running, run a new one - foreach (var job in jobs) - { - //TODO - } - } - } - - private void OnChanged(object source, FileSystemEventArgs e) - { - // filewatcher throws two events, we only need 1 - DateTime lastWriteTime = File.GetLastWriteTime(e.FullPath); - if (lastWriteTime > _lastRead.AddSeconds(1)) - { - _lastRead = lastWriteTime; - _log.Trace("File: " + e.FullPath + " " + e.ChangeType); - _log.Trace($"Reloading {System.Reflection.MethodBase.GetCurrentMethod().DeclaringType}"); - - // now terminate existing tasks and rerun - this.Shutdown(); - //StartupTasks.CleanupProcesses(); - this.Run(); - } - } - } - - public class ThreadJob - { - public string Id { get; set; } - public TimelineHandler Handler { get; set; } - public string ProcessName { get; set; } } -} + } diff --git a/src/ghosts.client.linux/TimelineManager/TimelineBuilder.cs b/src/ghosts.client.linux/TimelineManager/TimelineBuilder.cs deleted file mode 100644 index f65d21e1..00000000 --- a/src/ghosts.client.linux/TimelineManager/TimelineBuilder.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms. - -using System.IO; -using Ghosts.Domain; -using Ghosts.Domain.Code; -using Newtonsoft.Json; -using NLog; - -namespace ghosts.client.linux.timelineManager -{ - /// - /// Helper class that loads timeline and watches it for future changes - /// - public class TimelineBuilder - { - private static readonly Logger _log = LogManager.GetCurrentClassLogger(); - - public static string TimelineFile = ApplicationDetails.ConfigurationFiles.Timeline; - - public static FileInfo TimelineFilePath() - { - return new FileInfo(TimelineFile); - } - - /// - /// Get from local disk - /// - /// The local timeline to be executed - public static Timeline GetLocalTimeline() - { - _log.Trace($"Loading timeline config {TimelineFile }"); - - var raw = File.ReadAllText(TimelineFile); - var timeline = JsonConvert.DeserializeObject(raw); - - _log.Debug("Timeline config loaded successfully"); - - return timeline; - } - - /// - /// Save to local disk - /// - /// Raw timeline string (to be converted to `Timeline` type) - public static void SetLocalTimeline(string timelineString) - { - var timelineObject = JsonConvert.DeserializeObject(timelineString); - SetLocalTimeline(timelineObject); - } - - /// - /// Save to local disk - /// - /// `Timeline` type - public static void SetLocalTimeline(Timeline timeline) - { - using (var file = File.CreateText(ApplicationDetails.ConfigurationFiles.Timeline)) - { - var serializer = new JsonSerializer(); - serializer.Formatting = Formatting.Indented; - serializer.Serialize(file, timeline); - } - } - } -} diff --git a/src/ghosts.client.linux/config/application.json b/src/ghosts.client.linux/config/application.json index a0aea0fc..69d942c7 100755 --- a/src/ghosts.client.linux/config/application.json +++ b/src/ghosts.client.linux/config/application.json @@ -1,24 +1,24 @@ { "IdEnabled": true, - "IdUrl": "http://ghosts-api:5000/api/clientid", + "IdUrl": "http://ghosts-api:52388/api/clientid", "ClientResults": { "IsEnabled": true, "IsSecure": false, - "PostUrl": "http://ghosts-api:5000/api/clientresults", + "PostUrl": "http://ghosts-api:52388/api/clientresults", "CycleSleep": 9000 }, "ClientUpdates": { "IsEnabled": true, - "PostUrl": "http://ghosts-api:5000/api/clientupdates", + "PostUrl": "http://ghosts-api:52388/api/clientupdates", "CycleSleep": 9000 }, "Survey": { "IsEnabled": true, "IsSecure": false, "Frequency": "once", - "CycleSleepMinutes": 720, + "CycleSleepMinutes": 5, "OutputFormat": "indent", - "PostUrl": "http://ghosts-api:5000/api/clientsurvey" + "PostUrl": "http://ghosts-api:52388/api/clientsurvey" }, "HealthIsEnabled": true, "HandlersIsEnabled": true, diff --git a/src/ghosts.client.linux/config/timeline.json b/src/ghosts.client.linux/config/timeline.json index 01a64cc3..84b06366 100755 --- a/src/ghosts.client.linux/config/timeline.json +++ b/src/ghosts.client.linux/config/timeline.json @@ -2,14 +2,14 @@ "Status": "Run", "TimeLineHandlers": [ { - "HandlerType": "BrowserFirefox", + "HandlerType": "BrowserChrome", "HandlerArgs": { "isheadless": "false", "blockimages": "true", "blockstyles": "true", "blockflash": "true", "blockscripts": "true", - "stickiness": 75, + "stickiness": 100, "stickiness-depth-min": 5, "stickiness-depth-max": 10000, "incognito": "true" diff --git a/src/ghosts.client.linux/dockerfile-alpine b/src/ghosts.client.linux/dockerfile-alpine index 1b01eea6..8223b552 100755 --- a/src/ghosts.client.linux/dockerfile-alpine +++ b/src/ghosts.client.linux/dockerfile-alpine @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:latest AS build +FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS dev # copy csproj and restore as distinct layers COPY ghosts.linux.sln ./app/ @@ -15,15 +15,7 @@ COPY Ghosts.Domain/ ./Ghosts.Domain/ WORKDIR /app/ghosts.client.linux/ RUN dotnet publish -c Release -o out -FROM mcr.microsoft.com/dotnet/core/runtime:3.1-alpine - -# Install ASP.NET Core -RUN aspnetcore_version=3.1.4 \ - && wget -O aspnetcore.tar.gz https://dotnetcli.azureedge.net/dotnet/aspnetcore/Runtime/$aspnetcore_version/aspnetcore-runtime-$aspnetcore_version-linux-musl-x64.tar.gz \ - && aspnetcore_sha512='f60e9226a5b399470479fd6fdebd03442b0440128be1090adcbe473dba46a3e7a57a9e59b4abff96214e0dd0b1123c67fe764b74c61de1cb35c8b8ac45767eb9' \ - && echo "$aspnetcore_sha512 aspnetcore.tar.gz" | sha512sum -c - \ - && tar -ozxf aspnetcore.tar.gz -C /usr/share/dotnet ./shared/Microsoft.AspNetCore.App \ - && rm aspnetcore.tar.gz +FROM mcr.microsoft.com/dotnet/aspnet:6.0-alpine AS prod RUN apk add curl && \ apk add nmap && \ @@ -38,7 +30,7 @@ RUN apk add curl && \ cd sqlmap/ # python sqlmap.py --version -COPY --from=build /app/ghosts.client.linux/out ./app +COPY --from=dev /app/ghosts.client.linux/out ./app WORKDIR /app/ ENV ASPNETCORE_URLS=http://+:5000 diff --git a/src/ghosts.client.linux/dockerfile-kali b/src/ghosts.client.linux/dockerfile-kali deleted file mode 100644 index 1ef930c2..00000000 --- a/src/ghosts.client.linux/dockerfile-kali +++ /dev/null @@ -1,35 +0,0 @@ -FROM booyaabes/kali-linux-full:latest - -# install dotnetcore -RUN sudo apt-get --assume-yes install -y gpg && \ - wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg && \ - sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ && \ - wget -q https://packages.microsoft.com/config/ubuntu/19.04/prod.list && \ - sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list && \ - sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg && \ - sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list && \ - apt-get update && \ - sudo apt-get --assume-yes install -y apt-transport-https && \ - sudo apt-get update && \ - sudo apt-get --assume-yes install aspnetcore-runtime-3.1 - -WORKDIR /opt/ghosts -COPY bin/publish . -ENV ASPNETCORE_URLS=http://+:5000 -EXPOSE 5000/tcp - -# if we wanted to do other commands at startup in the future -# CMD [ "/usr/local/bin/start.sh" ] - -CMD ["dotnet", "./ghosts.client.linux.dll"] - - - -# dotnet publish -c Release -o bin/publish -# docker build . -f dockerfile-kali -t ghosts/staypuft - -# docker swarm init -# docker stack deploy ghosts-staypuft --compose-file docker-compose.yml -# Docker swarm leave --force - -# docker run -d -p 7000:5000 --name ghosts-staypuft ghosts/staypuft diff --git a/src/ghosts.client.linux/ghosts.client.linux.csproj b/src/ghosts.client.linux/ghosts.client.linux.csproj index ed61c89a..db5d5c8c 100755 --- a/src/ghosts.client.linux/ghosts.client.linux.csproj +++ b/src/ghosts.client.linux/ghosts.client.linux.csproj @@ -2,13 +2,12 @@ Exe - netcoreapp3.1 win7-x64;ubuntu.16.10-x64;osx.10.12-x64 - 3.0.0.0 - 3.0.0.0 - 3.0.0.0 - 3.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 + 6.0.0.0 false Dustin Updyke for Carnegie Mellon University @@ -17,6 +16,7 @@ GHOSTS NPC Orchestration Platform - please email ddupdyke@sei.cmu.edu with bugs/requests/other Carnegie Mellon University 2017 NU1701 + net6.0 diff --git a/src/ghosts.client.linux/nlog.config b/src/ghosts.client.linux/nlog.config index 1ecbc4f1..3cc176ab 100644 --- a/src/ghosts.client.linux/nlog.config +++ b/src/ghosts.client.linux/nlog.config @@ -8,7 +8,7 @@ - + diff --git a/src/ghosts.windows.sln b/src/ghosts.windows.sln index 96b49562..8b91216e 100755 --- a/src/ghosts.windows.sln +++ b/src/ghosts.windows.sln @@ -9,17 +9,27 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|x64.ActiveCfg = Debug|x64 + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Debug|x64.Build.0 = Debug|x64 {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|Any CPU.Build.0 = Release|Any CPU + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|x64.ActiveCfg = Release|x64 + {2667ECE7-34FA-456A-8877-BB9EEB40E9F7}.Release|x64.Build.0 = Release|x64 {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|x64.ActiveCfg = Debug|x64 + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Debug|x64.Build.0 = Debug|x64 {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|Any CPU.ActiveCfg = Release|Any CPU {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|Any CPU.Build.0 = Release|Any CPU + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|x64.ActiveCfg = Release|x64 + {9B185127-C1A8-479C-B48F-3BC51B97023D}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/ghosts.windows.sln.DotSettings b/src/ghosts.windows.sln.DotSettings new file mode 100644 index 00000000..cf1de47e --- /dev/null +++ b/src/ghosts.windows.sln.DotSettings @@ -0,0 +1,7 @@ + + True + True + True + True + True + True \ No newline at end of file