From eff75f471fd5a783f383ee9f2b231ca513a891ec Mon Sep 17 00:00:00 2001 From: Meindert Niemeijer Date: Mon, 10 Feb 2025 16:41:16 -0600 Subject: [PATCH] WIP added the ability to convert textile format text as used in Redmine markup to ASCIIDOC. Also support nested tables in ItemTemplates through the EmbedAsciidocTables function which converts standard ASCIIDOC tables to nested table format. --- .../Configuration/RedmineSLMSPlugin.toml | 7 + RoboClerk.Redmine/RedmineSLMSPlugin.cs | 22 +- RoboClerk.Redmine/TextileToASCIIDoc.cs | 221 ++++++++++++++++++ .../ItemTemplateSupport/ScriptingBridge.cs | 59 ++++- RoboClerk/ItemTemplates/Requirement.adoc | 2 +- .../PluginSupport/DataSourcePluginBase.cs | 36 ++- 6 files changed, 334 insertions(+), 13 deletions(-) create mode 100644 RoboClerk.Redmine/TextileToASCIIDoc.cs diff --git a/RoboClerk.Redmine/Configuration/RedmineSLMSPlugin.toml b/RoboClerk.Redmine/Configuration/RedmineSLMSPlugin.toml index 39b29ff..2e9cd50 100644 --- a/RoboClerk.Redmine/Configuration/RedmineSLMSPlugin.toml +++ b/RoboClerk.Redmine/Configuration/RedmineSLMSPlugin.toml @@ -17,6 +17,13 @@ Ignore = [ "Rejected" ] # variable to an empty string "". RedmineBaseURL = "http://localhost:3001/issues/" +# If so desired, the plugin can attempt to convert Textile formatting +# to ASCIIDOC. This will allow some of the markup applied in Redmine +# to be retained in the final output document. This conversion only +# covers the most commonly used Textile markup and is not intended to +# be complete. Set the following to true to attempt conversion. +ConvertTextile = true + # The following allows you to indicate the redmine trackers that map # to various entities in the RoboClerk software. Set the name to the # redmine ticket type. You can also indicate if the items are subject diff --git a/RoboClerk.Redmine/RedmineSLMSPlugin.cs b/RoboClerk.Redmine/RedmineSLMSPlugin.cs index 4385163..f79cbe9 100644 --- a/RoboClerk.Redmine/RedmineSLMSPlugin.cs +++ b/RoboClerk.Redmine/RedmineSLMSPlugin.cs @@ -16,6 +16,8 @@ public class RedmineSLMSPlugin : SLMSPluginBase private string apiEndpoint = string.Empty; private string apiKey = string.Empty; private string projectName = string.Empty; + private bool convertTextile = false; + private TextileToAsciiDocConverter converter = null; private List redmineVersionFields = new List(); private RestClient client = null; private List versions = null; @@ -40,7 +42,12 @@ public override void Initialize(IConfiguration configuration) apiKey = configuration.CommandLineOptionOrDefault("RedmineAPIKey", GetObjectForKey(config, "RedmineAPIKey", true)); projectName = configuration.CommandLineOptionOrDefault("RedmineProject", GetObjectForKey(config, "RedmineProject", true)); baseURL = configuration.CommandLineOptionOrDefault("RedmineBaseURL", GetObjectForKey(config, "RedmineBaseURL", false)); - if(config.ContainsKey("VersionCustomFields")) + convertTextile = configuration.CommandLineOptionOrDefault("ConvertTextile", GetObjectForKey(config, "ConvertTextile", false)?"TRUE":"FALSE").ToUpper() == "TRUE"; + if(convertTextile) + { + converter = new TextileToAsciiDocConverter(); + } + if (config.ContainsKey("VersionCustomFields")) { //this is needed specifically for Redmine because we cannot via the API figure out if a custom field is of type "version" //without having admin rights. @@ -309,7 +316,8 @@ private SoftwareSystemTestItem CreateTestCase(List issues, Redmine resultItem.Link = new Uri($"{baseURL}{resultItem.ItemID}"); } logger.Debug($"Getting test steps for item: {redmineItem.Id}"); - var testCaseSteps = GetTestSteps(redmineItem.Description ?? string.Empty); + string itemDescription = redmineItem.Description ?? string.Empty; + var testCaseSteps = GetTestSteps(convertTextile ? converter.ConvertTextile2AsciiDoc(itemDescription) : itemDescription); foreach (var testCaseStep in testCaseSteps) { resultItem.AddTestCaseStep(testCaseStep); @@ -356,7 +364,8 @@ private DocContentItem CreateDocContent(RedmineIssue redmineItem) resultItem.ItemRevision = redmineItem.UpdatedOn.ToString(); resultItem.ItemLastUpdated = (DateTime)redmineItem.UpdatedOn; resultItem.ItemStatus = redmineItem.Status.Name ?? string.Empty; - resultItem.DocContent = redmineItem.Description.ToString(); + string itemDescription = redmineItem.Description.ToString(); + resultItem.DocContent = convertTextile?converter.ConvertTextile2AsciiDoc(itemDescription):itemDescription; if (redmineItem.FixedVersion != null) { resultItem.ItemTargetVersion = redmineItem.FixedVersion.Name ?? string.Empty; @@ -417,7 +426,8 @@ private SOUPItem CreateSOUP(RedmineIssue redmineItem) } if (field.Name == "SOUP Detailed Description") { - resultItem.SOUPDetailedDescription = value.GetString(); + string detailedDescription = value.GetString(); + resultItem.SOUPDetailedDescription = convertTextile ? converter.ConvertTextile2AsciiDoc(detailedDescription) : detailedDescription; } else if (field.Name == "Performance Critical?") { @@ -725,8 +735,8 @@ private RequirementItem CreateRequirement(List issues, RedmineIssu { resultItem.RequirementAssignee = string.Empty; } - - resultItem.RequirementDescription = redmineItem.Description ?? string.Empty; + string itemDescription = redmineItem.Description ?? string.Empty; + resultItem.RequirementDescription = convertTextile ? converter.ConvertTextile2AsciiDoc(itemDescription) : itemDescription; resultItem.ItemID = redmineItem.Id.ToString(); resultItem.ItemRevision = redmineItem.UpdatedOn.ToString(); resultItem.ItemLastUpdated = (DateTime)redmineItem.UpdatedOn; diff --git a/RoboClerk.Redmine/TextileToASCIIDoc.cs b/RoboClerk.Redmine/TextileToASCIIDoc.cs new file mode 100644 index 0000000..30af017 --- /dev/null +++ b/RoboClerk.Redmine/TextileToASCIIDoc.cs @@ -0,0 +1,221 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace RoboClerk.Redmine +{ + public class TextileToAsciiDocConverter + { + /// + /// Converts a Textile string (as used by Redmine) into an AsciiDoc string. + /// + /// The Textile formatted string. + /// The converted AsciiDoc string. + public string ConvertTextile2AsciiDoc(string textile) + { + if (textile == null) + throw new ArgumentNullException(nameof(textile)); + + // --- Convert Headings --- + // Example: "h1. Heading" => "== Heading" + textile = Regex.Replace(textile, @"^h(\d)\.\s+(.*)$", m => + { + int level = int.Parse(m.Groups[1].Value); + // In AsciiDoc, a level-1 section is typically "==", level-2 "===", etc. + string prefix = new string('=', level + 1); + return $"{prefix} {m.Groups[2].Value}"; + }, RegexOptions.Multiline); + + // --- Convert Links --- + // Textile: "link text":http://example.com + // AsciiDoc: link:http://example.com[link text] + textile = Regex.Replace(textile, @"""([^""]+)""\s*:\s*(\S+)", "link:$2[$1]"); + + // --- Convert Images --- + // Textile: !http://example.com/image.png! + // AsciiDoc: image::http://example.com/image.png[] + textile = Regex.Replace(textile, @"!(\S+)!", + m => $"image::{m.Groups[1].Value}[]"); + + // --- Convert Unordered Lists --- + // Instead of using spaces for nesting, output multiple '*' characters. + // Example: Textile "* Item" or "** Nested item" become "* Item" and "** Item" + textile = Regex.Replace(textile, @"^(?\*+)\s+", m => + { + int level = m.Groups["stars"].Value.Length; + return new string('*', level) + " "; + }, RegexOptions.Multiline); + + // --- Convert Ordered Lists --- + // Instead of using spaces for nesting, output multiple '.' characters. + // Example: Textile "# Item" or "## Nested item" become ". Item" and ".. Item" + textile = Regex.Replace(textile, @"^(?#+)\s+", m => + { + int level = m.Groups["hashes"].Value.Length; + return new string('.', level) + " "; + }, RegexOptions.Multiline); + + // --- Convert Blockquotes (bq.) --- + // Textile blockquotes starting with "bq. " become AsciiDoc blockquotes. + textile = Regex.Replace(textile, @"^bq\.\s+(.*)$", m => + "____\n" + m.Groups[1].Value + "\n____", RegexOptions.Multiline); + + // --- Convert Blockquotes (lines beginning with '>') --- + // Lines starting with ">" are also treated as quotes. + textile = Regex.Replace(textile, @"^>\s*(.*)$", m => + "____\n" + m.Groups[1].Value + "\n____", RegexOptions.Multiline); + + // --- Convert Code Blocks --- + // Textile code blocks beginning with "bc. " are wrapped in AsciiDoc source block delimiters. + textile = Regex.Replace(textile, @"^bc\.\s+(.*)$", m => + "[source]\n----\n" + m.Groups[1].Value + "\n----", RegexOptions.Multiline); + + // --- Convert Inline Code --- + // Textile inline code marked with @ characters is converted to AsciiDoc inline code. + // Example: @print("hello")@ becomes `print("hello")` + textile = Regex.Replace(textile, @"@([^@]+)@", m => + "`" + m.Groups[1].Value + "`"); + + // --- Convert Strikethrough --- + // Textile uses hyphen-delimited text for strikethrough, e.g.: -deleted text- + // We convert it to AsciiDoc’s inline strike format: [strike]#text# + textile = Regex.Replace(textile, @"(^|\s)-(.+?)-(?=$|\s)", m => + m.Groups[1].Value + "[strike]#" + m.Groups[2].Value + "#"); + + // --- Process Tables --- + // Convert contiguous table lines into an AsciiDoc table block. + textile = ProcessTables(textile); + + // --- Ensure List Blocks Are Preceded by a Blank Line --- + textile = EnsureListBlocksHaveLeadingBlankLine(textile); + + return textile; + } + + /// + /// Processes contiguous table lines in the input and wraps them in an AsciiDoc table block. + /// + /// The text to process. + /// The text with any detected tables converted. + private string ProcessTables(string input) + { + var lines = input.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None); + var output = new StringBuilder(); + var tableBlock = new List(); + + foreach (var line in lines) + { + // A simple pattern: a table row starts and ends with a pipe. + if (Regex.IsMatch(line, @"^\|.*\|$")) + { + tableBlock.Add(line); + } + else + { + if (tableBlock.Count > 0) + { + output.Append(ProcessTableBlock(tableBlock)); + tableBlock.Clear(); + } + output.AppendLine(line); + } + } + if (tableBlock.Count > 0) + { + output.Append(ProcessTableBlock(tableBlock)); + } + return output.ToString(); + } + + /// + /// Converts a block of Textile table rows into an AsciiDoc table. + /// + /// A list of strings, each representing a table row in Textile. + /// A string containing the AsciiDoc table block. + private string ProcessTableBlock(List tableLines) + { + var sb = new StringBuilder(); + int numColumns = 0; + if (tableLines.Count > 0) + { + // Determine the number of columns from the first row. + var firstLine = tableLines[0].Trim(); + // Splitting by '|' leaves empty strings at the beginning and end. + var cells = Regex.Split(firstLine, @"\|") + .Where(x => !string.IsNullOrEmpty(x)) + .ToArray(); + numColumns = cells.Length; + sb.AppendLine($"[cols=\"{numColumns}*\"]"); + } + sb.AppendLine("|==="); + foreach (var line in tableLines) + { + string trimmedLine = line.Trim(); + // Remove the starting and ending pipe (if present) + if (trimmedLine.StartsWith("|")) + trimmedLine = trimmedLine.Substring(1); + if (trimmedLine.EndsWith("|")) + trimmedLine = trimmedLine.Substring(0, trimmedLine.Length - 1); + // Split the row into cells. + var cells = trimmedLine.Split(new char[] { '|' }, StringSplitOptions.None); + // Process each cell: if the cell starts with "_.", treat it as a header cell. + for (int i = 0; i < cells.Length; i++) + { + string cell = cells[i].Trim(); + if (cell.StartsWith("_.")) + { + // Remove the header marker and prefix with '^' for AsciiDoc header cell. + cell = cell.Substring(2).Trim(); + cells[i] = $"^{cell}"; + } + else + { + cells[i] = cell; + } + } + // In AsciiDoc, each row starts with a pipe, and cells are separated by " |" + sb.Append("|" + string.Join(" |", cells) + "\n"); + } + sb.AppendLine("|==="); + return sb.ToString(); + } + + /// + /// Ensures that list blocks (ordered or unordered) are preceded by a blank line. + /// This helps the AsciiDoc processor to correctly recognize them as lists. + /// + /// The converted text. + /// The text with a blank line inserted before list blocks where needed. + private string EnsureListBlocksHaveLeadingBlankLine(string text) + { + // Split by newline so we can process line by line. + var lines = text.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None); + var result = new List(); + + // We'll insert a blank line before the first line of any contiguous list block, + // provided it isn't already preceded by a blank line. + for (int i = 0; i < lines.Length; i++) + { + // Detect if the current line is a list item. + bool isListItem = Regex.IsMatch(lines[i], @"^(?:\*+|\.+)\s+"); + if (isListItem) + { + // If this is the first line of the file, or the previous line is not blank, + // and we are at the start of a list block, then insert a blank line. + if (i > 0 && !string.IsNullOrWhiteSpace(lines[i - 1])) + { + // Also, avoid inserting duplicate blank lines. + if (result.Count > 0 && !string.IsNullOrWhiteSpace(result.Last())) + { + result.Add(string.Empty); + } + } + } + result.Add(lines[i]); + } + return string.Join("\n", result); + } + } +} diff --git a/RoboClerk/ItemTemplateSupport/ScriptingBridge.cs b/RoboClerk/ItemTemplateSupport/ScriptingBridge.cs index 1b58678..4d595c3 100644 --- a/RoboClerk/ItemTemplateSupport/ScriptingBridge.cs +++ b/RoboClerk/ItemTemplateSupport/ScriptingBridge.cs @@ -1,6 +1,8 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Text; +using System.Text.RegularExpressions; namespace RoboClerk { @@ -161,10 +163,63 @@ public string GetValOrDef(string value, string defaultValue) /// Convenience function, calls ToString on input and returns resulting string. /// /// - /// + /// string representation public string Insert(object input) { return input.ToString(); } + + /// + /// Convenience function, takes any asciidoc tables in the input and makes them + /// embedded tables. + /// + /// + /// + public string EmbedAsciidocTables(string input) + { + // Split the input into lines (preserving newlines) + var lines = input.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None); + var outputLines = new List(); + bool inTable = false; + + foreach (var line in lines) + { + string trimmed = line.Trim(); + + // Detect the table delimiter + if (trimmed == "|===") + { + inTable = !inTable; + // Replace outer table delimiter with nested table delimiter + outputLines.Add(line.Replace("|===", "!===")); + } + else if (inTable) + { + // Preserve leading whitespace + int leadingSpaces = line.Length - line.TrimStart().Length; + string converted = line; + + // If the cell begins with a pipe, replace it with an exclamation mark, + // but only if the pipe is not escaped. + if (line.TrimStart().StartsWith("|")) + { + converted = new string(' ', leadingSpaces) + "!" + line.TrimStart().Substring(1); + } + + // Replace any unescaped cell separator pipe. + // The regex (?(IEnumerable items) { foreach (var obj in items) @@ -109,11 +135,13 @@ private void ScrubItemsFields(IEnumerable items) { if (property.PropertyType == typeof(string) && property.CanWrite) { - // asciidoc uses | to seperate fields in a table, if the fields - // themselves contain a | character it needs to be escaped. string currentValue = (string)property.GetValue(obj); - string newValue = Regex.Replace(currentValue, "(?