Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: loading the document requires knowing the format in advance #1929

Closed
wants to merge 20 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
ecdf133
Make format optional; infer it from the stream or textReader input in…
MaggieKimani1 Nov 13, 2024
76286ae
Convert the methods to async to avoid deadlocks
MaggieKimani1 Nov 13, 2024
ab059fc
Align wrapped params
MaggieKimani1 Nov 13, 2024
e01db74
Pass a cancellation token to allow cancelling requests/tasks
MaggieKimani1 Nov 13, 2024
f30add8
clean up code
MaggieKimani1 Nov 13, 2024
8d83694
Return the generic output of type T as part of the read result; clean…
MaggieKimani1 Nov 13, 2024
0c05c3f
Clean up tests to reflect new LoadAsync pattern
MaggieKimani1 Nov 13, 2024
917fae3
Update doc comments and doc initialization
MaggieKimani1 Nov 14, 2024
8b07082
Update public API
MaggieKimani1 Nov 14, 2024
a83461d
Refactor code to cater for non-seekable streams
MaggieKimani1 Nov 14, 2024
7468eb2
clean up code
MaggieKimani1 Nov 14, 2024
93ba1c1
chore: use of async life time for test initialization
baywet Nov 18, 2024
47bfb6f
Move code block to a reusable method to avoid duplication
MaggieKimani1 Nov 19, 2024
49a181a
use async lifetime for test initialization
MaggieKimani1 Nov 19, 2024
ecb9d6e
code refactor to remove Task.Run on CPU bound operation
MaggieKimani1 Nov 19, 2024
ebdbbe6
remove Task.Run
MaggieKimani1 Nov 19, 2024
9389d79
pass cancellation token
MaggieKimani1 Nov 19, 2024
9a2be7f
run operation asynchronously
MaggieKimani1 Nov 19, 2024
f07ff44
Update public API
MaggieKimani1 Nov 19, 2024
afc4055
Update src/Microsoft.OpenApi/Reader/OpenApiModelFactory.cs
MaggieKimani1 Nov 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ var httpClient = new HttpClient
var stream = await httpClient.GetStreamAsync("master/examples/v3.0/petstore.yaml");

// Read V3 as YAML
var openApiDocument = new OpenApiStreamReader().Read(stream, out var diagnostic);
var openApiDocument = OpenApiDocument.LoadAsync(stream).OpenApiDocument;

// Write V2 as JSON
var outputString = openApiDocument.Serialize(OpenApiSpecVersion.OpenApi2_0, OpenApiFormat.Json);
Expand Down
19 changes: 9 additions & 10 deletions src/Microsoft.OpenApi.Hidi/OpenApiService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public static async Task TransformOpenApiDocumentAsync(HidiOptions options, ILog
}

// Load OpenAPI document
var format = OpenApiModelFactory.GetFormat(options.OpenApi);
var format = await OpenApiModelFactory.GetFormatAsync(options.OpenApi, cancellationToken).ConfigureAwait(false);
var document = await GetOpenApiAsync(options, format, logger, options.MetadataVersion, cancellationToken).ConfigureAwait(false);

if (options.FilterOptions != null)
Expand Down Expand Up @@ -396,8 +396,7 @@ private static async Task<ReadResult> ParseOpenApiAsync(string openApiFile, bool
new Uri("file://" + new FileInfo(openApiFile).DirectoryName + Path.DirectorySeparatorChar)
};

var format = OpenApiModelFactory.GetFormat(openApiFile);
result = await OpenApiDocument.LoadAsync(stream, format, settings, cancellationToken).ConfigureAwait(false);
result = await OpenApiDocument.LoadAsync(stream, settings, cancellationToken).ConfigureAwait(false);

logger.LogTrace("{Timestamp}ms: Completed parsing.", stopwatch.ElapsedMilliseconds);

Expand All @@ -421,7 +420,7 @@ public static async Task<OpenApiDocument> ConvertCsdlToOpenApiAsync(Stream csdl,
settings ??= SettingsUtilities.GetConfiguration();

var document = edmModel.ConvertToOpenApi(SettingsUtilities.GetOpenApiConvertSettings(settings, metadataVersion));
document = FixReferences(document, format);
document = await FixReferencesAsync(document, format).ConfigureAwait(false);

return document;
}
Expand All @@ -431,17 +430,17 @@ public static async Task<OpenApiDocument> ConvertCsdlToOpenApiAsync(Stream csdl,
/// </summary>
/// <param name="document"> The converted OpenApiDocument.</param>
/// <returns> A valid OpenApiDocument instance.</returns>
public static OpenApiDocument FixReferences(OpenApiDocument document, string format)
public static async Task<OpenApiDocument> FixReferencesAsync(OpenApiDocument document, string format)
{
// This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance.
// So we write it out, and read it back in again to fix it up.

var sb = new StringBuilder();
document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb)));

var doc = OpenApiDocument.Parse(sb.ToString(), format).OpenApiDocument;
var result = await OpenApiDocument.ParseAsync(sb.ToString()).ConfigureAwait(false);

return doc;
return result.OpenApiDocument;
}

/// <summary>
Expand Down Expand Up @@ -587,7 +586,7 @@ private static string GetInputPathExtension(string? openapi = null, string? csdl
throw new ArgumentException("Please input a file path or URL");
}

var format = OpenApiModelFactory.GetFormat(options.OpenApi);
var format = await OpenApiModelFactory.GetFormatAsync(options.OpenApi, cancellationToken).ConfigureAwait(false);
var document = await GetOpenApiAsync(options, format, logger, null, cancellationToken).ConfigureAwait(false);

using (logger.BeginScope("Creating diagram"))
Expand Down Expand Up @@ -749,10 +748,10 @@ internal static async Task PluginManifestAsync(HidiOptions options, ILogger logg
}

// Load OpenAPI document
var format = OpenApiModelFactory.GetFormat(options.OpenApi);
var format = await OpenApiModelFactory.GetFormatAsync(options.OpenApi, cancellationToken).ConfigureAwait(false);
var document = await GetOpenApiAsync(options, format, logger, options.MetadataVersion, cancellationToken).ConfigureAwait(false);

cancellationToken.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();

if (options.FilterOptions != null)
{
Expand Down
23 changes: 11 additions & 12 deletions src/Microsoft.OpenApi.Readers/OpenApiYamlReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
using Microsoft.OpenApi.Reader;
using SharpYaml.Serialization;
using System.Linq;
using Microsoft.OpenApi.Models;

namespace Microsoft.OpenApi.Readers
{
Expand Down Expand Up @@ -46,26 +45,26 @@ public async Task<ReadResult> ReadAsync(TextReader input,
}

/// <inheritdoc/>
public T ReadFragment<T>(TextReader input,
OpenApiSpecVersion version,
out OpenApiDiagnostic diagnostic,
OpenApiReaderSettings settings = null) where T : IOpenApiElement
public async Task<ReadFragmentResult<T>> ReadFragmentAsync<T>(TextReader input,
OpenApiSpecVersion version,
OpenApiReaderSettings settings = null,
CancellationToken token = default) where T : IOpenApiElement
{
JsonNode jsonNode;

// Parse the YAML
try
{
jsonNode = LoadJsonNodesFromYamlDocument(input);
jsonNode = await Task.Run(() => LoadJsonNodesFromYamlDocument(input));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why are we adding the task run here?

Copy link
Contributor Author

@MaggieKimani1 MaggieKimani1 Nov 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The caller is async here

public static async Task<ReadFragmentResult<T>> LoadAsync<T>(Stream input,

And the interface method's equivalent in the JsonReader class is also async

public async Task<ReadFragmentResult<T>> ReadFragmentAsync<T>(TextReader input,

If I change this method's signature to be synchronous, I'll also need to update this JSON content to be read synchronously:

var content = await input.ReadToEndAsync();

}
catch (JsonException ex)
{
diagnostic = new();
var diagnostic = new OpenApiDiagnostic();
diagnostic.Errors.Add(new($"#line={ex.LineNumber}", ex.Message));
return default;
}

return ReadFragment<T>(jsonNode, version, out diagnostic);
return ReadFragment<T>(jsonNode, version, settings);
}

/// <summary>
Expand All @@ -82,15 +81,15 @@ static JsonNode LoadJsonNodesFromYamlDocument(TextReader input)
}

/// <inheritdoc/>
public async Task<ReadResult> ReadAsync(JsonNode jsonNode, OpenApiReaderSettings settings, string format = null, CancellationToken cancellationToken = default)
public async Task<ReadResult> ReadAsync(JsonNode jsonNode, OpenApiReaderSettings settings, CancellationToken cancellationToken = default)
{
return await OpenApiReaderRegistry.DefaultReader.ReadAsync(jsonNode, settings, OpenApiConstants.Yaml, cancellationToken);
return await OpenApiReaderRegistry.DefaultReader.ReadAsync(jsonNode, settings, cancellationToken);
}

/// <inheritdoc/>
public T ReadFragment<T>(JsonNode input, OpenApiSpecVersion version, out OpenApiDiagnostic diagnostic, OpenApiReaderSettings settings = null) where T : IOpenApiElement
public ReadFragmentResult<T> ReadFragment<T>(JsonNode input, OpenApiSpecVersion version, OpenApiReaderSettings settings = null) where T : IOpenApiElement
{
return OpenApiReaderRegistry.DefaultReader.ReadFragment<T>(input, version, out diagnostic);
return OpenApiReaderRegistry.DefaultReader.ReadFragment<T>(input, version, settings);
}
}
}
3 changes: 1 addition & 2 deletions src/Microsoft.OpenApi.Workbench/MainModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,7 @@ internal async Task ParseDocumentAsync()
}
}

var format = OpenApiModelFactory.GetFormat(_inputFile);
var readResult = await OpenApiDocument.LoadAsync(stream, format);
var readResult = await OpenApiDocument.LoadAsync(stream);
var document = readResult.OpenApiDocument;
var context = readResult.OpenApiDiagnostic;

Expand Down
10 changes: 4 additions & 6 deletions src/Microsoft.OpenApi/Interfaces/IOpenApiReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,28 +29,26 @@ public interface IOpenApiReader
/// <param name="jsonNode">The JsonNode input.</param>
/// <param name="settings">The Reader settings to be used during parsing.</param>
/// <param name="cancellationToken">Propagates notifications that operations should be cancelled.</param>
/// <param name="format">The OpenAPI format.</param>
/// <returns></returns>
Task<ReadResult> ReadAsync(JsonNode jsonNode, OpenApiReaderSettings settings, string format = null, CancellationToken cancellationToken = default);
Task<ReadResult> ReadAsync(JsonNode jsonNode, OpenApiReaderSettings settings, CancellationToken cancellationToken = default);

/// <summary>
/// Reads the TextReader input and parses the fragment of an OpenAPI description into an Open API Element.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="version">Version of the OpenAPI specification that the fragment conforms to.</param>
/// <param name="diagnostic">Returns diagnostic object containing errors detected during parsing.</param>
/// <param name="settings">The OpenApiReader settings.</param>
/// <param name="token"></param>
/// <returns>Instance of newly created IOpenApiElement.</returns>
T ReadFragment<T>(TextReader input, OpenApiSpecVersion version, out OpenApiDiagnostic diagnostic, OpenApiReaderSettings settings = null) where T : IOpenApiElement;
Task<ReadFragmentResult<T>> ReadFragmentAsync<T>(TextReader input, OpenApiSpecVersion version, OpenApiReaderSettings settings = null, CancellationToken token = default) where T : IOpenApiElement;

/// <summary>
/// Reads the JsonNode input and parses the fragment of an OpenAPI description into an Open API Element.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="version">Version of the OpenAPI specification that the fragment conforms to.</param>
/// <param name="diagnostic">Returns diagnostic object containing errors detected during parsing.</param>
/// <param name="settings">The OpenApiReader settings.</param>
/// <returns>Instance of newly created IOpenApiElement.</returns>
T ReadFragment<T>(JsonNode input, OpenApiSpecVersion version, out OpenApiDiagnostic diagnostic, OpenApiReaderSettings settings = null) where T : IOpenApiElement;
ReadFragmentResult<T> ReadFragment<T>(JsonNode input, OpenApiSpecVersion version, OpenApiReaderSettings settings = null) where T: IOpenApiElement;
}
}
62 changes: 10 additions & 52 deletions src/Microsoft.OpenApi/Models/OpenApiDocument.cs
Original file line number Diff line number Diff line change
Expand Up @@ -535,93 +535,51 @@ private static string ConvertByteArrayToString(byte[] hash)
return Workspace?.ResolveReference<IOpenApiReferenceable>(uriLocation);
}

/// <summary>
/// Parses a local file path or Url into an Open API document.
/// </summary>
/// <param name="url"> The path to the OpenAPI file.</param>
/// <param name="settings"></param>
/// <returns></returns>
public static ReadResult Load(string url, OpenApiReaderSettings? settings = null)
{
return OpenApiModelFactory.Load(url, settings);
}

/// <summary>
/// Reads the stream input and parses it into an Open API document.
/// </summary>
/// <param name="stream">Stream containing OpenAPI description to parse.</param>
/// <param name="format">The OpenAPI format to use during parsing.</param>
/// <param name="settings">The OpenApi reader settings.</param>
/// <returns></returns>
public static ReadResult Load(Stream stream,
string format,
OpenApiReaderSettings? settings = null)
{
return OpenApiModelFactory.Load(stream, format, settings);
}

/// <summary>
/// Reads the text reader content and parses it into an Open API document.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="format"> The OpenAPI format to use during parsing.</param>
/// <param name="settings">The OpenApi reader settings.</param>
/// <returns></returns>
public static ReadResult Load(TextReader input,
string format,
OpenApiReaderSettings? settings = null)
{
return OpenApiModelFactory.Load(input, format, settings);
}

/// <summary>
/// Parses a local file path or Url into an Open API document.
/// </summary>
/// <param name="url"> The path to the OpenAPI file.</param>
/// <param name="settings">The OpenApi reader settings.</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task<ReadResult> LoadAsync(string url, OpenApiReaderSettings? settings = null)
public static async Task<ReadResult> LoadAsync(string url, OpenApiReaderSettings? settings = null, CancellationToken cancellationToken = default)
{
return await OpenApiModelFactory.LoadAsync(url, settings);
return await OpenApiModelFactory.LoadAsync(url, settings, cancellationToken);
}

/// <summary>
/// Reads the stream input and parses it into an Open API document.
/// </summary>
/// <param name="stream">Stream containing OpenAPI description to parse.</param>
/// <param name="format">The OpenAPI format to use during parsing.</param>
/// <param name="settings">The OpenApi reader settings.</param>
/// <param name="cancellationToken">Propagates information about operation cancelling.</param>
/// <returns></returns>
public static async Task<ReadResult> LoadAsync(Stream stream, string format, OpenApiReaderSettings? settings = null, CancellationToken cancellationToken = default)
public static async Task<ReadResult> LoadAsync(Stream stream, OpenApiReaderSettings? settings = null, CancellationToken cancellationToken = default)
{
return await OpenApiModelFactory.LoadAsync(stream, format, settings, cancellationToken);
return await OpenApiModelFactory.LoadAsync(stream, settings: settings, cancellationToken: cancellationToken);
}

/// <summary>
/// Reads the text reader content and parses it into an Open API document.
/// </summary>
/// <param name="input">TextReader containing OpenAPI description to parse.</param>
/// <param name="format"> The OpenAPI format to use during parsing.</param>
/// <param name="settings">The OpenApi reader settings.</param>
/// <returns></returns>
public static async Task<ReadResult> LoadAsync(TextReader input, string format, OpenApiReaderSettings? settings = null)
public static async Task<ReadResult> LoadAsync(TextReader input, OpenApiReaderSettings? settings = null)
{
return await OpenApiModelFactory.LoadAsync(input, format, settings);
return await OpenApiModelFactory.LoadAsync(input, settings: settings);
}

/// <summary>
/// Parses a string into a <see cref="OpenApiDocument"/> object.
/// </summary>
/// <param name="input"> The string input.</param>
/// <param name="format"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static ReadResult Parse(string input,
string? format = null,
OpenApiReaderSettings? settings = null)
public static async Task<ReadResult> ParseAsync(string input,
OpenApiReaderSettings? settings = null)
{
return OpenApiModelFactory.Parse(input, format, settings);
return await OpenApiModelFactory.ParseAsync(input, settings);
}
}

Expand Down
Loading
Loading