Skip to content

Use new JSON Streaming methods for Invoke-RestMethod #15697

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
strawgate wants to merge 15 commits into from
Closed

Use new JSON Streaming methods for Invoke-RestMethod #15697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
cad3493
Remove preparse for array-based json
strawgate Jun 29, 2021
2948d04
WIP POC
strawgate Jun 29, 2021
db2a47b
improve token handling and if the first element is not an array then …
strawgate Jun 29, 2021
c2ecea3
Fold back into using a serializer so we can use the requested seriali…
strawgate Jun 29, 2021
9416881
Fixing some failing tests and responding to a PR comment
strawgate Jun 29, 2021
7039296
Remove commented out function
strawgate Jun 29, 2021
f197057
Code factor cleanup
strawgate Jun 29, 2021
f873392
Add some comments, additional empty line removal
strawgate Jun 29, 2021
e2de68f
Feedback from PR comments
strawgate Jun 29, 2021
1ceaf24
Cleanup line for code factor
strawgate Jun 29, 2021
5056eb2
Wire up invoke-restmethod with new json conversion streams
strawgate Jun 30, 2021
c52ceb4
refactor from testing, add support for asHashtable to invoke-restmethod
strawgate Jun 30, 2021
15dbebc
Fixes from CI testing
strawgate Jun 30, 2021
8b1756d
Did not mean to commit these changes
strawgate Jun 30, 2021
bddf4d8
Dont use the same names for stream and string json deserializers. Thi…
strawgate Jun 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 38 additions & 14 deletions ...hell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ public int MaximumFollowRelLink

set { base._maximumFollowRelLink = value; }
}
/// <summary>
/// Returned data structure is a Hashtable instead a CustomPSObject.
/// </summary>
[Parameter()]
public SwitchParameter AsHashtable { get; set; }

/// <summary>
/// Gets or sets the ResponseHeadersVariable property.
Expand Down Expand Up @@ -181,19 +186,20 @@ private static bool TryConvertToXml(string xml, out object doc, ref Exception ex
return (doc != null);
}

private static bool TryConvertToJson(string json, out object obj, ref Exception exRef)
private static bool TryConvertToJson(StreamReader stream, Encoding encoding, bool AsHashtable, out object obj, ref Exception exRef)
{
bool converted = false;
try
{
ErrorRecord error;
obj = JsonObject.ConvertFromJson(json, out error);
obj = JsonObject.ConvertFromJsonStream(stream, AsHashtable, out error);

if (obj == null)
{
// This ensures that a null returned by ConvertFromJson() is the actual JSON null literal.
// if not, the ArgumentException will be caught.
JToken.Parse(json);
string str = StreamHelper.DecodeStream(stream.BaseStream, ref encoding);
JToken.Parse(str);
}

if (error != null)
Expand Down Expand Up @@ -417,8 +423,6 @@ internal override void ProcessResponse(HttpResponseMessage response)
object obj = null;
Exception ex = null;

string str = StreamHelper.DecodeStream(responseStream, ref encoding);

string encodingVerboseName;
try
{
Expand All @@ -437,20 +441,40 @@ internal override void ProcessResponse(HttpResponseMessage response)
);
bool convertSuccess = false;

if (returnType == RestReturnType.Json)
{
convertSuccess = TryConvertToJson(str, out obj, ref ex) || TryConvertToXml(str, out obj, ref ex);
}
// default to try xml first since it's more common
else
string str;

switch (returnType)
{
convertSuccess = TryConvertToXml(str, out obj, ref ex) || TryConvertToJson(str, out obj, ref ex);
case RestReturnType.Json:
convertSuccess = TryConvertToJson(new StreamReader(responseStream), encoding, AsHashtable, out obj, ref ex);
break;
case RestReturnType.Xml:
str = StreamHelper.DecodeStream(responseStream, ref encoding);
convertSuccess = TryConvertToXml(str, out obj, ref ex);
break;
default:
str = StreamHelper.DecodeStream(responseStream, ref encoding);
convertSuccess = TryConvertToXml(str, out obj, ref ex);
break;
}

// If at first we don't succeed, try again with a different format
if (!convertSuccess)
{
// fallback to string
obj = str;
switch (returnType)
{
case RestReturnType.Json:
str = StreamHelper.DecodeStream(responseStream, ref encoding);
convertSuccess = TryConvertToXml(str, out obj, ref ex);
break;
case RestReturnType.Xml:
convertSuccess = TryConvertToJson(new StreamReader(responseStream), encoding, AsHashtable, out obj, ref ex);
break;
default:
str = StreamHelper.DecodeStream(responseStream, ref encoding);
obj = str;
break;
}
}

WriteObject(obj);
Expand Down
171 changes: 134 additions & 37 deletions src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/JsonObject.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using System.Management.Automation;
using System.Management.Automation.Language;
using System.Reflection;
Expand Down Expand Up @@ -110,6 +111,18 @@ public DuplicateMemberHashSet(int capacity)

#region ConvertFromJson

/// <summary>
/// Convert a Json string back to an object of type PSObject.
/// </summary>
/// <param name="stream">The json text to convert.</param>
/// <param name="error">An error record if the conversion failed.</param>
/// <returns>A PSObject.</returns>
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", Justification = "Preferring Json over JSON")]
public static object ConvertFromJsonStream(StreamReader stream, out ErrorRecord error)
{
return ConvertFromJsonStream(stream, returnHashtable: false, out error);
}

/// <summary>
/// Convert a Json string back to an object of type PSObject.
/// </summary>
Expand All @@ -122,6 +135,22 @@ public static object ConvertFromJson(string input, out ErrorRecord error)
return ConvertFromJson(input, returnHashtable: false, out error);
}

/// <summary>
/// Convert a Json string back to an object of type <see cref="System.Management.Automation.PSObject"/> or
/// <see cref="System.Collections.Hashtable"/> depending on parameter <paramref name="returnHashtable"/>.
/// </summary>
/// <param name="stream">The json text to convert.</param>
/// <param name="returnHashtable">True if the result should be returned as a <see cref="System.Collections.Hashtable"/>
/// instead of a <see cref="System.Management.Automation.PSObject"/></param>
/// <param name="error">An error record if the conversion failed.</param>
/// <returns>A <see cref="System.Management.Automation.PSObject"/> or a <see cref="System.Collections.Hashtable"/>
/// if the <paramref name="returnHashtable"/> parameter is true.</returns>
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", Justification = "Preferring Json over JSON")]
public static object ConvertFromJsonStream(StreamReader stream, bool returnHashtable, out ErrorRecord error)
{
return ConvertFromJsonStream(stream, returnHashtable, maxDepth: 1024, out error);
}

/// <summary>
/// Convert a Json string back to an object of type <see cref="System.Management.Automation.PSObject"/> or
/// <see cref="System.Collections.Hashtable"/> depending on parameter <paramref name="returnHashtable"/>.
Expand Down Expand Up @@ -152,53 +181,73 @@ public static object ConvertFromJson(string input, bool returnHashtable, out Err
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", Justification = "Preferring Json over JSON")]
public static object ConvertFromJson(string input, bool returnHashtable, int? maxDepth, out ErrorRecord error)
{
if (input == null)
StreamReader thisStream = new StreamReader(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(input)));
return ConvertFromJsonStream(thisStream, returnHashtable, maxDepth, out error);
}

/// <summary>
/// Convert a JSON string back to an object of type <see cref="System.Management.Automation.PSObject"/> or
/// <see cref="System.Collections.Hashtable"/> depending on parameter <paramref name="returnHashtable"/>.
/// </summary>
/// <param name="stream">The JSON text to convert.</param>
/// <param name="returnHashtable">True if the result should be returned as a <see cref="System.Collections.Hashtable"/>
/// instead of a <see cref="System.Management.Automation.PSObject"/>.</param>
/// <param name="maxDepth">The max depth allowed when deserializing the json input. Set to null for no maximum.</param>
/// <param name="error">An error record if the conversion failed.</param>
/// <returns>A <see cref="System.Management.Automation.PSObject"/> or a <see cref="System.Collections.Hashtable"/>
/// if the <paramref name="returnHashtable"/> parameter is true.</returns>
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", Justification = "Preferring Json over JSON")]
public static object ConvertFromJsonStream(StreamReader stream, bool returnHashtable, int? maxDepth, out ErrorRecord error)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(input));
throw new ArgumentNullException(nameof(stream));
}

error = null;
try
{
// JsonConvert.DeserializeObject does not throw an exception when an invalid Json array is passed.
// This issue is being tracked by https://github.com/JamesNK/Newtonsoft.Json/issues/1930.
// To work around this, we need to identify when input is a Json array, and then try to parse it via JArray.Parse().
var serializer = new JsonSerializer()
{
TypeNameHandling = TypeNameHandling.None,
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
MaxDepth = maxDepth
};

// If input starts with '[' (ignoring white spaces).
if (Regex.Match(input, @"^\s*\[").Success)
serializer.TypeNameHandling = TypeNameHandling.None;
serializer.MetadataPropertyHandling = MetadataPropertyHandling.Ignore;
serializer.MaxDepth = maxDepth;

using (JsonReader reader = new JsonTextReader(stream))
{
// JArray.Parse() will throw a JsonException if the array is invalid.
// This will be caught by the catch block below, and then throw an
// ArgumentException - this is done to have same behavior as the JavaScriptSerializer.
JArray.Parse(input);
var readResult = reader.Read();

// Please note that if the Json array is valid, we don't do anything,
// we just continue the deserialization.
}
// If the first token in our file is an array let's read in that token so that our next token is an object or a primitive.
// This will allow newtonsoft to deserialize the incoming json one object at a time instead of doing the whole object at once.
if (reader.TokenType == JsonToken.StartArray)
{
List<object> result;

result = DeserializeRootArrayHelper(reader, serializer, returnHashtable, out error);
if (error != null)
{
return null;
}

var obj = JsonConvert.DeserializeObject(
input,
new JsonSerializerSettings
return result.ToArray();
}
else
{
// This TypeNameHandling setting is required to be secure.
TypeNameHandling = TypeNameHandling.None,
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
MaxDepth = maxDepth
});
object result;

switch (obj)
{
case JObject dictionary:
// JObject is a IDictionary
return returnHashtable
? PopulateHashTableFromJDictionary(dictionary, out error)
: PopulateFromJDictionary(dictionary, new DuplicateMemberHashSet(dictionary.Count), out error);
case JArray list:
return returnHashtable
? PopulateHashTableFromJArray(list, out error)
: PopulateFromJArray(list, out error);
default:
return obj;
result = DeserializeObjectHelper(reader, serializer, returnHashtable, out error);
if (error != null)
{
return null;
}

return result;
}
}
}
catch (JsonException je)
Expand All @@ -210,6 +259,56 @@ public static object ConvertFromJson(string input, bool returnHashtable, int? ma
}
}

private static List<object> DeserializeRootArrayHelper(JsonReader reader, JsonSerializer serializer, bool returnHashtable, out ErrorRecord error)
Copy link
Member

@TravisEz13 TravisEz13 Jul 12, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add description of the method and parameters

{
error = null;

List<object> elements = new List<object>();

while (reader.Read() && reader.TokenType != JsonToken.EndArray)
{
elements.Add(DeserializeObjectHelper(reader, serializer, returnHashtable, out error));
if (error != null)
{
return null;
}
}

if (reader.TokenType != JsonToken.EndArray)
{
var errorMsg = string.Format(CultureInfo.CurrentCulture, WebCmdletStrings.JsonStringInBadFormat);
error = new ErrorRecord(
new InvalidOperationException(errorMsg),
"ArrayStartsButDoesNotEnd",
ErrorCategory.InvalidOperation,
null);
throw new ArgumentException(errorMsg);
}

return elements;
}

private static object DeserializeObjectHelper(JsonReader reader, JsonSerializer serializer, bool returnHashtable, out ErrorRecord error)
{
error = null;

var thisObject = serializer.Deserialize(reader);

switch (thisObject)
{
case JObject dictionary:
return returnHashtable
? PopulateHashTableFromJDictionary(dictionary, out error)
: PopulateFromJDictionary(dictionary, new DuplicateMemberHashSet(dictionary.Count), out error);
case JArray list:
return returnHashtable
? PopulateHashTableFromJArray(list, out error)
: PopulateFromJArray(list, out error);
default:
return thisObject;
}
}

// This function is a clone of PopulateFromDictionary using JObject as an input.
private static PSObject PopulateFromJDictionary(JObject entries, DuplicateMemberHashSet memberHashTracker, out ErrorRecord error)
{
Expand Down Expand Up @@ -447,7 +546,6 @@ private static ICollection<object> PopulateHashTableFromJArray(JArray list, out
#endregion ConvertFromJson

#region ConvertToJson

/// <summary>
/// Convert an object to JSON string.
/// </summary>
Expand Down Expand Up @@ -541,7 +639,6 @@ private static object ProcessValue(object obj, int currentDepth, in ConvertToJso
else
{
Type t = obj.GetType();

if (t.IsPrimitive)
{
rv = obj;
Expand Down