Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for HTTP operations on CoreCLR #78

Merged
merged 6 commits into from
Oct 3, 2020
Merged

Add support for HTTP operations on CoreCLR #78

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4aa1fe7
Add support for HTTP operations on CoreCLR
sblom Oct 2, 2020
a11168a
Refactored HttpClient usage into utility class
sblom Oct 2, 2020
d19bd8c
Re-add virtual to intended overrideable methods
sblom Oct 2, 2020
95f7f9d
Add `.ConfigureAwait(false)` everywhere we're doing async
sblom Oct 2, 2020
181da33
Merge branch 'master' into issues/21
sblom Oct 3, 2020
9e39cb9
Merge branch 'master' into issues/21
asbjornu Oct 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 55 additions & 175 deletions src/json-ld.net/Core/DocumentLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,73 @@
using JsonLD.Util;
using System.Net;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using System.Runtime.InteropServices;

namespace JsonLD.Core
{
public class DocumentLoader
{
const int MAX_REDIRECTS = 20;

/// <summary>An HTTP Accept header that prefers JSONLD.</summary>
/// <remarks>An HTTP Accept header that prefers JSONLD.</remarks>
public const string AcceptHeader = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1";

/// <exception cref="JsonLDNet.Core.JsonLdError"></exception>
public RemoteDocument LoadDocument(string url)
{
return LoadDocumentAsync(url).GetAwaiter().GetResult();
}

/// <exception cref="JsonLDNet.Core.JsonLdError"></exception>
public virtual RemoteDocument LoadDocument(string url)
public async Task<RemoteDocument> LoadDocumentAsync(string url)
{
#if !PORTABLE && !IS_CORECLR
RemoteDocument doc = new RemoteDocument(url, null);
HttpWebResponse resp;

try
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
req.Accept = AcceptHeader;
resp = (HttpWebResponse)req.GetResponse();
bool isJsonld = resp.Headers[HttpResponseHeader.ContentType] == "application/ld+json";
if (!resp.Headers[HttpResponseHeader.ContentType].Contains("json"))
HttpResponseMessage httpResponseMessage;

int redirects = 0;
int code;
string redirectedUrl = url;

// Manually follow redirects because .NET Core refuses to auto-follow HTTPS->HTTP redirects.
do
{
HttpRequestMessage httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, redirectedUrl);
httpRequestMessage.Headers.Add("Accept", AcceptHeader);
httpResponseMessage = await JSONUtils._HttpClient.SendAsync(httpRequestMessage);
if (httpResponseMessage.Headers.TryGetValues("Location", out var location))
{
redirectedUrl = location.First();
}

code = (int)httpResponseMessage.StatusCode;
} while (redirects++ < MAX_REDIRECTS && code >= 300 && code < 400);

if (redirects >= MAX_REDIRECTS)
{
throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, $"Too many redirects - {url}");
}

if (code >= 400)
{
throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, $"HTTP {code} {url}");
}

bool isJsonld = httpResponseMessage.Content.Headers.ContentType.MediaType == "application/ld+json";

// From RFC 6839, it looks like we should accept application/json and any MediaType ending in "+json".
if (httpResponseMessage.Content.Headers.ContentType.MediaType != "application/json" && !httpResponseMessage.Content.Headers.ContentType.MediaType.EndsWith("+json"))
{
throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url);
}

string[] linkHeaders = resp.Headers.GetValues("Link");
if (!isJsonld && linkHeaders != null)
if (!isJsonld && httpResponseMessage.Headers.TryGetValues("Link", out var linkHeaders))
{
linkHeaders = linkHeaders.SelectMany((h) => h.Split(",".ToCharArray()))
.Select(h => h.Trim()).ToArray();
Expand All @@ -41,189 +83,27 @@ public virtual RemoteDocument LoadDocument(string url)
}
string header = linkedContexts.First();
string linkedUrl = header.Substring(1, header.IndexOf(">") - 1);
string resolvedUrl = URL.Resolve(url, linkedUrl);
string resolvedUrl = URL.Resolve(redirectedUrl, linkedUrl);
var remoteContext = this.LoadDocument(resolvedUrl);
doc.contextUrl = remoteContext.documentUrl;
doc.context = remoteContext.document;
}

Stream stream = resp.GetResponseStream();
Stream stream = await httpResponseMessage.Content.ReadAsStreamAsync();

doc.DocumentUrl = req.Address.ToString();
doc.DocumentUrl = redirectedUrl;
doc.Document = JSONUtils.FromInputStream(stream);
}
catch (JsonLdError)
{
throw;
}
catch (WebException webException)
{
try
{
resp = (HttpWebResponse)webException.Response;
int baseStatusCode = (int)(Math.Floor((double)resp.StatusCode / 100)) * 100;
if (baseStatusCode == 300)
{
string location = resp.Headers[HttpResponseHeader.Location];
if (!string.IsNullOrWhiteSpace(location))
{
// TODO: Add recursion break or simply switch to HttpClient so we don't have to recurse on HTTP redirects.
return LoadDocument(location);
}
}
}
catch (Exception innerException)
{
throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url, innerException);
}

throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url, webException);
}
catch (Exception exception)
{
throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url, exception);
}
return doc;
#else
throw new PlatformNotSupportedException();
#endif
}

/// <summary>An HTTP Accept header that prefers JSONLD.</summary>
/// <remarks>An HTTP Accept header that prefers JSONLD.</remarks>
public const string AcceptHeader = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1";

// private static volatile IHttpClient httpClient;

// /// <summary>
// /// Returns a Map, List, or String containing the contents of the JSON
// /// resource resolved from the URL.
// /// </summary>
// /// <remarks>
// /// Returns a Map, List, or String containing the contents of the JSON
// /// resource resolved from the URL.
// /// </remarks>
// /// <param name="url">The URL to resolve</param>
// /// <returns>
// /// The Map, List, or String that represent the JSON resource
// /// resolved from the URL
// /// </returns>
// /// <exception cref="Com.Fasterxml.Jackson.Core.JsonParseException">If the JSON was not valid.
// /// </exception>
// /// <exception cref="System.IO.IOException">If there was an error resolving the resource.
// /// </exception>
// public static object FromURL(URL url)
// {
// MappingJsonFactory jsonFactory = new MappingJsonFactory();
// InputStream @in = OpenStreamFromURL(url);
// try
// {
// JsonParser parser = jsonFactory.CreateParser(@in);
// try
// {
// JsonToken token = parser.NextToken();
// Type type;
// if (token == JsonToken.StartObject)
// {
// type = typeof(IDictionary);
// }
// else
// {
// if (token == JsonToken.StartArray)
// {
// type = typeof(IList);
// }
// else
// {
// type = typeof(string);
// }
// }
// return parser.ReadValueAs(type);
// }
// finally
// {
// parser.Close();
// }
// }
// finally
// {
// @in.Close();
// }
// }

// /// <summary>
// /// Opens an
// /// <see cref="Java.IO.InputStream">Java.IO.InputStream</see>
// /// for the given
// /// <see cref="Java.Net.URL">Java.Net.URL</see>
// /// , including support
// /// for http and https URLs that are requested using Content Negotiation with
// /// application/ld+json as the preferred content type.
// /// </summary>
// /// <param name="url">The URL identifying the source.</param>
// /// <returns>An InputStream containing the contents of the source.</returns>
// /// <exception cref="System.IO.IOException">If there was an error resolving the URL.</exception>
// public static InputStream OpenStreamFromURL(URL url)
// {
// string protocol = url.GetProtocol();
// if (!JsonLDNet.Shims.EqualsIgnoreCase(protocol, "http") && !JsonLDNet.Shims.EqualsIgnoreCase
// (protocol, "https"))
// {
// // Can't use the HTTP client for those!
// // Fallback to Java's built-in URL handler. No need for
// // Accept headers as it's likely to be file: or jar:
// return url.OpenStream();
// }
// IHttpUriRequest request = new HttpGet(url.ToExternalForm());
// // We prefer application/ld+json, but fallback to application/json
// // or whatever is available
// request.AddHeader("Accept", AcceptHeader);
// IHttpResponse response = GetHttpClient().Execute(request);
// int status = response.GetStatusLine().GetStatusCode();
// if (status != 200 && status != 203)
// {
// throw new IOException("Can't retrieve " + url + ", status code: " + status);
// }
// return response.GetEntity().GetContent();
// }

// public static IHttpClient GetHttpClient()
// {
// IHttpClient result = httpClient;
// if (result == null)
// {
// lock (typeof(JSONUtils))
// {
// result = httpClient;
// if (result == null)
// {
// // Uses Apache SystemDefaultHttpClient rather than
// // DefaultHttpClient, thus the normal proxy settings for the
// // JVM will be used
// DefaultHttpClient client = new SystemDefaultHttpClient();
// // Support compressed data
// // http://hc.apache.org/httpcomponents-client-ga/tutorial/html/httpagent.html#d5e1238
// client.AddRequestInterceptor(new RequestAcceptEncoding());
// client.AddResponseInterceptor(new ResponseContentEncoding());
// CacheConfig cacheConfig = new CacheConfig();
// cacheConfig.SetMaxObjectSize(1024 * 128);
// // 128 kB
// cacheConfig.SetMaxCacheEntries(1000);
// // and allow caching
// httpClient = new CachingHttpClient(client, cacheConfig);
// result = httpClient;
// }
// }
// }
// return result;
// }

// public static void SetHttpClient(IHttpClient nextHttpClient)
// {
// lock (typeof(JSONUtils))
// {
// httpClient = nextHttpClient;
// }
// }
}
}
58 changes: 33 additions & 25 deletions src/json-ld.net/Util/JSONUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,30 @@
using System;
using System.Collections;
using System.IO;
using System.Linq;
using JsonLD.Util;
using Newtonsoft.Json;
using System.Net;
using Newtonsoft.Json.Linq;
using System.Net.Http;
using System.Threading.Tasks;

namespace JsonLD.Util
{
/// <summary>A bunch of functions to make loading JSON easy</summary>
/// <author>tristan</author>
public class JSONUtils
{
const int MAX_REDIRECTS = 20;
static internal HttpClient _HttpClient = new HttpClient();

/// <summary>An HTTP Accept header that prefers JSONLD.</summary>
/// <remarks>An HTTP Accept header that prefers JSONLD.</remarks>
protected internal const string AcceptHeader = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1";

//private static readonly ObjectMapper JsonMapper = new ObjectMapper();

//private static readonly JsonFactory JsonFactory = new JsonFactory(JsonMapper);

static JSONUtils()
{
// Disable default Jackson behaviour to close
// InputStreams/Readers/OutputStreams/Writers
//JsonFactory.Disable(JsonGenerator.Feature.AutoCloseTarget);
// Disable string retention features that may work for most JSON where
// the field names are in limited supply, but does not work for JSON-LD
// where a wide range of URIs are used for subjects and predicates
//JsonFactory.Disable(JsonFactory.Feature.InternFieldNames);
//JsonFactory.Disable(JsonFactory.Feature.CanonicalizeFieldNames);
}

// private static volatile IHttpClient httpClient;

/// <exception cref="Com.Fasterxml.Jackson.Core.JsonParseException"></exception>
/// <exception cref="System.IO.IOException"></exception>
public static JToken FromString(string jsonString)
Expand Down Expand Up @@ -130,6 +121,11 @@ public static string ToString(JToken obj)
return sw.ToString();
}

public static JToken FromURL(Uri url)
{
return FromURLAsync(url).GetAwaiter().GetResult();
}

/// <summary>
/// Returns a Map, List, or String containing the contents of the JSON
/// resource resolved from the URL.
Expand All @@ -147,17 +143,29 @@ public static string ToString(JToken obj)
/// </exception>
/// <exception cref="System.IO.IOException">If there was an error resolving the resource.
/// </exception>
public static JToken FromURL(Uri url)
public static async Task<JToken> FromURLAsync(Uri url)
{
#if !PORTABLE && !IS_CORECLR
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
req.Accept = AcceptHeader;
WebResponse resp = req.GetResponse();
Stream stream = resp.GetResponseStream();
return FromInputStream(stream);
#else
throw new PlatformNotSupportedException();
#endif
HttpResponseMessage httpResponseMessage = null;
int redirects = 0;

// Manually follow redirects because .NET Core refuses to auto-follow HTTPS->HTTP redirects.
do
{
HttpRequestMessage httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, url);
httpRequestMessage.Headers.Add("Accept", AcceptHeader);
httpResponseMessage = await _HttpClient.SendAsync(httpRequestMessage);
if (httpResponseMessage.Headers.TryGetValues("Location", out var location))
{
url = new Uri(location.First());
}
} while (redirects++ < MAX_REDIRECTS && (int)httpResponseMessage.StatusCode >= 300 && (int)httpResponseMessage.StatusCode < 400);

if (redirects >= MAX_REDIRECTS || (int)httpResponseMessage.StatusCode >= 400)
{
throw new InvalidOperationException("Couldn't load JSON from URL");
}

return FromInputStream(await httpResponseMessage.Content.ReadAsStreamAsync());
}
}
}
Loading