Skip to content

Commit

Permalink
Merge pull request #370 from microsoft/features/publish-20231109-1
Browse files Browse the repository at this point in the history 
Microsoft CDM Release Version Update: 1.7.5
  • Loading branch information
@miroslavplese
miroslavplese authored Nov 9, 2023
2 parents c9c5fb0 + 6482139 commit 270631e
Show file tree
Hide file tree
Showing 45 changed files with 404 additions and 250 deletions.
4 changes: 2 additions & 2 deletions objectModel/CSharp/Directory.Build.props
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<Project>
<PropertyGroup>
<CdmObjectModelVersion>1.7.4</CdmObjectModelVersion>
<CdmObjectModelVersion>1.7.5</CdmObjectModelVersion>
<CdmStandardsVersion>2.8.0</CdmStandardsVersion>
<CdmPQModelsVersion>2.8.0</CdmPQModelsVersion>
<CdmTelemetryVersion>1.7.4-preview1</CdmTelemetryVersion>
<CdmTelemetryVersion>1.7.5-preview1</CdmTelemetryVersion>
<RunSettingsFilePath>$(MSBuildThisFileDirectory)\CodeCoverage.runsettings</RunSettingsFilePath>
</PropertyGroup>
</Project>
98 changes: 73 additions & 25 deletions objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Storage/ADLSAdapter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,17 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;

using Microsoft.CommonDataModel.ObjectModel.Enums;
using Microsoft.CommonDataModel.ObjectModel.Utilities;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Network;
using Microsoft.CommonDataModel.ObjectModel.Utilities.Storage;

using Microsoft.Identity.Client;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
Expand Down Expand Up @@ -326,7 +325,7 @@ public override async Task WriteAsync(string corpusPath, string data)
catch (Exception e)
{
await this.DeleteContentAtPath(corpusPath, url, e);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'", e);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'. Reason: {e.Message}", e);
}
}

Expand Down Expand Up @@ -474,32 +473,81 @@ public override async Task<IDictionary<string, CdmFileMetadata>> FetchAllFilesMe
continuationToken = null;
cdmResponse.ResponseHeaders.TryGetValue(HttpXmsContinuation, out continuationToken);

string json = await cdmResponse.Content.ReadAsStringAsync();
JObject jObject1 = JObject.Parse(json);
JArray jArray = JArray.FromObject(jObject1.GetValue("paths"));
var stream = await cdmResponse.Content.ReadAsStreamAsync();

foreach (JObject jObject in jArray.Children<JObject>())
using (var streamReader = new StreamReader(stream))
{
jObject.TryGetValue("isDirectory", StringComparison.OrdinalIgnoreCase, out JToken isDirectory);
if (isDirectory == null || !isDirectory.ToObject<bool>())
using (var jsonReader = new JsonTextReader(streamReader))
{
jObject.TryGetValue("name", StringComparison.OrdinalIgnoreCase, out JToken name);

string nameWithoutSubPath = this.unescapedRootSubPath.Length > 0 && name.ToString().StartsWith(this.unescapedRootSubPath) ?
name.ToString().Substring(this.unescapedRootSubPath.Length + 1) : name.ToString();

string path = this.FormatCorpusPath(nameWithoutSubPath);

jObject.TryGetValue("contentLength", StringComparison.OrdinalIgnoreCase, out JToken contentLengthToken);
long.TryParse(contentLengthToken.ToString(), out long contentLength);

result.Add(path, new CdmFileMetadata { FileSizeBytes = contentLength });

jObject.TryGetValue("lastModified", StringComparison.OrdinalIgnoreCase, out JToken lastModifiedTime);

if (this.IsCacheEnabled && DateTimeOffset.TryParse(lastModifiedTime.ToString(), out DateTimeOffset offset))
while (jsonReader.Read())
{
fileModifiedTimeCache[path] = offset;
bool isDirectory = false;
string path = "";
long contentLength = 0;
string lastModified = "";

while (jsonReader.TokenType != JsonToken.EndObject)
{
if (jsonReader.Value != null)
{
if (jsonReader.TokenType == JsonToken.PropertyName)
{
string value = jsonReader.Value.ToString();
if (value == "contentLength")
{
if (!jsonReader.Read())
{
break;
}
contentLength = (long)Convert.ToDouble(jsonReader.Value.ToString());
}
else if (value == "lastModified")
{
if (!jsonReader.Read())
{
break;
}
lastModified = jsonReader.Value.ToString();
}
else if (value == "name")
{
if (!jsonReader.Read())
{
break;
}
string name = jsonReader.Value.ToString();
string nameWithoutSubPath = !string.IsNullOrWhiteSpace(this.unescapedRootSubPath) && name.StartsWith(this.unescapedRootSubPath) ?
name.Substring(this.unescapedRootSubPath.Length + 1) : name;
path = this.FormatCorpusPath(nameWithoutSubPath);
}
else if (value == "isDirectory")
{
isDirectory = true;
}
}
}

if (!jsonReader.Read())
{
break;
}
}

// if path is not found, assume we have reached the end of the array
if (string.IsNullOrWhiteSpace(path))
{
break;
}

if (!isDirectory)
{
result.Add(path, new CdmFileMetadata { FileSizeBytes = contentLength });

if (this.IsCacheEnabled && DateTimeOffset.TryParse(lastModified, out DateTimeOffset offset))
{
fileModifiedTimeCache[path] = offset;
}
}
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions ...soft.CommonDataModel.ObjectModel.Tests/Microsoft.CommonDataModel.ObjectModel.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@

<ItemGroup>
<PackageReference Include="FluentAssertions" Version="5.7.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
<PackageReference Include="MSTest.TestAdapter" Version="1.4.0" />
<PackageReference Include="MSTest.TestFramework" Version="1.4.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="NUnit" Version="3.12.0" />
<PackageReference Include="NUnit.ConsoleRunner" Version="3.10.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
<PackageReference Include="Snapper.Nunit" Version="2.0.0-beta" />
<PackageReference Include="Snapper.Nunit" Version="2.4.0" />
</ItemGroup>

<ItemGroup>
Expand Down
4 changes: 2 additions & 2 deletions objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Utilities/EventListTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,8 @@ public void TestStorageManagerEvents()
public async Task TestScoping()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
// For scoping test we need to use log level Info
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Info, DummyCorrelationId);
// For scoping test we need to use log level Progress
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Progress, DummyCorrelationId);

// Test fetching a good object, this should result event recorder empty
_ = await corpus.FetchObjectAsync<CdmDocumentDefinition>("local:/default.manifest.cdm.json");
Expand Down
4 changes: 2 additions & 2 deletions objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmCorpusDefinition.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1213,7 +1213,7 @@ async Task LoadDocs(string docPath)

if (loadedDoc != null)
{
Logger.Info(this.Ctx, Tag, nameof(LoadImportsAsync), loadedDoc.AtCorpusPath, $"resolved import for '{loadedDoc.Name}'");
Logger.Debug(this.Ctx, Tag, nameof(LoadImportsAsync), loadedDoc.AtCorpusPath, $"resolved import for '{loadedDoc.Name}'");
}
else
{
Expand Down Expand Up @@ -2201,7 +2201,7 @@ private void ResolveReferencesTraitsArguments(
}
if (found > 0 && found == resolved)
{
Logger.Info(ctx, Tag, nameof(ResolveReferencesTraitsArguments), CurrentDoc.AtCorpusPath, $"found and resolved '{found}' required parameters of trait '{rt.TraitName}' on '{obj.FetchObjectDefinition<CdmObjectDefinition>(resOpt)?.GetName()}'");
Logger.Debug(ctx, Tag, nameof(ResolveReferencesTraitsArguments), CurrentDoc.AtCorpusPath, $"found and resolved '{found}' required parameters of trait '{rt.TraitName}' on '{obj.FetchObjectDefinition<CdmObjectDefinition>(resOpt)?.GetName()}'");
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionDefinition.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public CdmDataPartitionDefinition(CdmCorpusContext ctx, string name) : base(ctx)
/// <summary>
/// Gets whether the data partition is incremental.
/// </summary>
public bool IsIncremental { get => this.TraitToPropertyMap.FetchPropertyValue(nameof(IsIncremental).Substring(0, 1).ToLower() + nameof(IsIncremental).Substring(1)); }
public bool IsIncremental { get => this.TraitToPropertyMap.FetchPropertyValue("isIncremental"); }

/// <inheritdoc />
[Obsolete]
Expand Down
2 changes: 1 addition & 1 deletion ...del/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public CdmDataPartitionPatternDefinition(CdmCorpusContext ctx, string name) : ba
/// <summary>
/// Gets or sets whether the data partition pattern is incremental.
/// </summary>
public bool IsIncremental { get => this.TraitToPropertyMap.FetchPropertyValue(nameof(IsIncremental).Substring(0, 1).ToLower() + nameof(IsIncremental).Substring(1)); }
public bool IsIncremental { get => this.TraitToPropertyMap.FetchPropertyValue("isIncremental"); }

/// <inheritdoc />
[Obsolete]
Expand Down
8 changes: 4 additions & 4 deletions objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDocumentDefinition.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ internal void CheckIntegrity()
obj.Ctx = this.Ctx;
}

Logger.Info(this.Ctx, Tag, nameof(CheckIntegrity), obj.AtCorpusPath, $"checked '{obj.AtCorpusPath}'");
Logger.Debug(this.Ctx, Tag, nameof(CheckIntegrity), obj.AtCorpusPath, $"checked '{obj.AtCorpusPath}'");
}

this.IsValid = errorCount == 0;
Expand Down Expand Up @@ -241,7 +241,7 @@ internal void DeclareObjectDefinitions()
this.InternalDeclarations.TryAdd(objPath, obj);
this.Ctx.Corpus.RegisterSymbol(objPath, this);

Logger.Info(this.Ctx, Tag, nameof(DeclareObjectDefinitions), corpusPath, $"declared '{objPath}'");
Logger.Debug(this.Ctx, Tag, nameof(DeclareObjectDefinitions), corpusPath, $"declared '{objPath}'");
}
break;
}
Expand Down Expand Up @@ -324,7 +324,7 @@ internal void ResolveObjectDefinitions(ResolveOptions resOpt)
}
else
{
Logger.Info(ctx, Tag, nameof(ResolveObjectDefinitions), this.AtCorpusPath, $"resolved '{reff.NamedReference}'");
Logger.Debug(ctx, Tag, nameof(ResolveObjectDefinitions), this.AtCorpusPath, $"resolved '{reff.NamedReference}'");
}
}
break;
Expand Down Expand Up @@ -396,7 +396,7 @@ internal bool LocalizeCorpusPaths(CdmFolderDefinition newFolder)
bool allWentWell = true;

// shout into the void
Logger.Info((ResolveContext)this.Ctx, Tag, nameof(LocalizeCorpusPaths), newFolder.AtCorpusPath, $"Localizing corpus paths in document '{this.Name}'");
Logger.Debug((ResolveContext)this.Ctx, Tag, nameof(LocalizeCorpusPaths), newFolder.AtCorpusPath, $"Localizing corpus paths in document '{this.Name}'");

// find anything in the document that is a corpus path
this.Visit("", new VisitCallback
Expand Down
2 changes: 1 addition & 1 deletion objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmParameterDefinition.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ internal dynamic ConstTypeCheck(ResolveOptions resOpt, CdmDocumentDefinition wrt
}
else
{
Logger.Info(ctx, Tag, nameof(ConstTypeCheck), wrtDoc.AtCorpusPath, $"resolved '{foundDesc}'");
Logger.Debug(ctx, Tag, nameof(ConstTypeCheck), wrtDoc.AtCorpusPath, $"resolved '{foundDesc}'");
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmStatusLevel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ namespace Microsoft.CommonDataModel.ObjectModel.Cdm
{
public enum CdmStatusLevel
{
Info,
Progress,
Info,
Warning,
Error,
None
Expand Down
2 changes: 1 addition & 1 deletion objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@
<value>A cycle was detected on this definition. An object cannot inherit from itself.</value>
</data>
<data name="ErrFetchingFileMetadataNull" xml:space="preserve">
<value>Could not fetch file metadata. The FetchAllFilesMetadata method of the '{0}' adapter may be misconfigured.</value>
<value>Could not fetch file metadata from root location, the '{0}' adapter may be misconfigured.</value>
</data>
<data name="ErrStorageCdmStandardsMissing" xml:space="preserve">
<value>Couldn't find assembly '{0}', please install the package, and add it as dependency of the project. Exception: {1}.</value>
Expand Down
35 changes: 29 additions & 6 deletions objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Storage/StorageManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;

public class StorageManager
Expand Down Expand Up @@ -390,38 +391,60 @@ public string CreateAbsoluteCorpusPath(string objectPath, CdmObject obj = null)
return null;
}

if (!string.IsNullOrEmpty(prefix) && prefix[prefix.Length - 1] != '/')
int maxPrefixLength = (!string.IsNullOrWhiteSpace(prefix) ? prefix.Length : 0) + 1;
StringBuilder prefixBuilder = new StringBuilder(maxPrefixLength);

if (!string.IsNullOrEmpty(prefix))
{
Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(CreateAbsoluteCorpusPath), null, CdmLogCode.WarnStorageExpectedPathPrefix, prefix);
prefix += "/";
prefixBuilder.Append(prefix);
if (prefix[prefix.Length - 1] != '/')
{
Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(CreateAbsoluteCorpusPath), null, CdmLogCode.WarnStorageExpectedPathPrefix, prefix);
prefixBuilder.Append('/');
}
}

StringBuilder fullPathBuilder = new StringBuilder();
// check if this is a relative path
if (!string.IsNullOrWhiteSpace(newObjectPath) && !newObjectPath.StartsWith("/"))
{
if (obj == null)
{
// relative path and no other info given, assume default and root
prefix = "/";
prefixBuilder.Clear();
prefixBuilder.Append('/');

}
if (!string.IsNullOrWhiteSpace(nameSpace) && nameSpace != namespaceFromObj)
{
Logger.Error(this.Ctx, Tag, nameof(CreateAbsoluteCorpusPath), null, CdmLogCode.ErrStorageNamespaceMismatch, nameSpace, newObjectPath, prefix, namespaceFromObj);
return null;
}
newObjectPath = prefix + newObjectPath;

finalNamespace = string.IsNullOrWhiteSpace(namespaceFromObj) ?
(string.IsNullOrWhiteSpace(nameSpace) ? this.DefaultNamespace : nameSpace) : namespaceFromObj;

if (!string.IsNullOrWhiteSpace(finalNamespace))
{
fullPathBuilder.Append(finalNamespace);
fullPathBuilder.Append(':');
}
fullPathBuilder.Append(prefixBuilder);
}
else
{
finalNamespace = string.IsNullOrWhiteSpace(nameSpace) ?
(string.IsNullOrWhiteSpace(namespaceFromObj) ? this.DefaultNamespace : namespaceFromObj) : nameSpace;
if (!string.IsNullOrWhiteSpace(finalNamespace))
{
fullPathBuilder.Append(finalNamespace);
fullPathBuilder.Append(':');
}
}

return (!string.IsNullOrWhiteSpace(finalNamespace) ? $"{finalNamespace}:" : "") + newObjectPath;
fullPathBuilder.Append(newObjectPath);

return fullPathBuilder.ToString();
}
}

Expand Down
14 changes: 13 additions & 1 deletion objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/Logging/EventList.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,19 @@ internal void Disable()
{
if (IsRecording)
{
base.Add(theEvent);
bool lockTaken = false;
try
{
spinLock.Enter(ref lockTaken);
base.Add(theEvent);
}
finally
{
if (lockTaken)
{
spinLock.Exit();
}
}
}
}
}
Expand Down
Loading

0 comments on commit 270631e

Please sign in to comment.