Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AVRO-4094: [C#] Updating mapped namespaces referenced in types #3247

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

AVRO-4094: [C#] Updating mapped namespaces referenced in types #3247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions lang/csharp/src/apache/main/CodeGen/CodeGen.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using Avro.IO.Parsing;
using Microsoft.CSharp;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace Avro
{
Expand Down Expand Up @@ -113,6 +116,8 @@ public virtual void AddProtocol(string protocolText, IEnumerable<KeyValuePair<st
{
// Map namespaces
protocolText = ReplaceMappedNamespacesInSchema(protocolText, namespaceMapping);
protocolText = ReplaceMappedNamespacesInSchemaTypes(protocolText, namespaceMapping);

Protocol protocol = Protocol.Parse(protocolText);
Protocols.Add(protocol);
}
Expand All @@ -135,6 +140,8 @@ public virtual void AddSchema(string schemaText, IEnumerable<KeyValuePair<string
{
// Map namespaces
schemaText = ReplaceMappedNamespacesInSchema(schemaText, namespaceMapping);
schemaText = ReplaceMappedNamespacesInSchemaTypes(schemaText, namespaceMapping);

Schema schema = Schema.Parse(schemaText);
Schemas.Add(schema);
}
Expand Down Expand Up @@ -1266,5 +1273,103 @@ private static string ReplaceMappedNamespacesInSchema(string input, IEnumerable<
return $@"""namespace""{m.Groups[1].Value}:{m.Groups[2].Value}""{ns}""";
});
}



/// <summary>
/// Replace namespaces in a parsed JSON schema object for all "type" fields.
/// </summary>
/// <param name="schemaJson">The JSON schema as a string.</param>
/// <param name="namespaceMapping">The mapping of old namespaces to new namespaces.</param>
/// <returns>The updated JSON schema as a string.</returns>
private static string ReplaceMappedNamespacesInSchemaTypes(string schemaJson, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
{
if (string.IsNullOrWhiteSpace(schemaJson) || namespaceMapping == null)
return schemaJson;

var schemaToken = JToken.Parse(schemaJson);

UpdateNamespacesInJToken(schemaToken, namespaceMapping);

return schemaToken.ToString(Formatting.Indented);
}

/// <summary>
/// Recursively navigates and updates "type" fields in a JToken.
/// </summary>
/// <param name="token">The current JToken to process.</param>
/// <param name="namespaceMapping">The mapping of old namespaces to new namespaces.</param>
private static void UpdateNamespacesInJToken(JToken token, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
{
if (token is JObject obj)
{
if (obj.ContainsKey("type"))
{
var typeToken = obj["type"];
if (typeToken is JValue) // Single type
{
string type = typeToken.ToString();
obj["type"] = ReplaceNamespace(type, namespaceMapping);
}
else if (typeToken is JArray typeArray) // Array of types
{
for (int i = 0; i < typeArray.Count; i++)
{
var arrayItem = typeArray[i];
if (arrayItem is JValue) // Simple type
{
string type = arrayItem.ToString();
typeArray[i] = ReplaceNamespace(type, namespaceMapping);
}
else if (arrayItem is JObject nestedObj) // Nested object
{
UpdateNamespacesInJToken(nestedObj, namespaceMapping);
}
}
}
else if (typeToken is JObject nestedTypeObj) // Complex type
{
UpdateNamespacesInJToken(nestedTypeObj, namespaceMapping);
}
}

// Recurse into all properties of the object
foreach (var property in obj.Properties())
{
UpdateNamespacesInJToken(property.Value, namespaceMapping);
}
}
else if (token is JArray array)
{
// Recurse into all elements of the array
foreach (var element in array)
{
UpdateNamespacesInJToken(element, namespaceMapping);
}
}
}

/// <summary>
/// Replace a namespace in a string based on the provided mapping.
/// </summary>
/// <param name="originalNamespace">The original namespace string.</param>
/// <param name="namespaceMapping">The mapping of old namespaces to new namespaces.</param>
/// <returns>The updated namespace string.</returns>
private static string ReplaceNamespace(string originalNamespace, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
{
foreach (var mapping in namespaceMapping)
{
if (originalNamespace == mapping.Key)
{
return mapping.Value;
}
else if (originalNamespace.StartsWith($"{mapping.Key}."))
{
return $"{mapping.Value}.{originalNamespace.Substring(mapping.Key.Length + 1)}";
}
}
return originalNamespace;
}

}
}
71 changes: 71 additions & 0 deletions lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,46 @@ class AvroGenSchemaTests
]
}";

private const string _fullyQualifiedTypeReferences = @"
[
{
""namespace"": ""org.apache.avro.codegentest.testdata.common"",
""type"": ""enum"",
""name"": ""Planet"",
""doc"" : ""Test mapping of types in other namespaces post-map"",
""symbols"": [
""Mercury"",
""Venus"",
""Earth"",
""Mars"",
""Jupiter"",
""Saturn"",
""Neptune"",
""Uranus""
],
},
{
""namespace"": ""org.apache.avro.codegentest.testdata.users"",
""type"": ""record"",
""name"": ""User"",
""doc"" : ""Test mapping of types in other namespaces post-map"",
""fields"": [
{
""name"": ""homePlanet"",
""type"": ""org.apache.avro.codegentest.testdata.common.Planet""
},
{
""name"": ""favouritePlanet"",
""type"": [
""null"",
""org.apache.avro.codegentest.testdata.common.Planet""
],
""default"": null
}]
},

]";

private Assembly TestSchema(
string schema,
IEnumerable<string> typeNamesToCheck = null,
Expand Down Expand Up @@ -606,6 +646,37 @@ public void GenerateSchemaWithNamespaceMapping(
AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary<string, string> { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck);
}

[TestCase(_fullyQualifiedTypeReferences,
new string[]
{
"org.apache.avro.codegentest.testdata.common:Test.Common",
"org.apache.avro.codegentest.testdata.users:Test.Users"
},
new string[]
{
"Test.Common.Planet",
"Test.Users.User",
},
new string[]
{
"Test/Common/Planet.cs",
"Test/Users/User.cs"
})]
public void GenerateSchemaWithMultipleNamespaceMapping(
string schema,
IEnumerable<string> namespaceMappings,
IEnumerable<string> typeNamesToCheck,
IEnumerable<string> generatedFilesToCheck)
{
var namespaceMappingsDict = new Dictionary<string, string>();

foreach(var mapping in namespaceMappings)
{
namespaceMappingsDict.Add(mapping.Split(':')[0], mapping.Split(':')[1]);
}
AvroGenHelper.TestSchema(schema, typeNamesToCheck, namespaceMappingsDict, generatedFilesToCheck);
}

[TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))]
[TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))]
public void NotSupportedSchema(string schema, Type expectedException)
Expand Down