From 685c66ba61cb2bb9742a7ec373a056167ee7defe Mon Sep 17 00:00:00 2001 From: Shaw Young Date: Thu, 28 Nov 2024 16:17:23 +0000 Subject: [PATCH] AVRO-4094: [C#] Updating mapped namespaces referenced in types --- .../csharp/src/apache/main/CodeGen/CodeGen.cs | 105 ++++++++++++++++++ .../apache/test/AvroGen/AvroGenSchemaTests.cs | 71 ++++++++++++ 2 files changed, 176 insertions(+) diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index 73b95852d7b..75614711784 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -25,7 +25,10 @@ using System.Reflection; using System.Text; using System.Text.RegularExpressions; +using Avro.IO.Parsing; using Microsoft.CSharp; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace Avro { @@ -113,6 +116,8 @@ public virtual void AddProtocol(string protocolText, IEnumerable + /// Replace namespaces in a parsed JSON schema object for all "type" fields. + /// + /// The JSON schema as a string. + /// The mapping of old namespaces to new namespaces. + /// The updated JSON schema as a string. + private static string ReplaceMappedNamespacesInSchemaTypes(string schemaJson, IEnumerable> namespaceMapping) + { + if (string.IsNullOrWhiteSpace(schemaJson) || namespaceMapping == null) + return schemaJson; + + var schemaToken = JToken.Parse(schemaJson); + + UpdateNamespacesInJToken(schemaToken, namespaceMapping); + + return schemaToken.ToString(Formatting.Indented); + } + + /// + /// Recursively navigates and updates "type" fields in a JToken. + /// + /// The current JToken to process. + /// The mapping of old namespaces to new namespaces. + private static void UpdateNamespacesInJToken(JToken token, IEnumerable> namespaceMapping) + { + if (token is JObject obj) + { + if (obj.ContainsKey("type")) + { + var typeToken = obj["type"]; + if (typeToken is JValue) // Single type + { + string type = typeToken.ToString(); + obj["type"] = ReplaceNamespace(type, namespaceMapping); + } + else if (typeToken is JArray typeArray) // Array of types + { + for (int i = 0; i < typeArray.Count; i++) + { + var arrayItem = typeArray[i]; + if (arrayItem is JValue) // Simple type + { + string type = arrayItem.ToString(); + typeArray[i] = ReplaceNamespace(type, namespaceMapping); + } + else if (arrayItem is JObject nestedObj) // Nested object + { + UpdateNamespacesInJToken(nestedObj, namespaceMapping); + } + } + } + else if (typeToken is JObject nestedTypeObj) // Complex type + { + UpdateNamespacesInJToken(nestedTypeObj, namespaceMapping); + } + } + + // Recurse into all properties of the object + foreach (var property in obj.Properties()) + { + UpdateNamespacesInJToken(property.Value, namespaceMapping); + } + } + else if (token is JArray array) + { + // Recurse into all elements of the array + foreach (var element in array) + { + UpdateNamespacesInJToken(element, namespaceMapping); + } + } } + + /// + /// Replace a namespace in a string based on the provided mapping. + /// + /// The original namespace string. + /// The mapping of old namespaces to new namespaces. + /// The updated namespace string. + private static string ReplaceNamespace(string originalNamespace, IEnumerable> namespaceMapping) + { + foreach (var mapping in namespaceMapping) + { + if (originalNamespace == mapping.Key) + { + return mapping.Value; + } + else if (originalNamespace.StartsWith($"{mapping.Key}.")) + { + return $"{mapping.Value}.{originalNamespace.Substring(mapping.Key.Length + 1)}"; + } + } + return originalNamespace; + } + +} } diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 807acbda92a..34d610869af 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -309,6 +309,46 @@ class AvroGenSchemaTests ] }"; + private const string _fullyQualifiedTypeReferences = @" +[ + { + ""namespace"": ""org.apache.avro.codegentest.testdata.common"", + ""type"": ""enum"", + ""name"": ""Planet"", + ""doc"" : ""Test mapping of types in other namespaces post-map"", + ""symbols"": [ + ""Mercury"", + ""Venus"", + ""Earth"", + ""Mars"", + ""Jupiter"", + ""Saturn"", + ""Neptune"", + ""Uranus"" + ], + }, + { + ""namespace"": ""org.apache.avro.codegentest.testdata.users"", + ""type"": ""record"", + ""name"": ""User"", + ""doc"" : ""Test mapping of types in other namespaces post-map"", + ""fields"": [ + { + ""name"": ""homePlanet"", + ""type"": ""org.apache.avro.codegentest.testdata.common.Planet"" + }, + { + ""name"": ""favouritePlanet"", + ""type"": [ + ""null"", + ""org.apache.avro.codegentest.testdata.common.Planet"" + ], + ""default"": null + }] + }, + +]"; + private Assembly TestSchema( string schema, IEnumerable typeNamesToCheck = null, @@ -606,6 +646,37 @@ public void GenerateSchemaWithNamespaceMapping( AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } + [TestCase(_fullyQualifiedTypeReferences, + new string[] + { + "org.apache.avro.codegentest.testdata.common:Test.Common", + "org.apache.avro.codegentest.testdata.users:Test.Users" + }, + new string[] + { + "Test.Common.Planet", + "Test.Users.User", + }, + new string[] + { + "Test/Common/Planet.cs", + "Test/Users/User.cs" + })] + public void GenerateSchemaWithMultipleNamespaceMapping( + string schema, + IEnumerable namespaceMappings, + IEnumerable typeNamesToCheck, + IEnumerable generatedFilesToCheck) + { + var namespaceMappingsDict = new Dictionary(); + + foreach(var mapping in namespaceMappings) + { + namespaceMappingsDict.Add(mapping.Split(':')[0], mapping.Split(':')[1]); + } + AvroGenHelper.TestSchema(schema, typeNamesToCheck, namespaceMappingsDict, generatedFilesToCheck); + } + [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] public void NotSupportedSchema(string schema, Type expectedException)