Skip to content

Commit 3ae4dbc

Browse files
tombuildsstufftombuildsstuff
authored
Merge pull request #354 from hashicorp/f/workaround-swagger-data-quality-issues
importer: working around swagger issues
2 parents 95d7145 + 5d90900 commit 3ae4dbc

26 files changed

+614
-143
lines changed

‎data/Pandora.Data/Transformers/Helpers.cs‎ renamed to ‎data/Pandora.Data/Helpers/Type.cs‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,58 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Reflection;
4+
using Pandora.Data.Transformers;
45
using Pandora.Definitions.CustomTypes;
56

6-
namespace Pandora.Data.Transformers
7+
namespace Pandora.Data.Helpers
78
{
8-
internal static class Helpers
9+
public static class TypeExtensions
910
{
10-
public static bool IsNativeType(Type input)
11+
/// <summary>
12+
/// GetActualType returns the Actual Type if this is a Csv/Dictionary/List - or input otherwise
13+
/// for example `List<Model>` will return the Type `Model`. This'll be null if a built-in, custom
14+
/// or Enum type is provided.
15+
/// </summary>
16+
internal static Type? GetActualType(this Type input, bool allowEnums)
17+
{
18+
if (!allowEnums && input.IsEnum)
19+
{
20+
return null;
21+
}
22+
23+
if (input.IsNativeType() || input.IsPandoraCustomType())
24+
{
25+
return null;
26+
}
27+
28+
// if it's nullable pull that out
29+
if (Nullable.GetUnderlyingType(input) != null)
30+
{
31+
var genericArgs = input.GetGenericArguments();
32+
var element = genericArgs[0];
33+
return GetActualType(element, allowEnums);
34+
}
35+
36+
if (input.IsAGenericCsv())
37+
{
38+
var valueType = input.GenericCsvElement();
39+
return GetActualType(valueType, allowEnums);
40+
}
41+
if (input.IsAGenericDictionary())
42+
{
43+
var valueType = input.GenericDictionaryValueElement();
44+
return GetActualType(valueType, allowEnums);
45+
}
46+
if (input.IsAGenericList())
47+
{
48+
var valueType = input.GenericListElement();
49+
return GetActualType(valueType, allowEnums);
50+
}
51+
52+
return input;
53+
}
54+
55+
public static bool IsNativeType(this Type input)
1156
{
1257
var nativeTypes = new List<Type>
1358
{
@@ -21,7 +66,7 @@ public static bool IsNativeType(Type input)
2166
return nativeTypes.Contains(input);
2267
}
2368

24-
public static bool IsPandoraCustomType(Type input)
69+
public static bool IsPandoraCustomType(this Type input)
2570
{
2671
var customTypes = new List<Type>
2772
{

‎data/Pandora.Data/ServiceDefinitionsTests.cs‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Pandora.Definitions.Interfaces;
99
using Pandora.Definitions.ResourceManager;
1010
using Pandora.Definitions.TestData;
11+
using ServiceDefinition = Pandora.Data.Models.ServiceDefinition;
1112

1213
namespace Pandora.Data
1314
{
@@ -48,13 +49,35 @@ private static void ValidateAssemblyContainingServiceDefinitions(ServicesDefinit
4849
};
4950
try
5051
{
51-
new ServiceReferencesRepository(wrapper);
52+
var repo = new ServiceReferencesRepository(wrapper);
53+
// first try mapping all of the resource manager calls
54+
TryMapping(repo.GetAll(true));
55+
// then try mapping all of the non-resource manager calls
56+
TryMapping(repo.GetAll(false));
5257
}
5358
catch (Exception ex)
5459
{
5560
throw new Exception($"Service Definition {serviceDefinition.Name} should validate but failed due to: {ex}");
5661
}
5762
}
5863
}
64+
65+
private static void TryMapping(IEnumerable<ServiceDefinition> services)
66+
{
67+
// since this is lazy-loaded we need to explicitly ToList on each of these
68+
foreach (var service in services)
69+
{
70+
Console.WriteLine($"Validating that Service {service.Name} maps..");
71+
foreach (var version in service.Versions)
72+
{
73+
Console.WriteLine($"Validating that Service {service.Name} Version {version.Version} maps..");
74+
var apis = version.Apis.ToList();
75+
if (apis.Count == 0)
76+
{
77+
throw new NotSupportedException($"Service {service.Name} / Version {version.Version} has no API's");
78+
}
79+
}
80+
}
81+
}
5982
}
6083
}

‎data/Pandora.Data/Transformers/APIDefinition.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,18 +62,18 @@ private static List<ConstantDefinition> ConstantsForOperation(ApiOperation input
6262

6363
if (input.RequestObject() != null)
6464
{
65-
definitions.AddRange(Constant.FromObject(input.RequestObject()!));
65+
definitions.AddRange(Constant.WithinObject(input.RequestObject()!));
6666
}
6767

6868
if (input.ResponseObject() != null)
6969
{
70-
definitions.AddRange(Constant.FromObject(input.ResponseObject()!));
70+
definitions.AddRange(Constant.WithinObject(input.ResponseObject()!));
7171
}
7272

7373
// pull out any constants which are referenced against the Options block
7474
if (input.OptionsObject() != null)
7575
{
76-
definitions.AddRange(Constant.FromObject(input.OptionsObject()!));
76+
definitions.AddRange(Constant.WithinObject(input.OptionsObject()!));
7777
}
7878

7979
return definitions.Distinct(new ConstantComparer()).ToList();

‎data/Pandora.Data/Transformers/Constant.cs‎

Lines changed: 67 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,104 @@
11
using System;
22
using System.Collections.Generic;
3-
using System.ComponentModel;
43
using System.Linq;
54
using System.Reflection;
5+
using System.Text.RegularExpressions;
6+
using NUnit.Framework;
67
using Pandora.Data.Helpers;
78
using Pandora.Data.Models;
89
using Pandora.Definitions.Attributes;
10+
using DescriptionAttribute = System.ComponentModel.DescriptionAttribute;
911

1012
namespace Pandora.Data.Transformers
1113
{
1214
public static class Constant
1315
{
14-
public static List<ConstantDefinition> FromObject(Type input)
16+
public static List<ConstantDefinition> WithinObject(Type input)
1517
{
1618
try
1719
{
18-
if (!input.IsClass)
20+
var actualType = input.GetActualType(true);
21+
if (actualType == null)
1922
{
20-
throw new NotSupportedException("expected a class");
23+
return new List<ConstantDefinition>();
2124
}
2225

23-
var constantDefinitions = new List<ConstantDefinition>();
24-
foreach (var property in input.GetProperties())
26+
if (actualType.IsEnum)
2527
{
26-
var propertyType = property.PropertyType;
27-
if (propertyType.IsGenericType && (propertyType.GetGenericTypeDefinition() == typeof(List<>) || propertyType.GetGenericTypeDefinition() == typeof(Nullable<>)))
28+
return new List<ConstantDefinition>
2829
{
29-
propertyType = propertyType.GetGenericArguments()[0];
30-
}
31-
32-
if (propertyType.FullName == input.FullName)
33-
{
34-
continue;
35-
}
30+
FromEnum(input)
31+
};
32+
}
3633

37-
if (propertyType.IsEnum)
38-
{
39-
var definition = FromEnum(propertyType);
40-
constantDefinitions.Add(definition);
41-
continue;
42-
}
34+
return WithinObject(input, new List<Type>());
35+
}
36+
catch (Exception ex)
37+
{
38+
throw new Exception($"Mapping Constants WithinObject {input.FullName}", ex);
39+
}
40+
}
4341

44-
if (Helpers.IsNativeType(propertyType) || Helpers.IsPandoraCustomType(propertyType) || !propertyType.IsClass)
45-
{
46-
continue;
47-
}
42+
private static List<ConstantDefinition> WithinObject(Type input, List<Type> knownTypes)
43+
{
44+
if (!input.IsClass)
45+
{
46+
throw new NotSupportedException("expected a class");
47+
}
4848

49-
var innerConstants = FromObject(propertyType);
50-
constantDefinitions.AddRange(innerConstants);
51-
}
49+
var found = new List<ConstantDefinition>();
50+
var types = Model.FindTypesWithinType(input, knownTypes);
51+
var allTypes = new List<Type>();
52+
allTypes.AddRange(knownTypes);
53+
allTypes.AddRange(types);
5254

53-
return constantDefinitions.Distinct(new ConstantComparer()).ToList();
55+
foreach (var type in types)
56+
{
57+
var constantsWithinType = UsedByType(type);
58+
found.AddRange(constantsWithinType);
5459
}
55-
catch (Exception ex)
60+
61+
return found.Distinct(new ConstantComparer()).ToList();
62+
}
63+
64+
private static List<ConstantDefinition> UsedByType(Type input)
65+
{
66+
var constants = new List<ConstantDefinition>();
67+
68+
foreach (var property in input.GetProperties())
5669
{
57-
throw new Exception($"Mapping Constant FromObject {input.FullName}", ex);
70+
var actualType = property.PropertyType.GetActualType(true);
71+
if (actualType == null)
72+
{
73+
continue;
74+
}
75+
76+
if (!actualType.IsEnum)
77+
{
78+
continue;
79+
}
80+
81+
var constant = FromEnum(property.PropertyType);
82+
constants.Add(constant);
5883
}
84+
85+
return constants;
5986
}
6087

6188
public static ConstantDefinition FromEnum(Type input)
6289
{
6390
try
6491
{
65-
if (!input.IsEnum)
92+
var actualType = input.GetActualType(true);
93+
if (actualType == null || !actualType.IsEnum)
6694
{
6795
throw new NotSupportedException("expected an enum");
6896
}
6997

70-
var name = input.Name.TrimSuffix("Constant");
71-
var caseInsensitive = IsCaseInsensitive(input);
72-
var variableType = TypeForEnum(input);
73-
var values = ValuesForEnum(input);
98+
var name = actualType.Name.TrimSuffix("Constant");
99+
var caseInsensitive = IsCaseInsensitive(actualType);
100+
var variableType = TypeForEnum(actualType);
101+
var values = ValuesForEnum(actualType);
74102

75103
// this enum has to have a 'description' tag for each of the values
76104
return new ConstantDefinition
@@ -124,7 +152,9 @@ private static Dictionary<string, string> ValuesForEnum(Type input)
124152
var values = input.GetEnumValues();
125153
foreach (var val in values)
126154
{
127-
var enumValue = input.GetMember(val.ToString()).First();
155+
// Some Enum's define `Equals` as a value, however all objects implement Equals too which apparently conflicts here
156+
// so filter to the member on this specific Enum
157+
var enumValue = input.GetMember(val.ToString()).First(m => m.DeclaringType == input);
128158
var description = enumValue.GetCustomAttribute<DescriptionAttribute>();
129159
if (description == null)
130160
{

0 commit comments

Comments
 (0)