From 752987a6f0ce2456d627c688f7db820856e36620 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 14 Dec 2021 09:26:00 +0300 Subject: [PATCH 1/7] Add example manifest generation pipeline Signed-off-by: Alper Rifat Ulucinar (cherry picked from commit af0dc440500da20e163c666f6d5333866d8bb573) --- go.mod | 3 +- go.sum | 15 ++ pkg/config/provider.go | 28 ++++ pkg/pipeline/crd.go | 3 + pkg/pipeline/example.go | 297 ++++++++++++++++++++++++++++++++++++ pkg/pipeline/run.go | 17 +++ pkg/types/builder.go | 27 +++- pkg/types/reference.go | 4 +- pkg/types/reference_test.go | 2 +- 9 files changed, 385 insertions(+), 11 deletions(-) create mode 100644 pkg/pipeline/example.go diff --git a/go.mod b/go.mod index 281e9221..7688ce23 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/crossplane/crossplane-runtime v0.16.0 + github.com/crossplane/terrajet v0.4.2 github.com/fatih/camelcase v1.0.0 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 @@ -21,6 +22,7 @@ require ( k8s.io/apimachinery v0.24.0 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/controller-runtime v0.12.1 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -127,5 +129,4 @@ require ( k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 99ae8047..5a9e5be1 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.61.0/go.mod h1:XukKJg4Y7QsUu0Hxg3qQKUWR4VuWivmyMK2+rUyxAqw= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= @@ -86,6 +87,7 @@ github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f/go.mod h1:k8feO4+ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/apparentlymart/go-cidr v1.0.1/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= @@ -105,6 +107,7 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= +github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.37.0 h1:GzFnhOIsrGyQ69s7VgqtrG2BG8v7X7vwB3Xpbd/DBBk= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -160,8 +163,11 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/crossplane/crossplane-runtime v0.15.1-0.20220106140106-428b7c390375/go.mod h1:CH05KIlxoEHEE4aLpUhPuvF+9qXsN6/H6YIDnUEjlDs= github.com/crossplane/crossplane-runtime v0.16.0 h1:NstJdHeK3C+u3By0vQjOG1Y6+v53JYOy00IgCL9GHAw= github.com/crossplane/crossplane-runtime v0.16.0/go.mod h1:IPT3HTsovwmbw3i+SdsOyaC3r3b7TW+otBMmZsHLnSU= +github.com/crossplane/terrajet v0.4.2 h1:tWgjjPDwzwlurRYgaujsXL7NHhehBy1C8PZloTUQ65U= +github.com/crossplane/terrajet v0.4.2/go.mod h1:oynNx4au8y/tiSb1OanWSMmUf55Cz/O4f1HxH+liI2A= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -375,6 +381,7 @@ github.com/hashicorp/go-getter v1.5.3/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXj github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= @@ -386,6 +393,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0= +github.com/hashicorp/go-plugin v1.4.1/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-plugin v1.4.4/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -424,6 +432,7 @@ github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90= +github.com/hashicorp/hcl/v2 v2.3.0/go.mod h1:d+FwDBbOLvpAM3Z6J7gPj/VoAGkNe/gm352ZhjJ/Zv8= github.com/hashicorp/hcl/v2 v2.8.2/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY= github.com/hashicorp/hcl/v2 v2.12.0 h1:PsYxySWpMD4KPaoJLnsHwtK5Qptvj/4Q6s0t4sUxZf4= github.com/hashicorp/hcl/v2 v2.12.0/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= @@ -435,17 +444,21 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/terraform-config-inspect v0.0.0-20191212124732-c6ae6269b9d7 h1:Pc5TCv9mbxFN6UVX0LH6CpQrdTM5YjbVI2w15237Pjk= github.com/hashicorp/terraform-config-inspect v0.0.0-20191212124732-c6ae6269b9d7/go.mod h1:p+ivJws3dpqbp1iP84+npOyAmTTOLMgCzrXd3GSdn/A= github.com/hashicorp/terraform-exec v0.13.3/go.mod h1:SSg6lbUsVB3DmFyCPjBPklqf6EYGX0TlQ6QTxOlikDU= +github.com/hashicorp/terraform-exec v0.14.0/go.mod h1:qrAASDq28KZiMPDnQ02sFS9udcqEkRly002EA2izXTA= github.com/hashicorp/terraform-exec v0.16.1/go.mod h1:aj0lVshy8l+MHhFNoijNHtqTJQI3Xlowv5EOsEaGO7M= github.com/hashicorp/terraform-json v0.10.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE= +github.com/hashicorp/terraform-json v0.12.0/go.mod h1:pmbq9o4EuL43db5+0ogX10Yofv1nozM+wskr/bGFJpI= github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= +github.com/hashicorp/terraform-plugin-go v0.3.0/go.mod h1:dFHsQMaTLpON2gWhVWT96fvtlc/MF1vSy3OdMhWBzdM= github.com/hashicorp/terraform-plugin-go v0.9.1 h1:vXdHaQ6aqL+OF076nMSBV+JKPdmXlzG5mzVDD04WyPs= github.com/hashicorp/terraform-plugin-go v0.9.1/go.mod h1:ItjVSlQs70otlzcCwlPcU8FRXLdO973oYFRZwAOxy8M= github.com/hashicorp/terraform-plugin-log v0.4.0 h1:F3eVnm8r2EfQCe2k9blPIiF/r2TT01SHijXnS7bujvc= github.com/hashicorp/terraform-plugin-log v0.4.0/go.mod h1:9KclxdunFownr4pIm1jdmwKRmE4d6HVG2c9XDq47rpg= github.com/hashicorp/terraform-plugin-sdk v1.17.2 h1:V7DUR3yBWFrVB9z3ddpY7kiYVSsq4NYR67NiTs93NQo= github.com/hashicorp/terraform-plugin-sdk v1.17.2/go.mod h1:wkvldbraEMkz23NxkkAsFS88A1R9eUiooiaUZyS6TLw= +github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0/go.mod h1:grseeRo9g3yNkYW09iFlV8LG78jTa1ssBgouogQg/RU= github.com/hashicorp/terraform-plugin-sdk/v2 v2.17.0 h1:Qr5fWNg1SPSfCRMtou67Y6Kcy9UnMYRNlIJTKRuUvXU= github.com/hashicorp/terraform-plugin-sdk/v2 v2.17.0/go.mod h1:b+LFg8WpYgFgvEBP/6Htk5H9/pJp1V1E8NJAekfH2Ws= github.com/hashicorp/terraform-plugin-test/v2 v2.2.1/go.mod h1:eZ9JL3O69Cb71Skn6OhHyj17sLmHRb+H6VrDcJjKrYU= @@ -473,6 +486,7 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -746,6 +760,7 @@ github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= diff --git a/pkg/config/provider.go b/pkg/config/provider.go index dfba217c..7fb2f81f 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package config import ( @@ -96,6 +112,10 @@ type Provider struct { // resource name. Resources map[string]*Resource + // ProviderMetadataPath is the scraped provider metadata file path + // from Terraform registry + ProviderMetadataPath string + // resourceConfigurators is a map holding resource configurators where key // is Terraform resource name. resourceConfigurators map[string]ResourceConfiguratorChain @@ -146,6 +166,14 @@ func WithDefaultResourceFn(f DefaultResourceFn) ProviderOption { } } +// WithProviderMetadata configures the Terraform metadata file scraped +// from the Terraform registry +func WithProviderMetadata(metadataPath string) ProviderOption { + return func(p *Provider) { + p.ProviderMetadataPath = metadataPath + } +} + // NewProviderWithSchema builds and returns a new Provider from provider // tfjson schema, that is generated using Terraform CLI with: // `terraform providers schema --json` diff --git a/pkg/pipeline/crd.go b/pkg/pipeline/crd.go index 432e1ea8..e6b780e6 100644 --- a/pkg/pipeline/crd.go +++ b/pkg/pipeline/crd.go @@ -42,6 +42,7 @@ type CRDGenerator struct { Group string ProviderShortName string LicenseHeaderPath string + Generated *tjtypes.Generated pkg *types.Package } @@ -65,6 +66,8 @@ func (cg *CRDGenerator) Generate(cfg *config.Resource) (string, error) { if err != nil { return "", errors.Wrapf(err, "cannot build types for %s", cfg.Kind) } + cg.Generated = &gen + // TODO(muvaf): TypePrinter uses the given scope to see if the type exists // before printing. We should ideally load the package in file system but // loading the local package will result in error if there is diff --git a/pkg/pipeline/example.go b/pkg/pipeline/example.go new file mode 100644 index 00000000..dc4dbf26 --- /dev/null +++ b/pkg/pipeline/example.go @@ -0,0 +1,297 @@ +/* +Copyright 2021 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/pkg/errors" + "sigs.k8s.io/yaml" + + "github.com/crossplane/terrajet/pkg/config" + "github.com/crossplane/terrajet/pkg/resource/json" + tjtypes "github.com/crossplane/terrajet/pkg/types" +) + +var ( + reRef = regexp.MustCompile(`\${(.+)}`) +) + +type pavedWithManifest struct { + manifestPath string + paved *fieldpath.Paved +} + +// ResourceExample represents the scraped example HCL configuration +// for a Terraform resource +type ResourceExample struct { + Manifest string `yaml:"manifest"` + References map[string]string `yaml:"references,omitempty"` +} + +// Resource represents the scraped metadata for a Terraform resource +type Resource struct { + SubCategory string `yaml:"subCategory"` + Description string `yaml:"description,omitempty"` + Name string `yaml:"name"` + TitleName string `yaml:"titleName"` + Examples []ResourceExample `yaml:"examples,omitempty"` + ArgumentDocs map[string]string `yaml:"argumentDocs"` + ImportStatements []string `yaml:"importStatements"` +} + +// ProviderMetadata metadata for a Terraform native provider +type ProviderMetadata struct { + Name string `yaml:"name"` + Resources map[string]*Resource `yaml:"resources"` +} + +// NewProviderMetadataFromFile loads metadata from the specified YAML-formatted file +func NewProviderMetadataFromFile(path string) (*ProviderMetadata, error) { + buff, err := ioutil.ReadFile(filepath.Clean(path)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read metadata file %q", path) + } + + metadata := &ProviderMetadata{} + return metadata, errors.Wrap(yaml.Unmarshal(buff, metadata), "failed to unmarshal provider metadata") +} + +// ExampleGenerator represents a pipeline for generating example manifests. +// Generates example manifests for Terraform resources under examples-generated. +type ExampleGenerator struct { + rootDir string + resourceMeta map[string]*Resource + resources map[string]*pavedWithManifest +} + +// NewExampleGenerator returns a configured ExampleGenerator +func NewExampleGenerator(rootDir string, resourceMeta map[string]*Resource) *ExampleGenerator { + return &ExampleGenerator{ + rootDir: rootDir, + resourceMeta: resourceMeta, + resources: make(map[string]*pavedWithManifest), + } +} + +// StoreExamples stores the generated example manifests under examples-generated in +// their respective API groups. +func (eg *ExampleGenerator) StoreExamples() error { + for n, pm := range eg.resources { + if err := eg.resolveReferences(pm.paved.UnstructuredContent()); err != nil { + return errors.Wrapf(err, "cannot resolve references for resource: %s", n) + } + buff, err := yaml.Marshal(pm.paved.UnstructuredContent()) + if err != nil { + return errors.Wrapf(err, "cannot marshal example manifest for resource: %s", n) + } + manifestDir := filepath.Dir(pm.manifestPath) + if err := os.MkdirAll(manifestDir, 0750); err != nil { + return errors.Wrapf(err, "cannot mkdir %s", manifestDir) + } + + b := bytes.Buffer{} + b.WriteString("# This example manifest is auto-generated, and has not been tested.\n") + b.WriteString("# Please make the necessary adjustments before using it.\n") + b.Write(buff) + // no sensitive info in the example manifest + if err := ioutil.WriteFile(pm.manifestPath, b.Bytes(), 0644); err != nil { // nolint:gosec + return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.manifestPath, n) + } + } + return nil +} + +func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) error { // nolint:gocyclo + for k, v := range params { + switch t := v.(type) { + case map[string]interface{}: + if err := eg.resolveReferences(t); err != nil { + return err + } + + case []interface{}: + for _, e := range t { + eM, ok := e.(map[string]interface{}) + if !ok { + continue + } + if err := eg.resolveReferences(eM); err != nil { + return err + } + } + + case string: + g := reRef.FindStringSubmatch(t) + if len(g) != 2 { + continue + } + path := strings.Split(g[1], ".") + // expected reference format is .. + if len(path) < 3 { + continue + } + pm := eg.resources[path[0]] + if pm == nil || pm.paved == nil { + continue + } + pathStr := strings.Join(append([]string{"spec", "forProvider"}, path[2:]...), ".") + s, err := pm.paved.GetString(pathStr) + if fieldpath.IsNotFound(err) { + continue + } + if err != nil { + return errors.Wrapf(err, "cannot get string value from paved: %s", pathStr) + } + params[k] = s + } + } + return nil +} + +// Generate generates an example manifest for the specified Terraform resource. +func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, fieldTransformations map[string]tjtypes.Transformation) error { + rm := eg.resourceMeta[r.Name] + if rm == nil || len(rm.Examples) == 0 { + return nil + } + var exampleParams map[string]interface{} + if err := json.TFParser.Unmarshal([]byte(rm.Examples[0].Manifest), &exampleParams); err != nil { + return errors.Wrapf(err, "cannot unmarshal example manifest for resource: %s", r.Name) + } + transformRefFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "") + + example := map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", group, version), + "kind": r.Kind, + "metadata": map[string]interface{}{ + "name": "example", + }, + "spec": map[string]interface{}{ + "forProvider": exampleParams, + "providerConfigRef": map[string]interface{}{ + "name": "example", + }, + }, + } + manifestDir := filepath.Join(eg.rootDir, "examples-generated", strings.ToLower(strings.Split(group, ".")[0])) + eg.resources[r.Name] = &pavedWithManifest{ + manifestPath: filepath.Join(manifestDir, fmt.Sprintf("%s.yaml", strings.ToLower(r.Kind))), + paved: fieldpath.Pave(example), + } + return nil +} + +func getHierarchicalName(prefix, name string) string { + if prefix == "" { + return name + } + return fmt.Sprintf("%s.%s", prefix, name) +} + +func transformRefFields(params map[string]interface{}, omittedFields []string, t map[string]tjtypes.Transformation, namePrefix string) { // nolint:gocyclo + for _, hn := range omittedFields { + for n := range params { + if hn == getHierarchicalName(namePrefix, n) { + delete(params, n) + break + } + } + } + + for n, v := range params { + switch pT := v.(type) { + case map[string]interface{}: + transformRefFields(pT, omittedFields, t, getHierarchicalName(namePrefix, n)) + + case []interface{}: + for _, e := range pT { + eM, ok := e.(map[string]interface{}) + if !ok { + continue + } + transformRefFields(eM, omittedFields, t, getHierarchicalName(namePrefix, n)) + } + } + } + + for hn, transform := range t { + for n, v := range params { + if hn == getHierarchicalName(namePrefix, n) { + delete(params, n) + if transform.IsRef { + if !transform.IsSensitive { + params[transform.TransformedName] = getRefField(v, + map[string]interface{}{ + "name": "example", + }) + } else { + secretName, secretKey := getSecretRef(v) + params[transform.TransformedName] = getRefField(v, + map[string]interface{}{ + "name": secretName, + "namespace": "crossplane-system", + "key": secretKey, + }) + } + } else { + params[transform.TransformedName] = v + } + break + } + } + } +} + +func getRefField(v interface{}, ref map[string]interface{}) interface{} { + switch v.(type) { + case []interface{}: + return []interface{}{ + ref, + } + + default: + return ref + } +} + +func getSecretRef(v interface{}) (string, string) { + secretName := "example-secret" + secretKey := "example-key" + s, ok := v.(string) + if !ok { + return secretName, secretKey + } + g := reRef.FindStringSubmatch(s) + if len(g) != 2 { + return secretName, secretKey + } + parts := strings.Split(g[1], ".") + if len(parts) < 3 { + return secretName, secretKey + } + secretName = fmt.Sprintf("example-%s", strings.Join(strings.Split(parts[0], "_")[1:], "-")) + secretKey = fmt.Sprintf("attribute.%s", strings.Join(parts[2:], ".")) + return secretName, secretKey +} diff --git a/pkg/pipeline/run.go b/pkg/pipeline/run.go index 735d9993..112dcc11 100644 --- a/pkg/pipeline/run.go +++ b/pkg/pipeline/run.go @@ -57,6 +57,15 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo resourcesGroups[group][resource.Version][name] = resource } + metaResources := make(map[string]*Resource) + if pc.ProviderMetadataPath != "" { + providerMetadata, err := NewProviderMetadataFromFile(filepath.Join(rootDir, pc.ProviderMetadataPath)) + if err != nil { + panic(errors.Wrap(err, "cannot read Terraform provider metadata")) + } + metaResources = providerMetadata.Resources + } + // Add ProviderConfig API package to the list of API version packages. apiVersionPkgList := make([]string, 0) for _, p := range pc.BasePackages.APIVersion { @@ -68,6 +77,7 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo controllerPkgList = append(controllerPkgList, filepath.Join(pc.ModulePath, p)) } count := 0 + exampleGen := NewExampleGenerator(rootDir, metaResources) for group, versions := range resourcesGroups { for version, resources := range versions { var tfResources []*terraformedInput @@ -90,6 +100,9 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo panic(errors.Wrapf(err, "cannot generate controller for resource %s", name)) } controllerPkgList = append(controllerPkgList, ctrlPkgPath) + if err := exampleGen.Generate(group, version, resources[name], crdGen.Generated.FieldTransformations); err != nil { + panic(errors.Wrapf(err, "cannot generate example manifest for resource %s", name)) + } count++ } @@ -104,6 +117,10 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo } } + if err := exampleGen.StoreExamples(); err != nil { + panic(errors.Wrapf(err, "cannot store examples")) + } + if err := NewRegisterGenerator(rootDir, pc.ModulePath).Generate(apiVersionPkgList); err != nil { panic(errors.Wrap(err, "cannot generate register file")) } diff --git a/pkg/types/builder.go b/pkg/types/builder.go index a7a3e5d9..79b546c2 100644 --- a/pkg/types/builder.go +++ b/pkg/types/builder.go @@ -26,6 +26,15 @@ const ( emptyStruct = "struct{}" ) +// Transformation represents a transformation applied to a +// Terraform resource attribute. It's used to record any +// transformations applied by the CRD generation pipeline. +type Transformation struct { + TransformedName string + IsRef bool + IsSensitive bool +} + // Generated is a struct that holds generated types type Generated struct { Types []*types.Named @@ -33,6 +42,8 @@ type Generated struct { ForProviderType *types.Named AtProviderType *types.Named + + FieldTransformations map[string]Transformation } // Builder is used to generate Go type equivalence of given Terraform schema. @@ -53,16 +64,18 @@ func NewBuilder(pkg *types.Package) *Builder { // Build returns parameters and observation types built out of Terraform schema. func (g *Builder) Build(cfg *config.Resource) (Generated, error) { - fp, ap, err := g.buildResource(cfg.TerraformResource, cfg, nil, nil, false, cfg.Kind) + fieldTransformations := make(map[string]Transformation) + fp, ap, err := g.buildResource(cfg.TerraformResource, cfg, nil, nil, false, fieldTransformations, cfg.Kind) return Generated{ - Types: g.genTypes, - Comments: g.comments, - ForProviderType: fp, - AtProviderType: ap, + Types: g.genTypes, + Comments: g.comments, + ForProviderType: fp, + AtProviderType: ap, + FieldTransformations: fieldTransformations, }, errors.Wrapf(err, "cannot build the Types") } -func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPath []string, xpPath []string, asBlocksMode bool, names ...string) (*types.Named, *types.Named, error) { //nolint:gocyclo +func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPath []string, xpPath []string, asBlocksMode bool, t map[string]Transformation, names ...string) (*types.Named, *types.Named, error) { //nolint:gocyclo // NOTE(muvaf): There can be fields in the same CRD with same name but in // different types. Since we generate the type using the field name, there // can be collisions. In order to be able to generate unique names consistently, @@ -130,7 +143,7 @@ func (g *Builder) AddToBuilder(typeNames *TypeNames, r *resource) (*types.Named, return paramType, obsType } -func (g *Builder) buildSchema(f *Field, cfg *config.Resource, names []string, r *resource) (types.Type, error) { // nolint:gocyclo +func (g *Builder) buildSchema(f *Field, cfg *config.Resource, t map[string]Transformation, names []string, r *resource) (types.Type, error) { // nolint:gocyclo switch f.Schema.Type { case schema.TypeBool: return types.NewPointer(types.Universe.Lookup("bool").Type()), nil diff --git a/pkg/types/reference.go b/pkg/types/reference.go index 53409782..1ff60468 100644 --- a/pkg/types/reference.go +++ b/pkg/types/reference.go @@ -27,7 +27,7 @@ var typeSelectorField types.Type var typeSecretKeySelector types.Type var commentOptional *comments.Comment -func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, r config.Reference) (fields []*types.Var, tags []string) { +func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, r config.Reference) (fields []*types.Var, tags []string, names []name.Name) { _, isSlice := f.Type().(*types.Slice) rfn := r.RefFieldName @@ -59,7 +59,7 @@ func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, r con g.comments.AddFieldComment(t, rfn, commentOptional.Build()) g.comments.AddFieldComment(t, sfn, commentOptional.Build()) - return []*types.Var{ref, sel}, []string{refTag, selTag} + return []*types.Var{ref, sel}, []string{refTag, selTag}, []name.Name{rn, sn} } func init() { diff --git a/pkg/types/reference_test.go b/pkg/types/reference_test.go index 215c3239..b76b028f 100644 --- a/pkg/types/reference_test.go +++ b/pkg/types/reference_test.go @@ -124,7 +124,7 @@ func TestBuilder_generateReferenceFields(t *testing.T) { g := &Builder{ comments: twtypes.Comments{}, } - gotFields, gotTags := g.generateReferenceFields(tc.args.t, tc.args.f, tc.args.r) + gotFields, gotTags, _ := g.generateReferenceFields(tc.args.t, tc.args.f, tc.args.r) if diff := cmp.Diff(tc.want.outFields, gotFields, cmp.Comparer(func(a, b *types.Var) bool { return a.String() == b.String() })); diff != "" { From ab5a68043c897d6b27d2f57d675dc5ae56013ed8 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 6 Jan 2022 19:10:59 +0300 Subject: [PATCH 2/7] Comment out all lines in the generated example manifests Signed-off-by: Alper Rifat Ulucinar (cherry picked from commit 0ad041887b062973f041c7d5cf16939000af4418) --- pkg/pipeline/example.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/example.go b/pkg/pipeline/example.go index dc4dbf26..52788b75 100644 --- a/pkg/pipeline/example.go +++ b/pkg/pipeline/example.go @@ -114,7 +114,7 @@ func (eg *ExampleGenerator) StoreExamples() error { b := bytes.Buffer{} b.WriteString("# This example manifest is auto-generated, and has not been tested.\n") b.WriteString("# Please make the necessary adjustments before using it.\n") - b.Write(buff) + b.Write(commentOut(buff)) // no sensitive info in the example manifest if err := ioutil.WriteFile(pm.manifestPath, b.Bytes(), 0644); err != nil { // nolint:gosec return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.manifestPath, n) @@ -123,6 +123,22 @@ func (eg *ExampleGenerator) StoreExamples() error { return nil } +func commentOut(buff []byte) []byte { + lines := strings.Split(string(buff), "\n") + commentedOutLines := make([]string, 0, len(lines)) + for _, l := range lines { + trimmed := strings.TrimSpace(l) + if len(trimmed) == 0 { + continue + } + if !strings.HasPrefix(trimmed, "#") { + l = "#" + l + } + commentedOutLines = append(commentedOutLines, l) + } + return []byte(strings.Join(commentedOutLines, "\n")) +} + func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) error { // nolint:gocyclo for k, v := range params { switch t := v.(type) { From 31b0a79dd10e84edf863cd69dbe02b106c72604b Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 21 Jan 2022 04:33:57 +0300 Subject: [PATCH 3/7] Remove spec.forProvider.depends_on if exists Signed-off-by: Alper Rifat Ulucinar (cherry picked from commit 0f9468d385e58b1fb2a0772aece828842f878a43) --- go.mod | 1 - go.sum | 15 --------------- pkg/pipeline/example.go | 10 ++++++---- pkg/types/builder.go | 26 +++++++++++++++++--------- pkg/types/field.go | 16 ++++++++-------- 5 files changed, 31 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index 7688ce23..73f08bdf 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.17 require ( github.com/crossplane/crossplane-runtime v0.16.0 - github.com/crossplane/terrajet v0.4.2 github.com/fatih/camelcase v1.0.0 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 diff --git a/go.sum b/go.sum index 5a9e5be1..99ae8047 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,6 @@ cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.61.0/go.mod h1:XukKJg4Y7QsUu0Hxg3qQKUWR4VuWivmyMK2+rUyxAqw= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= @@ -87,7 +86,6 @@ github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f/go.mod h1:k8feO4+ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apparentlymart/go-cidr v1.0.1/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= @@ -107,7 +105,6 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= -github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.37.0 h1:GzFnhOIsrGyQ69s7VgqtrG2BG8v7X7vwB3Xpbd/DBBk= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -163,11 +160,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/crossplane/crossplane-runtime v0.15.1-0.20220106140106-428b7c390375/go.mod h1:CH05KIlxoEHEE4aLpUhPuvF+9qXsN6/H6YIDnUEjlDs= github.com/crossplane/crossplane-runtime v0.16.0 h1:NstJdHeK3C+u3By0vQjOG1Y6+v53JYOy00IgCL9GHAw= github.com/crossplane/crossplane-runtime v0.16.0/go.mod h1:IPT3HTsovwmbw3i+SdsOyaC3r3b7TW+otBMmZsHLnSU= -github.com/crossplane/terrajet v0.4.2 h1:tWgjjPDwzwlurRYgaujsXL7NHhehBy1C8PZloTUQ65U= -github.com/crossplane/terrajet v0.4.2/go.mod h1:oynNx4au8y/tiSb1OanWSMmUf55Cz/O4f1HxH+liI2A= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -381,7 +375,6 @@ github.com/hashicorp/go-getter v1.5.3/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXj github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= @@ -393,7 +386,6 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0= -github.com/hashicorp/go-plugin v1.4.1/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-plugin v1.4.4/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -432,7 +424,6 @@ github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90= -github.com/hashicorp/hcl/v2 v2.3.0/go.mod h1:d+FwDBbOLvpAM3Z6J7gPj/VoAGkNe/gm352ZhjJ/Zv8= github.com/hashicorp/hcl/v2 v2.8.2/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY= github.com/hashicorp/hcl/v2 v2.12.0 h1:PsYxySWpMD4KPaoJLnsHwtK5Qptvj/4Q6s0t4sUxZf4= github.com/hashicorp/hcl/v2 v2.12.0/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= @@ -444,21 +435,17 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/terraform-config-inspect v0.0.0-20191212124732-c6ae6269b9d7 h1:Pc5TCv9mbxFN6UVX0LH6CpQrdTM5YjbVI2w15237Pjk= github.com/hashicorp/terraform-config-inspect v0.0.0-20191212124732-c6ae6269b9d7/go.mod h1:p+ivJws3dpqbp1iP84+npOyAmTTOLMgCzrXd3GSdn/A= github.com/hashicorp/terraform-exec v0.13.3/go.mod h1:SSg6lbUsVB3DmFyCPjBPklqf6EYGX0TlQ6QTxOlikDU= -github.com/hashicorp/terraform-exec v0.14.0/go.mod h1:qrAASDq28KZiMPDnQ02sFS9udcqEkRly002EA2izXTA= github.com/hashicorp/terraform-exec v0.16.1/go.mod h1:aj0lVshy8l+MHhFNoijNHtqTJQI3Xlowv5EOsEaGO7M= github.com/hashicorp/terraform-json v0.10.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE= -github.com/hashicorp/terraform-json v0.12.0/go.mod h1:pmbq9o4EuL43db5+0ogX10Yofv1nozM+wskr/bGFJpI= github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= -github.com/hashicorp/terraform-plugin-go v0.3.0/go.mod h1:dFHsQMaTLpON2gWhVWT96fvtlc/MF1vSy3OdMhWBzdM= github.com/hashicorp/terraform-plugin-go v0.9.1 h1:vXdHaQ6aqL+OF076nMSBV+JKPdmXlzG5mzVDD04WyPs= github.com/hashicorp/terraform-plugin-go v0.9.1/go.mod h1:ItjVSlQs70otlzcCwlPcU8FRXLdO973oYFRZwAOxy8M= github.com/hashicorp/terraform-plugin-log v0.4.0 h1:F3eVnm8r2EfQCe2k9blPIiF/r2TT01SHijXnS7bujvc= github.com/hashicorp/terraform-plugin-log v0.4.0/go.mod h1:9KclxdunFownr4pIm1jdmwKRmE4d6HVG2c9XDq47rpg= github.com/hashicorp/terraform-plugin-sdk v1.17.2 h1:V7DUR3yBWFrVB9z3ddpY7kiYVSsq4NYR67NiTs93NQo= github.com/hashicorp/terraform-plugin-sdk v1.17.2/go.mod h1:wkvldbraEMkz23NxkkAsFS88A1R9eUiooiaUZyS6TLw= -github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0/go.mod h1:grseeRo9g3yNkYW09iFlV8LG78jTa1ssBgouogQg/RU= github.com/hashicorp/terraform-plugin-sdk/v2 v2.17.0 h1:Qr5fWNg1SPSfCRMtou67Y6Kcy9UnMYRNlIJTKRuUvXU= github.com/hashicorp/terraform-plugin-sdk/v2 v2.17.0/go.mod h1:b+LFg8WpYgFgvEBP/6Htk5H9/pJp1V1E8NJAekfH2Ws= github.com/hashicorp/terraform-plugin-test/v2 v2.2.1/go.mod h1:eZ9JL3O69Cb71Skn6OhHyj17sLmHRb+H6VrDcJjKrYU= @@ -486,7 +473,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -760,7 +746,6 @@ github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= -github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= diff --git a/pkg/pipeline/example.go b/pkg/pipeline/example.go index 52788b75..5b09de1d 100644 --- a/pkg/pipeline/example.go +++ b/pkg/pipeline/example.go @@ -29,9 +29,9 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/yaml" - "github.com/crossplane/terrajet/pkg/config" - "github.com/crossplane/terrajet/pkg/resource/json" - tjtypes "github.com/crossplane/terrajet/pkg/types" + "github.com/upbound/upjet/pkg/config" + "github.com/upbound/upjet/pkg/resource/json" + tjtypes "github.com/upbound/upjet/pkg/types" ) var ( @@ -102,7 +102,9 @@ func (eg *ExampleGenerator) StoreExamples() error { if err := eg.resolveReferences(pm.paved.UnstructuredContent()); err != nil { return errors.Wrapf(err, "cannot resolve references for resource: %s", n) } - buff, err := yaml.Marshal(pm.paved.UnstructuredContent()) + u := pm.paved.UnstructuredContent() + delete(u["spec"].(map[string]interface{})["forProvider"].(map[string]interface{}), "depends_on") + buff, err := yaml.Marshal(u) if err != nil { return errors.Wrapf(err, "cannot marshal example manifest for resource: %s", n) } diff --git a/pkg/types/builder.go b/pkg/types/builder.go index 79b546c2..b252dca8 100644 --- a/pkg/types/builder.go +++ b/pkg/types/builder.go @@ -96,29 +96,33 @@ func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPa } var f *Field + tr := Transformation{} switch { case res.Schema[snakeFieldName].Sensitive: var drop bool - f, drop, err = NewSensitiveField(g, cfg, r, res.Schema[snakeFieldName], snakeFieldName, tfPath, xpPath, names, asBlocksMode) + f, drop, err = NewSensitiveField(g, cfg, r, res.Schema[snakeFieldName], snakeFieldName, tfPath, xpPath, names, asBlocksMode, t) if err != nil { return nil, nil, err } if drop { continue } + tr.IsSensitive = true case reference != nil: - f, err = NewReferenceField(g, cfg, r, res.Schema[snakeFieldName], reference, snakeFieldName, tfPath, xpPath, names, asBlocksMode) + f, err = NewReferenceField(g, cfg, r, res.Schema[snakeFieldName], reference, snakeFieldName, tfPath, xpPath, names, asBlocksMode, t) if err != nil { return nil, nil, err } + tr.IsRef = true default: - f, err = NewField(g, cfg, r, res.Schema[snakeFieldName], snakeFieldName, tfPath, xpPath, names, asBlocksMode) + f, err = NewField(g, cfg, r, res.Schema[snakeFieldName], snakeFieldName, tfPath, xpPath, names, asBlocksMode, t) if err != nil { return nil, nil, err } } - - f.AddToResource(g, r, typeNames) + tr.TransformedName = f.Name.LowerCamelComputed + t[fieldPath(f.TerraformPaths)] = tr + f.AddToResource(g, r, typeNames, t) } paramType, obsType := g.AddToBuilder(typeNames, r) @@ -173,7 +177,7 @@ func (g *Builder) buildSchema(f *Field, cfg *config.Resource, t map[string]Trans return nil, errors.Errorf("element type of %s is basic but not one of known basic types", fieldPath(names)) } case *schema.Schema: - newf, err := NewField(g, cfg, r, et, f.Name.Snake, f.TerraformPaths, f.CRDPaths, names, false) + newf, err := NewField(g, cfg, r, et, f.Name.Snake, f.TerraformPaths, f.CRDPaths, names, false, t) if err != nil { return nil, err } @@ -185,7 +189,7 @@ func (g *Builder) buildSchema(f *Field, cfg *config.Resource, t map[string]Trans if f.Schema.ConfigMode == schema.SchemaConfigModeAttr { asBlocksMode = true } - paramType, obsType, err := g.buildResource(et, cfg, f.TerraformPaths, f.CRDPaths, asBlocksMode, names...) + paramType, obsType, err := g.buildResource(et, cfg, f.TerraformPaths, f.CRDPaths, asBlocksMode, t, names...) if err != nil { return nil, errors.Wrapf(err, "cannot infer type from resource schema of element type of %s", fieldPath(names)) } @@ -289,10 +293,14 @@ func (r *resource) addObservationField(f *Field, field *types.Var) { r.obsTags = append(r.obsTags, fmt.Sprintf(`json:"%s" tf:"%s"`, f.JSONTag, f.TFTag)) } -func (r *resource) addReferenceFields(g *Builder, paramName *types.TypeName, field *types.Var, ref config.Reference) { - refFields, refTags := g.generateReferenceFields(paramName, field, ref) +func (r *resource) addReferenceFields(g *Builder, paramName *types.TypeName, field *types.Var, ref config.Reference, t map[string]Transformation, fPath string) { + refFields, refTags, names := g.generateReferenceFields(paramName, field, ref) r.paramTags = append(r.paramTags, refTags...) r.paramFields = append(r.paramFields, refFields...) + t[fPath] = Transformation{ + TransformedName: names[0].LowerCamelComputed, + IsRef: true, + } } // generateTypeName generates a unique name for the type if its original name diff --git a/pkg/types/field.go b/pkg/types/field.go index 70db51a4..814ee6a0 100644 --- a/pkg/types/field.go +++ b/pkg/types/field.go @@ -28,7 +28,7 @@ type Field struct { } // NewField returns a constructed Field object. -func NewField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, snakeFieldName string, tfPath, xpPath, names []string, asBlocksMode bool) (*Field, error) { +func NewField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, snakeFieldName string, tfPath, xpPath, names []string, asBlocksMode bool, t map[string]Transformation) (*Field, error) { f := &Field{ Schema: sch, Name: name.NewFromSnake(snakeFieldName), @@ -62,7 +62,7 @@ func NewField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, } } - fieldType, err := g.buildSchema(f, cfg, names, r) + fieldType, err := g.buildSchema(f, cfg, t, names, r) if err != nil { return nil, errors.Wrapf(err, "cannot infer type from schema of field %s", f.Name.Snake) } @@ -72,8 +72,8 @@ func NewField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, } // NewSensitiveField returns a constructed sensitive Field object. -func NewSensitiveField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, snakeFieldName string, tfPath, xpPath, names []string, asBlocksMode bool) (*Field, bool, error) { //nolint:gocyclo - f, err := NewField(g, cfg, r, sch, snakeFieldName, tfPath, xpPath, names, asBlocksMode) +func NewSensitiveField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, snakeFieldName string, tfPath, xpPath, names []string, asBlocksMode bool, t map[string]Transformation) (*Field, bool, error) { //nolint:gocyclo + f, err := NewField(g, cfg, r, sch, snakeFieldName, tfPath, xpPath, names, asBlocksMode, t) if err != nil { return nil, false, err } @@ -114,8 +114,8 @@ func NewSensitiveField(g *Builder, cfg *config.Resource, r *resource, sch *schem } // NewReferenceField returns a constructed reference Field object. -func NewReferenceField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, ref *config.Reference, snakeFieldName string, tfPath, xpPath, names []string, asBlocksMode bool) (*Field, error) { - f, err := NewField(g, cfg, r, sch, snakeFieldName, tfPath, xpPath, names, asBlocksMode) +func NewReferenceField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, ref *config.Reference, snakeFieldName string, tfPath, xpPath, names []string, asBlocksMode bool, t map[string]Transformation) (*Field, error) { + f, err := NewField(g, cfg, r, sch, snakeFieldName, tfPath, xpPath, names, asBlocksMode, t) if err != nil { return nil, err } @@ -128,7 +128,7 @@ func NewReferenceField(g *Builder, cfg *config.Resource, r *resource, sch *schem } // AddToResource adds built field to the resource. -func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames) { +func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames, t map[string]Transformation) { if f.Comment.TerrajetOptions.FieldTFTag != nil { f.TFTag = *f.Comment.TerrajetOptions.FieldTFTag } @@ -148,7 +148,7 @@ func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames) { } if f.Reference != nil { - r.addReferenceFields(g, typeNames.ParameterTypeName, field, *f.Reference) + r.addReferenceFields(g, typeNames.ParameterTypeName, field, *f.Reference, t, fieldPath(f.TerraformPaths)) } g.comments.AddFieldComment(typeNames.ParameterTypeName, f.FieldNameCamel, f.Comment.Build()) From ed0a8e62d1044dc062c63e3f558b8c5aecead7ea Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 1 Jun 2022 19:49:40 +0300 Subject: [PATCH 4/7] Fix the bug involving multi-level reference resolution Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 14 +---------- pkg/pipeline/example.go | 52 ++++++++++++++++++++--------------------- pkg/types/builder.go | 13 ++++------- pkg/types/field.go | 9 ++++--- pkg/types/reference.go | 10 ++++---- 5 files changed, 42 insertions(+), 56 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 7fb2f81f..0fa573ff 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -1,17 +1,5 @@ /* -Copyright 2021 The Crossplane Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright 2022 Upbound Inc. */ package config diff --git a/pkg/pipeline/example.go b/pkg/pipeline/example.go index 5b09de1d..d85c1a35 100644 --- a/pkg/pipeline/example.go +++ b/pkg/pipeline/example.go @@ -17,7 +17,6 @@ limitations under the License. package pipeline import ( - "bytes" "fmt" "io/ioutil" "os" @@ -35,12 +34,14 @@ import ( ) var ( - reRef = regexp.MustCompile(`\${(.+)}`) + reRef = regexp.MustCompile(`\${(.+)}`) + reFile = regexp.MustCompile(`file\("(.+)"\)`) ) type pavedWithManifest struct { manifestPath string paved *fieldpath.Paved + refsResolved bool } // ResourceExample represents the scraped example HCL configuration @@ -99,7 +100,7 @@ func NewExampleGenerator(rootDir string, resourceMeta map[string]*Resource) *Exa // their respective API groups. func (eg *ExampleGenerator) StoreExamples() error { for n, pm := range eg.resources { - if err := eg.resolveReferences(pm.paved.UnstructuredContent()); err != nil { + if err := eg.resolveReferencesOfPaved(pm); err != nil { return errors.Wrapf(err, "cannot resolve references for resource: %s", n) } u := pm.paved.UnstructuredContent() @@ -112,33 +113,20 @@ func (eg *ExampleGenerator) StoreExamples() error { if err := os.MkdirAll(manifestDir, 0750); err != nil { return errors.Wrapf(err, "cannot mkdir %s", manifestDir) } - - b := bytes.Buffer{} - b.WriteString("# This example manifest is auto-generated, and has not been tested.\n") - b.WriteString("# Please make the necessary adjustments before using it.\n") - b.Write(commentOut(buff)) // no sensitive info in the example manifest - if err := ioutil.WriteFile(pm.manifestPath, b.Bytes(), 0644); err != nil { // nolint:gosec + if err := ioutil.WriteFile(pm.manifestPath, buff, 0644); err != nil { // nolint:gosec return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.manifestPath, n) } } return nil } -func commentOut(buff []byte) []byte { - lines := strings.Split(string(buff), "\n") - commentedOutLines := make([]string, 0, len(lines)) - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if len(trimmed) == 0 { - continue - } - if !strings.HasPrefix(trimmed, "#") { - l = "#" + l - } - commentedOutLines = append(commentedOutLines, l) +func (eg *ExampleGenerator) resolveReferencesOfPaved(pm *pavedWithManifest) error { + if pm.refsResolved { + return nil } - return []byte(strings.Join(commentedOutLines, "\n")) + pm.refsResolved = true + return errors.Wrap(eg.resolveReferences(pm.paved.UnstructuredContent()), "failed to resolve references of paved") } func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) error { // nolint:gocyclo @@ -174,6 +162,9 @@ func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) err if pm == nil || pm.paved == nil { continue } + if err := eg.resolveReferencesOfPaved(pm); err != nil { + return errors.Wrapf(err, "cannot recursively resolve references for %q", path[0]) + } pathStr := strings.Join(append([]string{"spec", "forProvider"}, path[2:]...), ".") s, err := pm.paved.GetString(pathStr) if fieldpath.IsNotFound(err) { @@ -305,11 +296,18 @@ func getSecretRef(v interface{}) (string, string) { if len(g) != 2 { return secretName, secretKey } - parts := strings.Split(g[1], ".") - if len(parts) < 3 { - return secretName, secretKey + f := reFile.FindStringSubmatch(g[1]) + switch { + case len(f) == 2: // then a file reference + _, file := filepath.Split(f[1]) + secretKey = fmt.Sprintf("attribute.%s", file) + default: + parts := strings.Split(g[1], ".") + if len(parts) < 3 { + return secretName, secretKey + } + secretName = fmt.Sprintf("example-%s", strings.Join(strings.Split(parts[0], "_")[1:], "-")) + secretKey = fmt.Sprintf("attribute.%s", strings.Join(parts[2:], ".")) } - secretName = fmt.Sprintf("example-%s", strings.Join(strings.Split(parts[0], "_")[1:], "-")) - secretKey = fmt.Sprintf("attribute.%s", strings.Join(parts[2:], ".")) return secretName, secretKey } diff --git a/pkg/types/builder.go b/pkg/types/builder.go index b252dca8..6c62a7f8 100644 --- a/pkg/types/builder.go +++ b/pkg/types/builder.go @@ -108,6 +108,7 @@ func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPa continue } tr.IsSensitive = true + tr.IsRef = true case reference != nil: f, err = NewReferenceField(g, cfg, r, res.Schema[snakeFieldName], reference, snakeFieldName, tfPath, xpPath, names, asBlocksMode, t) if err != nil { @@ -120,9 +121,9 @@ func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPa return nil, nil, err } } - tr.TransformedName = f.Name.LowerCamelComputed + f.AddToResource(g, r, typeNames) + tr.TransformedName = f.TransformedName t[fieldPath(f.TerraformPaths)] = tr - f.AddToResource(g, r, typeNames, t) } paramType, obsType := g.AddToBuilder(typeNames, r) @@ -293,14 +294,10 @@ func (r *resource) addObservationField(f *Field, field *types.Var) { r.obsTags = append(r.obsTags, fmt.Sprintf(`json:"%s" tf:"%s"`, f.JSONTag, f.TFTag)) } -func (r *resource) addReferenceFields(g *Builder, paramName *types.TypeName, field *types.Var, ref config.Reference, t map[string]Transformation, fPath string) { - refFields, refTags, names := g.generateReferenceFields(paramName, field, ref) +func (r *resource) addReferenceFields(g *Builder, paramName *types.TypeName, field *types.Var, f *Field) { + refFields, refTags := g.generateReferenceFields(paramName, field, f) r.paramTags = append(r.paramTags, refTags...) r.paramFields = append(r.paramFields, refFields...) - t[fPath] = Transformation{ - TransformedName: names[0].LowerCamelComputed, - IsRef: true, - } } // generateTypeName generates a unique name for the type if its original name diff --git a/pkg/types/field.go b/pkg/types/field.go index 814ee6a0..857e51f5 100644 --- a/pkg/types/field.go +++ b/pkg/types/field.go @@ -25,6 +25,7 @@ type Field struct { FieldType types.Type AsBlocksMode bool Reference *config.Reference + TransformedName string } // NewField returns a constructed Field object. @@ -43,6 +44,7 @@ func NewField(g *Builder, cfg *config.Resource, r *resource, sch *schema.Schema, f.Comment = comment f.TFTag = fmt.Sprintf("%s,omitempty", f.Name.Snake) f.JSONTag = fmt.Sprintf("%s,omitempty", f.Name.LowerCamelComputed) + f.TransformedName = f.Name.LowerCamelComputed // Terraform paths, e.g. { "lifecycle_rule", "*", "transition", "*", "days" } for https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket#lifecycle_rule f.TerraformPaths = append(tfPath, f.Name.Snake) // nolint:gocritic @@ -105,7 +107,8 @@ func NewSensitiveField(g *Builder, cfg *config.Resource, r *resource, sch *schem case "map[string]string", "map[string]*string": f.FieldType = types.NewMap(types.Universe.Lookup("string").Type(), typeSecretKeySelector) } - f.JSONTag = name.NewFromCamel(f.FieldNameCamel).LowerCamelComputed + f.TransformedName = name.NewFromCamel(f.FieldNameCamel).LowerCamelComputed + f.JSONTag = f.TransformedName if f.Schema.Optional { f.FieldType = types.NewPointer(f.FieldType) f.JSONTag += ",omitempty" @@ -128,7 +131,7 @@ func NewReferenceField(g *Builder, cfg *config.Resource, r *resource, sch *schem } // AddToResource adds built field to the resource. -func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames, t map[string]Transformation) { +func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames) { if f.Comment.TerrajetOptions.FieldTFTag != nil { f.TFTag = *f.Comment.TerrajetOptions.FieldTFTag } @@ -148,7 +151,7 @@ func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames, t m } if f.Reference != nil { - r.addReferenceFields(g, typeNames.ParameterTypeName, field, *f.Reference, t, fieldPath(f.TerraformPaths)) + r.addReferenceFields(g, typeNames.ParameterTypeName, field, f) } g.comments.AddFieldComment(typeNames.ParameterTypeName, f.FieldNameCamel, f.Comment.Build()) diff --git a/pkg/types/reference.go b/pkg/types/reference.go index 1ff60468..e857c7a5 100644 --- a/pkg/types/reference.go +++ b/pkg/types/reference.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" "golang.org/x/tools/go/packages" - "github.com/upbound/upjet/pkg/config" "github.com/upbound/upjet/pkg/types/comments" "github.com/upbound/upjet/pkg/types/name" ) @@ -27,10 +26,10 @@ var typeSelectorField types.Type var typeSecretKeySelector types.Type var commentOptional *comments.Comment -func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, r config.Reference) (fields []*types.Var, tags []string, names []name.Name) { +func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, field *Field) (fields []*types.Var, tags []string) { _, isSlice := f.Type().(*types.Slice) - rfn := r.RefFieldName + rfn := field.Reference.RefFieldName if rfn == "" { rfn = f.Name() + "Ref" if isSlice { @@ -38,7 +37,7 @@ func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, r con } } - sfn := r.SelectorFieldName + sfn := field.Reference.SelectorFieldName if sfn == "" { sfn = f.Name() + "Selector" } @@ -58,8 +57,9 @@ func (g *Builder) generateReferenceFields(t *types.TypeName, f *types.Var, r con g.comments.AddFieldComment(t, rfn, commentOptional.Build()) g.comments.AddFieldComment(t, sfn, commentOptional.Build()) + field.TransformedName = rn.LowerCamelComputed - return []*types.Var{ref, sel}, []string{refTag, selTag}, []name.Name{rn, sn} + return []*types.Var{ref, sel}, []string{refTag, selTag} } func init() { From 38bec5acc41deade566561dfbebe70a4db2373c2 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 7 Jun 2022 01:27:06 +0300 Subject: [PATCH 5/7] Remove config.WithProviderMetadata - Instead, metadata document path is a direct parameter of config.NewProviderWithSchema Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 0fa573ff..1dd444d6 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -154,18 +154,10 @@ func WithDefaultResourceFn(f DefaultResourceFn) ProviderOption { } } -// WithProviderMetadata configures the Terraform metadata file scraped -// from the Terraform registry -func WithProviderMetadata(metadataPath string) ProviderOption { - return func(p *Provider) { - p.ProviderMetadataPath = metadataPath - } -} - // NewProviderWithSchema builds and returns a new Provider from provider // tfjson schema, that is generated using Terraform CLI with: // `terraform providers schema --json` -func NewProviderWithSchema(schema []byte, prefix string, modulePath string, opts ...ProviderOption) *Provider { +func NewProviderWithSchema(schema []byte, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider { ps := tfjson.ProviderSchemas{} if err := ps.UnmarshalJSON(schema); err != nil { panic(err) @@ -178,13 +170,13 @@ func NewProviderWithSchema(schema []byte, prefix string, modulePath string, opts rs = v.ResourceSchemas break } - return NewProvider(conversiontfjson.GetV2ResourceMap(rs), prefix, modulePath, opts...) + return NewProvider(conversiontfjson.GetV2ResourceMap(rs), prefix, modulePath, metadataPath, opts...) } // NewProvider builds and returns a new Provider. // Deprecated: This function will be removed soon, please use // NewProviderWithSchema instead. -func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, opts ...ProviderOption) *Provider { +func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider { p := &Provider{ ModulePath: modulePath, TerraformResourcePrefix: fmt.Sprintf("%s_", prefix), @@ -197,6 +189,7 @@ func NewProvider(resourceMap map[string]*schema.Resource, prefix string, moduleP ".+", }, Resources: map[string]*Resource{}, + ProviderMetadataPath: metadataPath, resourceConfigurators: map[string]ResourceConfiguratorChain{}, } From 7a587ee438a0242d8ad1a0b78d37638e971ad5d6 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 7 Jun 2022 01:39:54 +0300 Subject: [PATCH 6/7] Rename config.NewProviderWithSchema as NewProvider Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 12 +++--------- pkg/pipeline/example.go | 14 +------------- pkg/types/reference_test.go | 2 +- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 1dd444d6..2f76f8a4 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -154,10 +154,10 @@ func WithDefaultResourceFn(f DefaultResourceFn) ProviderOption { } } -// NewProviderWithSchema builds and returns a new Provider from provider +// NewProvider builds and returns a new Provider from provider // tfjson schema, that is generated using Terraform CLI with: // `terraform providers schema --json` -func NewProviderWithSchema(schema []byte, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider { +func NewProvider(schema []byte, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider { ps := tfjson.ProviderSchemas{} if err := ps.UnmarshalJSON(schema); err != nil { panic(err) @@ -170,13 +170,8 @@ func NewProviderWithSchema(schema []byte, prefix string, modulePath string, meta rs = v.ResourceSchemas break } - return NewProvider(conversiontfjson.GetV2ResourceMap(rs), prefix, modulePath, metadataPath, opts...) -} -// NewProvider builds and returns a new Provider. -// Deprecated: This function will be removed soon, please use -// NewProviderWithSchema instead. -func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider { + resourceMap := conversiontfjson.GetV2ResourceMap(rs) p := &Provider{ ModulePath: modulePath, TerraformResourcePrefix: fmt.Sprintf("%s_", prefix), @@ -213,7 +208,6 @@ func NewProvider(resourceMap map[string]*schema.Resource, prefix string, moduleP p.Resources[name] = p.DefaultResourceFn(name, terraformResource) } - return p } diff --git a/pkg/pipeline/example.go b/pkg/pipeline/example.go index d85c1a35..08633dd4 100644 --- a/pkg/pipeline/example.go +++ b/pkg/pipeline/example.go @@ -1,17 +1,5 @@ /* -Copyright 2021 The Crossplane Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright 2022 Upbound Inc. */ package pipeline diff --git a/pkg/types/reference_test.go b/pkg/types/reference_test.go index b76b028f..91e4e650 100644 --- a/pkg/types/reference_test.go +++ b/pkg/types/reference_test.go @@ -124,7 +124,7 @@ func TestBuilder_generateReferenceFields(t *testing.T) { g := &Builder{ comments: twtypes.Comments{}, } - gotFields, gotTags, _ := g.generateReferenceFields(tc.args.t, tc.args.f, tc.args.r) + gotFields, gotTags := g.generateReferenceFields(tc.args.t, tc.args.f, &Field{Reference: &tc.args.r}) if diff := cmp.Diff(tc.want.outFields, gotFields, cmp.Comparer(func(a, b *types.Var) bool { return a.String() == b.String() })); diff != "" { From 00b7d18eaf84d8b13f5e3949a0a13999e1ec52e2 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 8 Jun 2022 16:05:46 +0300 Subject: [PATCH 7/7] Rename pipeline.transformRefFields as pipeline.TransformFields Signed-off-by: Alper Rifat Ulucinar --- pkg/pipeline/example.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/pipeline/example.go b/pkg/pipeline/example.go index 08633dd4..99f10681 100644 --- a/pkg/pipeline/example.go +++ b/pkg/pipeline/example.go @@ -177,7 +177,7 @@ func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, if err := json.TFParser.Unmarshal([]byte(rm.Examples[0].Manifest), &exampleParams); err != nil { return errors.Wrapf(err, "cannot unmarshal example manifest for resource: %s", r.Name) } - transformRefFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "") + transformFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "") example := map[string]interface{}{ "apiVersion": fmt.Sprintf("%s/%s", group, version), @@ -207,7 +207,7 @@ func getHierarchicalName(prefix, name string) string { return fmt.Sprintf("%s.%s", prefix, name) } -func transformRefFields(params map[string]interface{}, omittedFields []string, t map[string]tjtypes.Transformation, namePrefix string) { // nolint:gocyclo +func transformFields(params map[string]interface{}, omittedFields []string, t map[string]tjtypes.Transformation, namePrefix string) { // nolint:gocyclo for _, hn := range omittedFields { for n := range params { if hn == getHierarchicalName(namePrefix, n) { @@ -220,7 +220,7 @@ func transformRefFields(params map[string]interface{}, omittedFields []string, t for n, v := range params { switch pT := v.(type) { case map[string]interface{}: - transformRefFields(pT, omittedFields, t, getHierarchicalName(namePrefix, n)) + transformFields(pT, omittedFields, t, getHierarchicalName(namePrefix, n)) case []interface{}: for _, e := range pT { @@ -228,7 +228,7 @@ func transformRefFields(params map[string]interface{}, omittedFields []string, t if !ok { continue } - transformRefFields(eM, omittedFields, t, getHierarchicalName(namePrefix, n)) + transformFields(eM, omittedFields, t, getHierarchicalName(namePrefix, n)) } } }