diff --git a/README.md b/README.md index e23361b7a1..a705a23033 100644 --- a/README.md +++ b/README.md @@ -714,7 +714,7 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_DATA_HOME`: Custom path to be used for `elastic-package` data directory. By default this is `~/.elastic-package`. - Related to the build process: - - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license. + - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license. This path should be relative to the repository root. - `ELASTIC_PACKAGE_LINKS_FILE_PATH`: Path to the links table file (e.g. `links_table.yml`) with the link definitions to be used in the build process of a package. - Related to signing packages: diff --git a/cmd/benchmark.go b/cmd/benchmark.go index e8e7394021..cb7588ffbe 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" @@ -136,10 +137,12 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.BenchNumTopProcsFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) } + + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } @@ -203,6 +206,7 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error { pipeline.WithESAPI(esClient.API), pipeline.WithNumTopProcs(numTopProcs), pipeline.WithFormat(reportFormat), + pipeline.WithRepositoryRoot(repositoryRoot), ) runner := pipeline.NewPipelineBenchmark(opts) @@ -291,17 +295,18 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { } var packageRootPath string - var found bool if len(packageName) == 0 { - packageRootPath, found, err = packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err = packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + profile, err := cobraext.GetProfileFlag(cmd) if err != nil { return err @@ -336,6 +341,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { rally.WithRallyDryRun(rallyDryRun), rally.WithRallyPackageFromRegistry(packageName, packageVersion), rally.WithRallyCorpusAtPath(corpusAtPath), + rally.WithRepositoryRoot(repositoryRoot), } esMetricsClient, err := initializeESMetricsClient(ctx) @@ -465,14 +471,16 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.BenchStreamTimestampFieldFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + profile, err := cobraext.GetProfileFlag(cmd) if err != nil { return err @@ -507,6 +515,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { stream.WithESAPI(esClient.API), stream.WithKibanaClient(kc), stream.WithProfile(profile), + stream.WithRepositoryRoot(repositoryRoot), } runner := stream.NewStreamBenchmark(stream.NewOptions(withOpts...)) @@ -572,10 +581,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.BenchReindexToMetricstoreFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } diff --git a/cmd/build.go b/cmd/build.go index a558a4eba8..e56c8a3da8 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -61,18 +61,38 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { } } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + defer repositoryRoot.Close() + packageRoot, err := packages.MustFindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } + // Currently the build directory is placed inside the repository build/ folder. + // In the future we might want to make this configurable. buildDir, err := builder.BuildDirectory() if err != nil { return fmt.Errorf("can't prepare build directory: %w", err) } logger.Debugf("Use build directory: %s", buildDir) - targets, err := docs.UpdateReadmes(packageRoot, buildDir) + target, err := builder.BuildPackage(cmd.Context(), builder.BuildOptions{ + PackageRootPath: packageRoot, + BuildDir: buildDir, + CreateZip: createZip, + SignPackage: signPackage, + SkipValidation: skipValidation, + RepositoryRoot: repositoryRoot, + }) + if err != nil { + return fmt.Errorf("building package failed: %w", err) + } + + targets, err := docs.UpdateReadmes(repositoryRoot, packageRoot, buildDir) if err != nil { return fmt.Errorf("updating files failed: %w", err) } @@ -82,16 +102,6 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { cmd.Printf("%s file rendered: %s\n", fileName, target) } - target, err := builder.BuildPackage(cmd.Context(), builder.BuildOptions{ - PackageRoot: packageRoot, - BuildDir: buildDir, - CreateZip: createZip, - SignPackage: signPackage, - SkipValidation: skipValidation, - }) - if err != nil { - return fmt.Errorf("building package failed: %w", err) - } cmd.Printf("Package built: %s\n", target) cmd.Println("Done") diff --git a/cmd/create_data_stream.go b/cmd/create_data_stream.go index 01e10e3868..102b7475a4 100644 --- a/cmd/create_data_stream.go +++ b/cmd/create_data_stream.go @@ -37,13 +37,13 @@ type newDataStreamAnswers struct { func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { cmd.Println("Create a new data stream") - packageRoot, found, err := packages.FindPackageRoot() + packageRoot, err := packages.FindPackageRoot() if err != nil { + if errors.Is(err, packages.ErrPackageRootNotFound) { + return errors.New("package root not found, you can only create new data stream in the package context") + } return fmt.Errorf("locating package root failed: %w", err) } - if !found { - return errors.New("package root not found, you can only create new data stream in the package context") - } manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) if err != nil { diff --git a/cmd/format.go b/cmd/format.go index 8a654f790f..8976ac6bb0 100644 --- a/cmd/format.go +++ b/cmd/format.go @@ -5,7 +5,6 @@ package cmd import ( - "errors" "fmt" "github.com/spf13/cobra" @@ -35,13 +34,10 @@ func setupFormatCommand() *cobraext.Command { func formatCommandAction(cmd *cobra.Command, args []string) error { cmd.Println("Format the package") - packageRoot, found, err := packages.FindPackageRoot() + packageRoot, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } - if !found { - return errors.New("package root not found") - } ff, err := cmd.Flags().GetBool(cobraext.FailFastFlagName) if err != nil { diff --git a/cmd/install.go b/cmd/install.go index a3e9c0132a..adabd025fe 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -5,12 +5,12 @@ package cmd import ( - "errors" "fmt" "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/packages" @@ -74,22 +74,24 @@ func installCommandAction(cmd *cobra.Command, _ []string) error { } if zipPathFile == "" && packageRootPath == "" { - var found bool var err error - packageRootPath, found, err = packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err = packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + installer, err := installer.NewForPackage(cmd.Context(), installer.Options{ - Kibana: kibanaClient, - RootPath: packageRootPath, - SkipValidation: skipValidation, - ZipPath: zipPathFile, + Kibana: kibanaClient, + PackageRootPath: packageRootPath, + SkipValidation: skipValidation, + ZipPath: zipPathFile, + RepositoryRoot: repositoryRoot, }) if err != nil { return fmt.Errorf("package installation failed: %w", err) diff --git a/cmd/links.go b/cmd/links.go index 7fdbccfdbf..4315c8ba22 100644 --- a/cmd/links.go +++ b/cmd/links.go @@ -7,7 +7,6 @@ package cmd import ( "fmt" "os" - "path/filepath" "github.com/spf13/cobra" @@ -56,8 +55,14 @@ func linksCheckCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("reading current working directory failed: %w", err) } + // Find the repository root to create the links filesystem reference tied to the repository root + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("finding repository root: %w", err) + } + defer repositoryRoot.Close() - linksFS, err := files.CreateLinksFSFromPath(pwd) + linksFS, err := files.CreateLinksFSFromPath(repositoryRoot, pwd) if err != nil { return fmt.Errorf("creating links filesystem failed: %w", err) } @@ -68,7 +73,7 @@ func linksCheckCommandAction(cmd *cobra.Command, args []string) error { } for _, f := range linkedFiles { if !f.UpToDate { - cmd.Printf("%s is outdated.\n", filepath.Join(f.WorkDir, f.LinkFilePath)) + cmd.Printf("%s is outdated.\n", f.LinkFilePath) } } if len(linkedFiles) > 0 { @@ -95,7 +100,14 @@ func linksUpdateCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("reading current working directory failed: %w", err) } - linksFS, err := files.CreateLinksFSFromPath(pwd) + // Find the repository root to create the links filesystem reference tied to the repository root + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("finding repository root: %w", err) + } + defer repositoryRoot.Close() + + linksFS, err := files.CreateLinksFSFromPath(repositoryRoot, pwd) if err != nil { return fmt.Errorf("creating links filesystem failed: %w", err) } @@ -135,7 +147,14 @@ func linksListCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("reading current working directory failed: %w", err) } - linksFS, err := files.CreateLinksFSFromPath(pwd) + // Find the repository root to create the links filesystem reference tied to the repository root + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("finding repository root: %w", err) + } + defer repositoryRoot.Close() + + linksFS, err := files.CreateLinksFSFromPath(repositoryRoot, pwd) if err != nil { return fmt.Errorf("creating links filesystem failed: %w", err) } diff --git a/cmd/lint.go b/cmd/lint.go index bba2188f9d..a0759752f7 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -5,13 +5,13 @@ package cmd import ( - "errors" "fmt" "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/docs" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/validation" @@ -45,7 +45,19 @@ func setupLintCommand() *cobraext.Command { func lintCommandAction(cmd *cobra.Command, args []string) error { cmd.Println("Lint the package") - readmeFiles, err := docs.AreReadmesUpToDate() + + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + defer repositoryRoot.Close() + + packageRoot, err := packages.MustFindPackageRoot() + if err != nil { + return fmt.Errorf("package root not found: %w", err) + } + + readmeFiles, err := docs.AreReadmesUpToDate(repositoryRoot, packageRoot) if err != nil { for _, f := range readmeFiles { if !f.UpToDate { @@ -61,10 +73,7 @@ func lintCommandAction(cmd *cobra.Command, args []string) error { } func validateSourceCommandAction(cmd *cobra.Command, args []string) error { - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } diff --git a/cmd/service.go b/cmd/service.go index ed542838ce..76c4ee7723 100644 --- a/cmd/service.go +++ b/cmd/service.go @@ -5,7 +5,6 @@ package cmd import ( - "errors" "fmt" "path/filepath" @@ -47,13 +46,10 @@ func setupServiceCommand() *cobraext.Command { func upCommandAction(cmd *cobra.Command, args []string) error { cmd.Println("Boot up the service stack") - packageRoot, found, err := packages.FindPackageRoot() + packageRoot, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } - if !found { - return errors.New("package root not found") - } var dataStreamPath string dataStreamFlag, _ := cmd.Flags().GetString(cobraext.DataStreamFlagName) diff --git a/cmd/status.go b/cmd/status.go index 0b87da5ee7..1d4c7cf8ba 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -163,11 +163,11 @@ func getPackageStatus(packageName string, options registry.SearchOptions) (*stat if packageName != "" { return status.RemotePackage(packageName, options) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return nil, errors.New("no package specified and package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { + if errors.Is(err, packages.ErrPackageRootNotFound) { + return nil, errors.New("no package specified and package root not found") + } return nil, fmt.Errorf("locating package root failed: %w", err) } return status.LocalPackage(packageRootPath, options) diff --git a/cmd/testrunner.go b/cmd/testrunner.go index dae556c3ee..f06a33045c 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" @@ -145,14 +146,16 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(fmt.Errorf("coverage format not available: %s", testCoverageFormat), cobraext.TestCoverageFormatFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRootPath) if err != nil { return fmt.Errorf("reading package manifest failed (path: %s): %w", packageRootPath, err) @@ -177,6 +180,7 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error { GlobalTestConfig: globalTestConfig.Asset, WithCoverage: testCoverage, CoverageType: testCoverageFormat, + RepositoryRoot: repositoryRoot, }) results, err := testrunner.RunSuite(ctx, runner) @@ -235,10 +239,7 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(fmt.Errorf("coverage format not available: %s", testCoverageFormat), cobraext.TestCoverageFormatFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } @@ -342,10 +343,12 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.DeferCleanupFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) } + + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } @@ -388,6 +391,7 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error { CoverageType: testCoverageFormat, DeferCleanup: deferCleanup, GlobalTestConfig: globalTestConfig.Pipeline, + RepositoryRoot: repositoryRoot, }) results, err := testrunner.RunSuite(ctx, runner) @@ -486,14 +490,16 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.VariantFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + runSetup, err := cmd.Flags().GetBool(cobraext.SetupFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.SetupFlagName) @@ -578,6 +584,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { GlobalTestConfig: globalTestConfig.System, WithCoverage: testCoverage, CoverageType: testCoverageFormat, + RepositoryRoot: repositoryRoot, }) logger.Debugf("Running suite...") @@ -651,14 +658,16 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(fmt.Errorf("coverage format not available: %s", testCoverageFormat), cobraext.TestCoverageFormatFlagName) } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } + repositoryRoot, err := files.FindRepositoryRoot() + if err != nil { + return fmt.Errorf("locating repository root failed: %w", err) + } + dataStreams, err := getDataStreamsFlag(cmd, packageRootPath) if err != nil { return err @@ -691,6 +700,7 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error { GlobalTestConfig: globalTestConfig.Policy, WithCoverage: testCoverage, CoverageType: testCoverageFormat, + RepositoryRoot: repositoryRoot, }) results, err := testrunner.RunSuite(ctx, runner) diff --git a/cmd/uninstall.go b/cmd/uninstall.go index 1624d93c30..01075d3922 100644 --- a/cmd/uninstall.go +++ b/cmd/uninstall.go @@ -5,7 +5,6 @@ package cmd import ( - "errors" "fmt" "github.com/spf13/cobra" @@ -35,10 +34,7 @@ func setupUninstallCommand() *cobraext.Command { } func uninstallCommandAction(cmd *cobra.Command, args []string) error { - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } + packageRootPath, err := packages.FindPackageRoot() if err != nil { return fmt.Errorf("locating package root failed: %w", err) } diff --git a/internal/benchrunner/runners/pipeline/options.go b/internal/benchrunner/runners/pipeline/options.go index c2699d6be4..90db1964f2 100644 --- a/internal/benchrunner/runners/pipeline/options.go +++ b/internal/benchrunner/runners/pipeline/options.go @@ -5,6 +5,8 @@ package pipeline import ( + "os" + "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -17,6 +19,7 @@ type Options struct { API *elasticsearch.API NumTopProcs int Format Format + RepositoryRoot *os.Root } type OptionFunc func(*Options) @@ -64,3 +67,9 @@ func WithBenchmarkName(name string) OptionFunc { opts.BenchName = name } } + +func WithRepositoryRoot(root *os.Root) OptionFunc { + return func(opts *Options) { + opts.RepositoryRoot = root + } +} diff --git a/internal/benchrunner/runners/pipeline/runner.go b/internal/benchrunner/runners/pipeline/runner.go index 17b6532d9d..2cc8db9e28 100644 --- a/internal/benchrunner/runners/pipeline/runner.go +++ b/internal/benchrunner/runners/pipeline/runner.go @@ -48,7 +48,7 @@ func (r *runner) SetUp(ctx context.Context) error { return errors.New("data stream root not found") } - r.entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(ctx, r.options.API, dataStreamPath) + r.entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(ctx, r.options.API, dataStreamPath, r.options.RepositoryRoot) if err != nil { return fmt.Errorf("installing ingest pipelines failed: %w", err) } diff --git a/internal/benchrunner/runners/rally/options.go b/internal/benchrunner/runners/rally/options.go index 0f7eae2c22..769730e844 100644 --- a/internal/benchrunner/runners/rally/options.go +++ b/internal/benchrunner/runners/rally/options.go @@ -5,6 +5,7 @@ package rally import ( + "os" "time" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -29,6 +30,7 @@ type Options struct { PackageName string PackageVersion string CorpusAtPath string + RepositoryRoot *os.Root } type ClientOptions struct { @@ -118,3 +120,9 @@ func WithRallyCorpusAtPath(c string) OptionFunc { opts.CorpusAtPath = c } } + +func WithRepositoryRoot(r *os.Root) OptionFunc { + return func(opts *Options) { + opts.RepositoryRoot = r + } +} diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index ebf6d9a649..99840ba683 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -486,9 +486,10 @@ func (r *runner) installPackageFromRegistry(ctx context.Context, packageName, pa func (r *runner) installPackageFromPackageRoot(ctx context.Context) error { logger.Debug("Installing package...") installer, err := installer.NewForPackage(ctx, installer.Options{ - Kibana: r.options.KibanaClient, - RootPath: r.options.PackageRootPath, - SkipValidation: true, + Kibana: r.options.KibanaClient, + PackageRootPath: r.options.PackageRootPath, + SkipValidation: true, + RepositoryRoot: r.options.RepositoryRoot, }) if err != nil { return fmt.Errorf("failed to initialize package installer: %w", err) diff --git a/internal/benchrunner/runners/stream/options.go b/internal/benchrunner/runners/stream/options.go index e1c0889aff..f554021e1f 100644 --- a/internal/benchrunner/runners/stream/options.go +++ b/internal/benchrunner/runners/stream/options.go @@ -5,6 +5,7 @@ package stream import ( + "os" "time" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -25,6 +26,7 @@ type Options struct { PackageRootPath string Variant string Profile *profile.Profile + RepositoryRoot *os.Root } type ClientOptions struct { @@ -107,3 +109,9 @@ func WithTimestampField(t string) OptionFunc { opts.TimestampField = t } } + +func WithRepositoryRoot(r *os.Root) OptionFunc { + return func(opts *Options) { + opts.RepositoryRoot = r + } +} diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index 9322d279b5..fb2d64f029 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -253,9 +253,10 @@ func (r *runner) installPackage(ctx context.Context) error { func (r *runner) installPackageFromPackageRoot(ctx context.Context) error { logger.Debug("Installing package...") installer, err := installer.NewForPackage(ctx, installer.Options{ - Kibana: r.options.KibanaClient, - RootPath: r.options.PackageRootPath, - SkipValidation: true, + Kibana: r.options.KibanaClient, + PackageRootPath: r.options.PackageRootPath, + SkipValidation: true, + RepositoryRoot: r.options.RepositoryRoot, }) if err != nil { diff --git a/internal/builder/packages.go b/internal/builder/packages.go index d0cbbd1d86..4b1914faab 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -26,8 +26,9 @@ const licenseTextFileName = "LICENSE.txt" var repositoryLicenseEnv = environment.WithElasticPackagePrefix("REPOSITORY_LICENSE") type BuildOptions struct { - PackageRoot string - BuildDir string + PackageRootPath string // path to the package source content + BuildDir string // directory where all the built packages are placed and zipped packages are stored + RepositoryRoot *os.Root CreateZip bool SignPackage bool @@ -104,9 +105,9 @@ func BuildPackagesDirectory(packageRoot string, buildDir string) (string, error) return filepath.Join(buildDir, m.Name, m.Version), nil } -// buildPackagesZipPath function locates the target zipped package path. +// buildPackagesZipPath function returns the path to zipped built package. func buildPackagesZipPath(packageRoot string) (string, error) { - buildDir, err := buildPackagesRootDirectory() + buildPackagesDir, err := buildPackagesRootDirectory() if err != nil { return "", fmt.Errorf("can't locate build packages root directory: %w", err) } @@ -114,7 +115,7 @@ func buildPackagesZipPath(packageRoot string) (string, error) { if err != nil { return "", fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err) } - return ZippedBuiltPackagePath(buildDir, *m), nil + return ZippedBuiltPackagePath(buildPackagesDir, *m), nil } // ZippedBuiltPackagePath function returns the path to zipped built package. @@ -163,94 +164,100 @@ func FindBuildPackagesDirectory() (string, bool, error) { // BuildPackage function builds the package. func BuildPackage(ctx context.Context, options BuildOptions) (string, error) { - destinationDir, err := BuildPackagesDirectory(options.PackageRoot, options.BuildDir) + // builtPackageDir is the directory where the built package content is placed + // eg. /packages// + builtPackageDir, err := BuildPackagesDirectory(options.PackageRootPath, options.BuildDir) if err != nil { return "", fmt.Errorf("can't locate build directory: %w", err) } - logger.Debugf("Build directory: %s\n", destinationDir) + logger.Debugf("Build directory: %s\n", builtPackageDir) - logger.Debugf("Clear target directory (path: %s)", destinationDir) - err = files.ClearDir(destinationDir) + logger.Debugf("Clear target directory (path: %s)", builtPackageDir) + err = files.ClearDir(builtPackageDir) if err != nil { return "", fmt.Errorf("clearing package contents failed: %w", err) } - logger.Debugf("Copy package content (source: %s)", options.PackageRoot) - err = files.CopyWithoutDev(options.PackageRoot, destinationDir) + logger.Debugf("Copy package content (source: %s)", options.PackageRootPath) + err = files.CopyWithoutDev(options.PackageRootPath, builtPackageDir) if err != nil { return "", fmt.Errorf("copying package contents failed: %w", err) } logger.Debug("Copy license file if needed") - err = copyLicenseTextFile(filepath.Join(destinationDir, licenseTextFileName)) + destinationLicenseFilePath := filepath.Join(builtPackageDir, licenseTextFileName) + err = copyLicenseTextFile(options.RepositoryRoot, destinationLicenseFilePath) if err != nil { return "", fmt.Errorf("copying license text file: %w", err) } - logger.Debug("Encode dashboards") - err = encodeDashboards(destinationDir) + // when CopyWithoutDev is used, .link files are skipped. + // Include them before resolving external fields + logger.Debug("Include linked files") + linksFS, err := files.CreateLinksFSFromPath(options.RepositoryRoot, options.PackageRootPath) if err != nil { - return "", fmt.Errorf("encoding dashboards failed: %w", err) + return "", fmt.Errorf("creating links filesystem failed: %w", err) } - logger.Debug("Resolve external fields") - err = resolveExternalFields(options.PackageRoot, destinationDir) + links, err := linksFS.IncludeLinkedFiles(builtPackageDir) if err != nil { - return "", fmt.Errorf("resolving external fields failed: %w", err) + return "", fmt.Errorf("including linked files failed: %w", err) + } + for _, l := range links { + logger.Debugf("Linked file included (path: %s)", l.TargetRelPath) } - err = addDynamicMappings(options.PackageRoot, destinationDir) + logger.Debug("Encode dashboards") + err = encodeDashboards(builtPackageDir) if err != nil { - return "", fmt.Errorf("adding dynamic mappings: %w", err) + return "", fmt.Errorf("encoding dashboards failed: %w", err) } - logger.Debug("Include linked files") - linksFS, err := files.CreateLinksFSFromPath(options.PackageRoot) + logger.Debug("Resolve external fields") + err = resolveExternalFields(options.PackageRootPath, builtPackageDir) if err != nil { - return "", fmt.Errorf("creating links filesystem failed: %w", err) + return "", fmt.Errorf("resolving external fields failed: %w", err) } - links, err := linksFS.IncludeLinkedFiles(destinationDir) + err = addDynamicMappings(options.PackageRootPath, builtPackageDir) if err != nil { - return "", fmt.Errorf("including linked files failed: %w", err) - } - for _, l := range links { - logger.Debugf("Linked file included (path: %s)", l.TargetFilePath(destinationDir)) + return "", fmt.Errorf("adding dynamic mappings: %w", err) } - err = resolveTransformDefinitions(destinationDir) + err = resolveTransformDefinitions(builtPackageDir) if err != nil { return "", fmt.Errorf("resolving transform manifests failed: %w", err) } if options.CreateZip { - return buildZippedPackage(ctx, options, destinationDir) + return buildZippedPackage(ctx, options, builtPackageDir) } if options.SkipValidation { logger.Debug("Skip validation of the built package") - return destinationDir, nil + return builtPackageDir, nil } - logger.Debugf("Validating built package (path: %s)", destinationDir) - errs, skipped := validation.ValidateAndFilterFromPath(destinationDir) + logger.Debugf("Validating built package (path: %s)", builtPackageDir) + errs, skipped := validation.ValidateAndFilterFromPath(builtPackageDir) if skipped != nil { logger.Infof("Skipped errors: %v", skipped) } if errs != nil { return "", fmt.Errorf("invalid content found in built package: %w", errs) } - return destinationDir, nil + return builtPackageDir, nil } -func buildZippedPackage(ctx context.Context, options BuildOptions, destinationDir string) (string, error) { +// buildZippedPackage function builds the zipped package from the builtPackageDir and stores it in buildPackagesDir. +func buildZippedPackage(ctx context.Context, options BuildOptions, builtPackageDir string) (string, error) { logger.Debug("Build zipped package") - zippedPackagePath, err := buildPackagesZipPath(options.PackageRoot) + zippedPackagePath, err := buildPackagesZipPath(options.PackageRootPath) if err != nil { return "", fmt.Errorf("can't evaluate path for the zipped package: %w", err) } - err = files.Zip(ctx, destinationDir, zippedPackagePath) + err = files.Zip(ctx, builtPackageDir, zippedPackagePath) if err != nil { return "", fmt.Errorf("can't compress the built package (compressed file path: %s): %w", zippedPackagePath, err) } @@ -280,9 +287,9 @@ func buildZippedPackage(ctx context.Context, options BuildOptions, destinationDi func signZippedPackage(options BuildOptions, zippedPackagePath string) error { logger.Debug("Sign the package") - m, err := packages.ReadPackageManifestFromPackageRoot(options.PackageRoot) + m, err := packages.ReadPackageManifestFromPackageRoot(options.PackageRootPath) if err != nil { - return fmt.Errorf("reading package manifest failed (path: %s): %w", options.PackageRoot, err) + return fmt.Errorf("reading package manifest failed (path: %s): %w", options.PackageRootPath, err) } err = files.Sign(zippedPackagePath, files.SignOptions{ @@ -295,19 +302,39 @@ func signZippedPackage(options BuildOptions, zippedPackagePath string) error { return nil } -func copyLicenseTextFile(licensePath string) error { - _, err := os.Stat(licensePath) - if err == nil { +// copyLicenseTextFile checks the targetLicencePath and copies the license file from the repository root if needed. +// If the targetLicensePath already exists, it will skip copying. +// If the targetLicensePath does not exist, it will look for a source license file in the repository root and copy it to the targetLicensePath. +// The source license file name can be overridden by setting the REPOSITORY_LICENSE environment variable. +func copyLicenseTextFile(repositoryRoot *os.Root, targetLicensePath string) error { + if !filepath.IsAbs(targetLicensePath) { + return fmt.Errorf("target license path (%s) is not an absolute path", targetLicensePath) + } + + // if the given path exists, skip copying + info, err := os.Stat(targetLicensePath) + if err == nil && !info.IsDir() { logger.Debug("License file in the package will be used") return nil } + // if the given path does not exist, continue + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("can't check license path (%s): %w", targetLicensePath, err) + } + // if the given path exists, but is a directory, return an error + if info != nil && info.IsDir() { + return fmt.Errorf("license path (%s) is a directory", targetLicensePath) + } + // lookup for the license file in the repository + // default license name can be overridden by the user repositoryLicenseTextFileName, userDefined := os.LookupEnv(repositoryLicenseEnv) if !userDefined { repositoryLicenseTextFileName = licenseTextFileName } - sourceLicensePath, err := findRepositoryLicense(repositoryLicenseTextFileName) + // sourceLicensePath is an absolute path to the repositoryLicenseTextFileName in the repository root + sourceLicensePath, err := findRepositoryLicensePath(repositoryRoot, repositoryLicenseTextFileName) if !userDefined && errors.Is(err, os.ErrNotExist) { logger.Debug("No license text file is included in package") return nil @@ -317,7 +344,7 @@ func copyLicenseTextFile(licensePath string) error { } logger.Infof("License text found in %q will be included in package", sourceLicensePath) - err = sh.Copy(licensePath, sourceLicensePath) + err = sh.Copy(targetLicensePath, sourceLicensePath) if err != nil { return fmt.Errorf("can't copy license from repository: %w", err) } @@ -346,17 +373,28 @@ func createBuildDirectory(dirs ...string) (string, error) { return buildDir, nil } -func findRepositoryLicense(licenseTextFileName string) (string, error) { - dir, err := files.FindRepositoryRootDirectory() +// findRepositoryLicensePath checks if a license file exists at the specified path and its not empty. +// If the file exists, it returns the path; otherwise, it returns an error indicating +// that the repository license could not be found. +// +// Parameters: +// +// repositoryLicenseTextFileName - the relative path to the license file from the repository root. +// +// Returns: +// +// string - the license file absolute path if found. +// error - an error if the license file does not exist. +func findRepositoryLicensePath(repositoryRoot *os.Root, repositoryLicenseTextFileName string) (string, error) { + // root.ReadFile is supported after go1.25, + // https://go.dev/doc/go1.25 + bytes, err := os.ReadFile(filepath.Join(repositoryRoot.Name(), repositoryLicenseTextFileName)) if err != nil { - return "", err + return "", fmt.Errorf("failed to read repository license: %w", err) } - - sourceFileName := filepath.Join(dir, licenseTextFileName) - _, err = os.Stat(sourceFileName) - if err != nil { - return "", fmt.Errorf("failed to find repository license: %w", err) + if len(bytes) == 0 { + return "", fmt.Errorf("repository license file is empty") } - - return sourceFileName, nil + path := filepath.Join(repositoryRoot.Name(), repositoryLicenseTextFileName) + return path, nil } diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go new file mode 100644 index 0000000000..002b0ddda5 --- /dev/null +++ b/internal/builder/packages_test.go @@ -0,0 +1,157 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package builder + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindRepositoryLicense(t *testing.T) { + t.Run("FileExists", func(t *testing.T) { + root, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer root.Close() + + // Create a LICENSE.txt file in the temp directory + expectedPath := filepath.Join(root.Name(), "LICENSE.txt") + err = os.WriteFile(expectedPath, []byte("license content"), 0644) + require.NoError(t, err) + + path, err := findRepositoryLicensePath(root, "LICENSE.txt") + require.NoError(t, err) + assert.Equal(t, expectedPath, path) + }) + + t.Run("FileDoesNotExist", func(t *testing.T) { + root, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer root.Close() + + path, err := findRepositoryLicensePath(root, "NON_EXISTENT_LICENSE.txt") + require.Error(t, err) + assert.Empty(t, path) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("FileOutsideRoot", func(t *testing.T) { + root, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer root.Close() + + path, err := findRepositoryLicensePath(root, filepath.Join("..", "..", "out.txt")) + require.Error(t, err) + assert.Empty(t, path) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + +} + +func TestCopyLicenseTextFile_UsesExistingLicenseFile(t *testing.T) { + + t.Run("targetLicensePath is relative", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + licensePathRel := filepath.Join("LICENSE.txt") + + // Should not attempt to copy, just return nil + err = copyLicenseTextFile(repositoryRoot, licensePathRel) + assert.Error(t, err) + + }) + + t.Run("targetLicensePath is absolute", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + licensePath := filepath.Join(repositoryRoot.Name(), "LICENSE.txt") + err = os.WriteFile(licensePath, []byte("existing license"), 0644) + require.NoError(t, err) + + // Should not attempt to copy, just return nil + err = copyLicenseTextFile(repositoryRoot, licensePath) + assert.NoError(t, err) + + // License file should remain unchanged + content, err := os.ReadFile(licensePath) + require.NoError(t, err) + assert.Equal(t, "existing license", string(content)) + + }) + + t.Run("ExistingDirectory", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + targetLicensePath := filepath.Join(t.TempDir()) + + err = copyLicenseTextFile(repositoryRoot, targetLicensePath) + assert.Error(t, err) + assert.Contains(t, err.Error(), "is a directory") + }) + + t.Run("RepoLicenseDefaultFileName", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + targetLicensePath := filepath.Join(t.TempDir(), "REPO_LICENSE.txt") + err = os.WriteFile(filepath.Join(repositoryRoot.Name(), licenseTextFileName), []byte("repo license"), 0644) + require.NoError(t, err) + + err = copyLicenseTextFile(repositoryRoot, targetLicensePath) + assert.NoError(t, err) + + content, err := os.ReadFile(targetLicensePath) + require.NoError(t, err) + assert.Equal(t, "repo license", string(content)) + }) + + t.Run("RepoLicenseCustomFileName", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + targetLicensePath := filepath.Join(t.TempDir(), "REPO_LICENSE.txt") + + // original license file path + err = os.WriteFile(filepath.Join(repositoryRoot.Name(), "CUSTOM_LICENSE.txt"), []byte("repo license"), 0644) + require.NoError(t, err) + + t.Setenv(repositoryLicenseEnv, "CUSTOM_LICENSE.txt") + + err = copyLicenseTextFile(repositoryRoot, targetLicensePath) + assert.NoError(t, err) + + content, err := os.ReadFile(targetLicensePath) + require.NoError(t, err) + assert.Equal(t, "repo license", string(content)) + }) + + t.Run("RepoLicenseFileDoesNotExist", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + targetLicensePath := filepath.Join(t.TempDir(), "REPO_LICENSE.txt") + err = copyLicenseTextFile(repositoryRoot, targetLicensePath) + assert.NoError(t, err) + + _, err = repositoryRoot.Stat("LICENSE.txt") + assert.ErrorIs(t, err, os.ErrNotExist) + + _, err = repositoryRoot.Stat("REPO_LICENSE.txt") + assert.ErrorIs(t, err, os.ErrNotExist) + }) + +} diff --git a/internal/docs/links_map.go b/internal/docs/links_map.go index d024bf6906..1af18f5029 100644 --- a/internal/docs/links_map.go +++ b/internal/docs/links_map.go @@ -12,7 +12,6 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/environment" - "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" ) @@ -28,10 +27,10 @@ type linkOptions struct { caption string } -func newLinkMap() linkMap { - var links linkMap - links.Links = make(map[string]string) - return links +func newEmptyLinkMap() linkMap { + return linkMap{ + Links: make(map[string]string), + } } func (l linkMap) Get(key string) (string, error) { @@ -49,29 +48,28 @@ func (l linkMap) Add(key, value string) error { return nil } -func readLinksMap() (linkMap, error) { - linksFilePath, err := linksDefinitionsFilePath() - if err != nil { - return linkMap{}, fmt.Errorf("locating links file failed: %w", err) - } - - links := newLinkMap() +// readLinksMap reads the links definitions file from the given repository root directory, +// parses its YAML contents, and returns a populated linkMap. If the links file does not exist, +// it returns an empty linkMap. Returns an error if locating, reading, or unmarshalling the file fails. +func readLinksMap(linksFilePath string) (linkMap, error) { + // No links file, return empty map with Links initialized if linksFilePath == "" { - return links, nil + return newEmptyLinkMap(), nil } logger.Debugf("Using links definitions file: %s", linksFilePath) contents, err := os.ReadFile(linksFilePath) if err != nil { - return linkMap{}, fmt.Errorf("readfile failed (path: %s): %w", linksFilePath, err) + return newEmptyLinkMap(), fmt.Errorf("readfile failed (path: %s): %w", linksFilePath, err) } - err = yaml.Unmarshal(contents, &links) + var lmap linkMap + err = yaml.Unmarshal(contents, &lmap) if err != nil { - return linkMap{}, err + return newEmptyLinkMap(), err } - return links, nil + return lmap, nil } func (l linkMap) RenderLink(key string, options linkOptions) (string, error) { @@ -85,32 +83,27 @@ func (l linkMap) RenderLink(key string, options linkOptions) (string, error) { return url, nil } -// linksDefinitionsFilePath returns the path where links definitions are located or empty string if the file does not exist. -// If linksMapFilePathEnvVar is defined, it returns the value of that env var. -func linksDefinitionsFilePath() (string, error) { - var err error - linksFilePath, ok := os.LookupEnv(linksMapFilePathEnvVar) - if ok { - _, err = os.Stat(linksFilePath) - if err != nil { +// linksDefinitionsFilePath returns the file path to the links definitions file. +// It first checks if the environment variable specified by linksMapFilePathEnvVar is set. +// If set, it verifies that the file exists and returns its path, or an error if not found. +// If the environment variable is not set, it falls back to the default file path +// constructed from repositoryRoot and linksMapFileNameDefault, returning the path if the file exists, +// or nil if it does not. +func linksDefinitionsFilePath(repositoryRoot *os.Root) (string, error) { + linksFilePath := os.Getenv(linksMapFilePathEnvVar) + if linksFilePath != "" { + if _, err := os.Stat(linksFilePath); err != nil { // if env var is defined, file must exist return "", fmt.Errorf("links definitions file set with %s doesn't exist: %s", linksMapFilePathEnvVar, linksFilePath) } return linksFilePath, nil } - dir, err := files.FindRepositoryRootDirectory() - if err != nil { - return "", err - } - - linksFilePath = filepath.Join(dir, linksMapFileNameDefault) - _, err = os.Stat(linksFilePath) - if err != nil { + linksFilePath = filepath.Join(repositoryRoot.Name(), linksMapFileNameDefault) + if _, err := os.Stat(linksFilePath); err != nil { logger.Debugf("links definitions default file doesn't exist: %s", linksFilePath) return "", nil } return linksFilePath, nil - } diff --git a/internal/docs/links_map_test.go b/internal/docs/links_map_test.go index b0753191ae..12d7737947 100644 --- a/internal/docs/links_map_test.go +++ b/internal/docs/links_map_test.go @@ -90,93 +90,117 @@ func TestRenderLink(t *testing.T) { } func TestLinksDefinitionsFilePath(t *testing.T) { - currentDirectory, _ := os.Getwd() - temporalDirecotry := t.TempDir() - - cases := []struct { - title string - createFileFromEnvVar bool - createDefaultFile bool - linksFilePath string - expectedErrors bool - expected string - }{ - { - title: "No env var and no default file", - createFileFromEnvVar: false, - createDefaultFile: false, - linksFilePath: "", - expectedErrors: false, - expected: "", - }, - { - title: "No env var - default file", - createFileFromEnvVar: false, - createDefaultFile: true, - linksFilePath: "", - expectedErrors: false, - expected: filepath.Join(currentDirectory, "links_table.yml"), - }, - { - title: "Env var defined", - createFileFromEnvVar: true, - createDefaultFile: false, - linksFilePath: filepath.Join(temporalDirecotry, "links_table.yml"), - expectedErrors: false, - expected: filepath.Join(temporalDirecotry, "links_table.yml"), - }, - { - title: "Env var defined but just default file exists", - createFileFromEnvVar: false, - createDefaultFile: true, - linksFilePath: filepath.Join(temporalDirecotry, "links_table_2.yml"), - expectedErrors: true, - expected: "", - }, - } - - createGitFolder() - defer removeGitFolder() - - for _, c := range cases { - t.Run(c.title, func(t *testing.T) { - var err error - if c.linksFilePath != "" { - err = os.Setenv(linksMapFilePathEnvVar, c.linksFilePath) - require.NoError(t, err) - defer os.Unsetenv(linksMapFilePathEnvVar) - } - - if c.createFileFromEnvVar { - err = createLinksFile(c.linksFilePath) - defer removeLinksFile(c.linksFilePath) - require.NoError(t, err) - } - - if c.createDefaultFile { - err = createLinksFile(linksMapFileNameDefault) - require.NoError(t, err) - defer removeLinksFile(linksMapFileNameDefault) - } - - path, err := linksDefinitionsFilePath() - - if c.expectedErrors { - require.Error(t, err) - } else { - require.NoError(t, err) - assert.Equal(t, c.expected, path) - } - }) - } + t.Run("env var set and file exists", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + defaultFilePath := filepath.Join(repositoryRoot.Name(), linksMapFileNameDefault) + testFile := filepath.Join(repositoryRoot.Name(), "custom_links.yml") + require.NoError(t, createLinksFile(testFile)) + require.NoError(t, createLinksFile(defaultFilePath)) // to ensure default file is ignored + t.Setenv(linksMapFilePathEnvVar, testFile) + + path, err := linksDefinitionsFilePath(repositoryRoot) + require.NoError(t, err) + assert.Equal(t, testFile, path) + }) + + t.Run("env var set but file does not exist", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + + missingFile := filepath.Join(repositoryRoot.Name(), "missing_links.yml") + t.Setenv(linksMapFilePathEnvVar, missingFile) + + path, err := linksDefinitionsFilePath(repositoryRoot) + require.Error(t, err) + assert.Empty(t, path) + }) + + t.Run("env var not set, default file exists", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + defaultFilePath := filepath.Join(repositoryRoot.Name(), linksMapFileNameDefault) + + require.NoError(t, createLinksFile(defaultFilePath)) + + path, err := linksDefinitionsFilePath(repositoryRoot) + require.NoError(t, err) + + assert.Equal(t, defaultFilePath, path) + assert.Empty(t, os.Getenv(linksMapFilePathEnvVar)) + }) + + t.Run("env var not set, default file does not exist", func(t *testing.T) { + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + defer repositoryRoot.Close() + defaultFilePath := filepath.Join(repositoryRoot.Name(), linksMapFileNameDefault) + + _, err = os.Stat(defaultFilePath) + require.ErrorIs(t, err, os.ErrNotExist) + + path, err := linksDefinitionsFilePath(repositoryRoot) + require.NoError(t, err) + assert.Empty(t, path) + }) } -func createGitFolder() error { - return os.MkdirAll(".git", os.ModePerm) -} - -func removeGitFolder() error { - return os.RemoveAll(".git") +func TestReadLinksMap(t *testing.T) { + t.Run("empty path returns empty map", func(t *testing.T) { + lmap, err := readLinksMap("") + require.NoError(t, err) + require.NotNil(t, lmap) + assert.Empty(t, lmap.Links) + }) + + t.Run("non-existent file returns error", func(t *testing.T) { + tmpDir := t.TempDir() + missingFile := filepath.Join(tmpDir, "missing.yml") + lmap, err := readLinksMap(missingFile) + require.Error(t, err) + assert.NotNil(t, lmap) + assert.Empty(t, lmap.Links) + }) + + t.Run("invalid YAML returns error", func(t *testing.T) { + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "invalid.yml") + require.NoError(t, os.WriteFile(filePath, []byte("not: valid: yaml: ["), 0644)) + lmap, err := readLinksMap(filePath) + require.Error(t, err) + assert.NotNil(t, lmap) + assert.Empty(t, lmap.Links) + }) + + t.Run("valid YAML returns populated map", func(t *testing.T) { + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "links.yml") + yamlContent := []byte(`links: + intro: http://package-spec.test/intro + docs: http://package-spec.test/docs +`) + require.NoError(t, os.WriteFile(filePath, yamlContent, 0644)) + lmap, err := readLinksMap(filePath) + require.NoError(t, err) + require.NotNil(t, lmap) + assert.Equal(t, "http://package-spec.test/intro", lmap.Links["intro"]) + assert.Equal(t, "http://package-spec.test/docs", lmap.Links["docs"]) + }) + + t.Run("valid YAML with empty links returns empty map", func(t *testing.T) { + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "empty.yml") + yamlContent := []byte("links: {}\n") + require.NoError(t, os.WriteFile(filePath, yamlContent, 0644)) + lmap, err := readLinksMap(filePath) + require.NoError(t, err) + require.NotNil(t, lmap) + assert.Empty(t, lmap.Links) + }) } func createLinksFile(filepath string) error { @@ -187,7 +211,3 @@ func createLinksFile(filepath string) error { defer file.Close() return nil } - -func removeLinksFile(filepath string) error { - return os.Remove(filepath) -} diff --git a/internal/docs/readme.go b/internal/docs/readme.go index 0e44238db7..39791172ee 100644 --- a/internal/docs/readme.go +++ b/internal/docs/readme.go @@ -16,7 +16,6 @@ import ( "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/packages" ) // ReadmeFile contains file name and status of each readme file. @@ -33,10 +32,10 @@ const ( ) // AreReadmesUpToDate function checks if all the .md readme files are up-to-date. -func AreReadmesUpToDate() ([]ReadmeFile, error) { - packageRoot, err := packages.MustFindPackageRoot() +func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFile, error) { + linksFilePath, err := linksDefinitionsFilePath(repositoryRoot) if err != nil { - return nil, fmt.Errorf("package root not found: %w", err) + return nil, fmt.Errorf("locating links file failed: %w", err) } files, err := filepath.Glob(filepath.Join(packageRoot, "_dev", "build", "docs", "*.md")) @@ -47,7 +46,7 @@ func AreReadmesUpToDate() ([]ReadmeFile, error) { var readmeFiles []ReadmeFile for _, filePath := range files { fileName := filepath.Base(filePath) - ok, diff, err := isReadmeUpToDate(fileName, packageRoot) + ok, diff, err := isReadmeUpToDate(fileName, linksFilePath, packageRoot) if !ok || err != nil { readmeFile := ReadmeFile{ FileName: fileName, @@ -65,10 +64,12 @@ func AreReadmesUpToDate() ([]ReadmeFile, error) { return readmeFiles, nil } -func isReadmeUpToDate(fileName, packageRoot string) (bool, string, error) { +// isReadmeUpToDate function checks if a single readme file is up-to-date. +func isReadmeUpToDate(fileName, linksFilePath, packageRoot string) (bool, string, error) { logger.Debugf("Check if %s is up-to-date", fileName) - rendered, shouldBeRendered, err := generateReadme(fileName, packageRoot) + // the readme is generated within the package root, so source should be the packageRoot files too + rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot, packageRoot) if err != nil { return false, "", fmt.Errorf("generating readme file failed: %w", err) } @@ -99,7 +100,12 @@ func isReadmeUpToDate(fileName, packageRoot string) (bool, string, error) { // UpdateReadmes function updates all .md readme files using a defined template // files. The function doesn't perform any action if the template file is not present. -func UpdateReadmes(packageRoot, buildDir string) ([]string, error) { +func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildDir string) ([]string, error) { + linksFilePath, err := linksDefinitionsFilePath(repositoryRoot) + if err != nil { + return nil, fmt.Errorf("locating links file failed: %w", err) + } + readmeFiles, err := filepath.Glob(filepath.Join(packageRoot, "_dev", "build", "docs", "*.md")) if err != nil { return nil, fmt.Errorf("reading directory entries failed: %w", err) @@ -108,7 +114,7 @@ func UpdateReadmes(packageRoot, buildDir string) ([]string, error) { var targets []string for _, filePath := range readmeFiles { fileName := filepath.Base(filePath) - target, err := updateReadme(fileName, packageRoot, buildDir) + target, err := updateReadme(fileName, linksFilePath, packageRoot, buildDir) if err != nil { return nil, fmt.Errorf("updating readme file %s failed: %w", fileName, err) } @@ -120,10 +126,17 @@ func UpdateReadmes(packageRoot, buildDir string) ([]string, error) { return targets, nil } -func updateReadme(fileName, packageRoot, buildDir string) (string, error) { +// updateReadme function updates a single readme file using a defined template file. +// It writes the rendered file to both the package directory and the package build directory. +func updateReadme(fileName, linksFilePath, packageRoot, buildDir string) (string, error) { logger.Debugf("Update the %s file", fileName) - rendered, shouldBeRendered, err := generateReadme(fileName, packageRoot) + packageBuildRoot, err := builder.BuildPackagesDirectory(packageRoot, buildDir) + if err != nil { + return "", fmt.Errorf("package build root not found: %w", err) + } + + rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot, packageBuildRoot) if err != nil { return "", err } @@ -136,11 +149,6 @@ func updateReadme(fileName, packageRoot, buildDir string) (string, error) { return "", fmt.Errorf("writing %s file failed: %w", fileName, err) } - packageBuildRoot, err := builder.BuildPackagesDirectory(packageRoot, buildDir) - if err != nil { - return "", fmt.Errorf("package build root not found: %w", err) - } - _, err = writeReadme(fileName, packageBuildRoot, rendered) if err != nil { return "", fmt.Errorf("writing %s file failed: %w", fileName, err) @@ -148,7 +156,12 @@ func updateReadme(fileName, packageRoot, buildDir string) (string, error) { return target, nil } -func generateReadme(fileName, packageRoot string) ([]byte, bool, error) { +// generateReadme function generates the readme file content +// the readme takes a template that lives under the _dev/build/docs directory at the package root. +// the readme template reads data from the sourceFilesRoot directory. +// sourceFilesRoot is usually the package root when generating readme for checking up-to-dateness, +// and the built package root when generating readme for the built package. +func generateReadme(fileName, linksFilePath, packageRoot, sourceFilesRoot string) ([]byte, bool, error) { logger.Debugf("Generate %s file (package: %s)", fileName, packageRoot) templatePath, found, err := findReadmeTemplatePath(fileName, packageRoot) if err != nil { @@ -160,18 +173,21 @@ func generateReadme(fileName, packageRoot string) ([]byte, bool, error) { } logger.Debugf("Template file for %s found: %s", fileName, templatePath) - linksMap, err := readLinksMap() + linksMap, err := readLinksMap(linksFilePath) if err != nil { return nil, false, err } - rendered, err := renderReadme(fileName, packageRoot, templatePath, linksMap) + // templatePath lives under the _dev/build/docs directory at the package root. + // builtPackageRoot is the root directory of the built package. + rendered, err := renderReadme(fileName, sourceFilesRoot, templatePath, linksMap) if err != nil { return nil, true, fmt.Errorf("rendering Readme failed: %w", err) } return rendered, true, nil } +// findReadmeTemplatePath function looks for the README template file in the _dev/build/docs directory. func findReadmeTemplatePath(fileName, packageRoot string) (string, bool, error) { templatePath := filepath.Join(packageRoot, "_dev", "build", "docs", fileName) _, err := os.Stat(templatePath) @@ -184,23 +200,24 @@ func findReadmeTemplatePath(fileName, packageRoot string) (string, bool, error) return templatePath, true, nil } -func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) ([]byte, error) { - logger.Debugf("Render %s file (package: %s, templatePath: %s)", fileName, packageRoot, templatePath) +// renderReadme function renders the readme file reading from +func renderReadme(fileName, sourceFilesRoot, templatePath string, linksMap linkMap) ([]byte, error) { + logger.Debugf("Render %s file (package: %s, templatePath: %s)", fileName, sourceFilesRoot, templatePath) t := template.New(fileName) t, err := t.Funcs(template.FuncMap{ "event": func(args ...string) (string, error) { if len(args) > 0 { - return renderSampleEvent(packageRoot, args[0]) + return renderSampleEvent(sourceFilesRoot, args[0]) } - return renderSampleEvent(packageRoot, "") + return renderSampleEvent(sourceFilesRoot, "") }, "fields": func(args ...string) (string, error) { if len(args) > 0 { - dataStreamPath := filepath.Join(packageRoot, "data_stream", args[0]) + dataStreamPath := filepath.Join(sourceFilesRoot, "data_stream", args[0]) return renderExportedFields(dataStreamPath) } - return renderExportedFields(packageRoot) + return renderExportedFields(sourceFilesRoot) }, "url": func(args ...string) (string, error) { options := linkOptions{} @@ -210,7 +227,7 @@ func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) return linksMap.RenderLink(args[0], options) }, "inputDocs": func() (string, error) { - return renderInputDocs(packageRoot) + return renderInputDocs(sourceFilesRoot) }, "generatedHeader": func() string { return doNotModifyStr diff --git a/internal/docs/readme_test.go b/internal/docs/readme_test.go index d63c069b0b..a06be43071 100644 --- a/internal/docs/readme_test.go +++ b/internal/docs/readme_test.go @@ -18,15 +18,13 @@ import ( func TestGenerateReadme(t *testing.T) { cases := []struct { title string - packageRoot string filename string readmeTemplateContents string expected string }{ { - title: "Pure markdown", - packageRoot: t.TempDir(), - filename: "README.md", + title: "Pure markdown", + filename: "README.md", readmeTemplateContents: ` # README Introduction to the package`, @@ -35,9 +33,8 @@ Introduction to the package`, Introduction to the package`, }, { - title: "Generated headers", - packageRoot: t.TempDir(), - filename: "README.md", + title: "Generated headers", + filename: "README.md", readmeTemplateContents: ` {{- generatedHeader }} # README @@ -49,7 +46,6 @@ Introduction to the package`, }, { title: "Static README", - packageRoot: t.TempDir(), filename: "README.md", readmeTemplateContents: "", expected: "", @@ -57,10 +53,12 @@ Introduction to the package`, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { - err := createReadmeFile(c.packageRoot, c.readmeTemplateContents) + + dir := t.TempDir() + err := createReadmeFile(dir, c.readmeTemplateContents) require.NoError(t, err) - rendered, isTemplate, err := generateReadme(c.filename, c.packageRoot) + rendered, isTemplate, err := generateReadme(c.filename, "", dir, dir) require.NoError(t, err) if c.readmeTemplateContents != "" { @@ -76,7 +74,7 @@ Introduction to the package`, } func TestRenderReadmeWithLinks(t *testing.T) { - minimumLinksMap := newLinkMap() + minimumLinksMap := newEmptyLinkMap() minimumLinksMap.Add("foo", "http://www.example.com/bar") cases := []struct { @@ -161,7 +159,7 @@ An example event for ` + "`example`" + ` looks as following: }, } - linksMap := newLinkMap() + linksMap := newEmptyLinkMap() for _, c := range cases { t.Run(c.title, func(t *testing.T) { filename := filepath.Base(c.templatePath) @@ -266,7 +264,7 @@ Introduction to the package }, } - linksMap := newLinkMap() + linksMap := newEmptyLinkMap() for _, c := range cases { t.Run(c.title, func(t *testing.T) { filename := filepath.Base(c.templatePath) diff --git a/internal/elasticsearch/ingest/datastream.go b/internal/elasticsearch/ingest/datastream.go index bb6a4115df..3bd9280252 100644 --- a/internal/elasticsearch/ingest/datastream.go +++ b/internal/elasticsearch/ingest/datastream.go @@ -49,7 +49,7 @@ type RerouteProcessor struct { Namespace []string `yaml:"namespace"` } -func InstallDataStreamPipelines(ctx context.Context, api *elasticsearch.API, dataStreamPath string) (string, []Pipeline, error) { +func InstallDataStreamPipelines(ctx context.Context, api *elasticsearch.API, dataStreamPath string, repositoryRoot *os.Root) (string, []Pipeline, error) { dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(dataStreamPath, packages.DataStreamManifestFile)) if err != nil { return "", nil, fmt.Errorf("reading data stream manifest failed: %w", err) @@ -58,7 +58,7 @@ func InstallDataStreamPipelines(ctx context.Context, api *elasticsearch.API, dat nonce := time.Now().UnixNano() mainPipeline := GetPipelineNameWithNonce(dataStreamManifest.GetPipelineNameOrDefault(), nonce) - pipelines, err := LoadIngestPipelineFiles(dataStreamPath, nonce) + pipelines, err := LoadIngestPipelineFiles(dataStreamPath, nonce, repositoryRoot) if err != nil { return "", nil, fmt.Errorf("loading ingest pipeline files failed: %w", err) } @@ -73,7 +73,7 @@ func InstallDataStreamPipelines(ctx context.Context, api *elasticsearch.API, dat // LoadIngestPipelineFiles returns the set of pipelines found in the directory // elasticsearch/ingest_pipeline under the provided data stream path. The names // of the pipelines are decorated with the provided nonce. -func LoadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, error) { +func LoadIngestPipelineFiles(dataStreamPath string, nonce int64, repositoryRoot *os.Root) ([]Pipeline, error) { elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline") var pipelineFiles []string @@ -85,7 +85,7 @@ func LoadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er pipelineFiles = append(pipelineFiles, files...) } - linksFS, err := files.CreateLinksFSFromPath(elasticsearchPath) + linksFS, err := files.CreateLinksFSFromPath(repositoryRoot, elasticsearchPath) if err != nil { return nil, fmt.Errorf("creating links filesystem failed: %w", err) } diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 95563ec641..8cce1a1f01 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -257,12 +257,12 @@ func WithOTELValidation(otelValidation bool) ValidatorOption { } type packageRootFinder interface { - FindPackageRoot() (string, bool, error) + FindPackageRoot() (string, error) } type packageRoot struct{} -func (p packageRoot) FindPackageRoot() (string, bool, error) { +func (p packageRoot) FindPackageRoot() (string, error) { return packages.FindPackageRoot() } @@ -288,13 +288,14 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa var fdm *DependencyManager if !v.disabledDependencyManagement { - packageRoot, found, err := finder.FindPackageRoot() + packageRoot, err := finder.FindPackageRoot() if err != nil { + if errors.Is(err, packages.ErrPackageRootNotFound) { + return nil, errors.New("package root not found and dependency management is enabled") + } return nil, fmt.Errorf("can't find package root: %w", err) } - if !found { - return nil, errors.New("package root not found and dependency management is enabled") - } + fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema) if err != nil { return nil, fmt.Errorf("failed to initialize dependency management: %w", err) diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 5f3a9a4b10..dcbf5c5e77 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -28,8 +28,8 @@ type packageRootTestFinder struct { packageRootPath string } -func (p packageRootTestFinder) FindPackageRoot() (string, bool, error) { - return p.packageRootPath, true, nil +func (p packageRootTestFinder) FindPackageRoot() (string, error) { + return p.packageRootPath, nil } func TestValidate_NoWildcardFields(t *testing.T) { diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go index 597999b50f..ad9c25fa29 100644 --- a/internal/files/linkedfiles.go +++ b/internal/files/linkedfiles.go @@ -20,6 +20,13 @@ import ( "github.com/elastic/elastic-package/internal/packages" ) +var ( + errEmptyWorkDir = fmt.Errorf("working directory is empty") + errInvalidWorkDir = fmt.Errorf("working directory must be an absolute path or a path relative to the repository root") + errInvalidWorkDirNotDir = fmt.Errorf("working directory is not a directory") + errFileNotUpToDate = fmt.Errorf("linked file is not up to date") +) + const linkExtension = ".link" // PackageLinks represents linked files grouped by package. @@ -29,23 +36,34 @@ type PackageLinks struct { } // CreateLinksFSFromPath creates a LinksFS for the given directory within the repository. -func CreateLinksFSFromPath(workDir string) (*LinksFS, error) { - repoRoot, err := FindRepositoryRootDirectory() - if err != nil { - return nil, fmt.Errorf("finding repository root: %w", err) +// +// - workDir can be an absolute path or a path relative to the repository root. +// in both cases, it must point to a directory within the repository. +func CreateLinksFSFromPath(repositoryRoot *os.Root, workDir string) (*LinksFS, error) { + if workDir == "" { + return nil, errEmptyWorkDir } - root, err := os.OpenRoot(repoRoot) - if err != nil { - return nil, fmt.Errorf("opening repository root: %w", err) + var relWorkDir string + if filepath.IsAbs(workDir) { + var err error + relWorkDir, err = filepath.Rel(repositoryRoot.Name(), workDir) + if err != nil { + return nil, fmt.Errorf("unable to find rel path for %s: %w: %w", workDir, errInvalidWorkDir, err) + } + } else { + relWorkDir = workDir } - absWorkDir, err := filepath.Abs(workDir) + info, err := repositoryRoot.Stat(relWorkDir) if err != nil { - return nil, fmt.Errorf("obtaining absolute path of working directory: %w", err) + return nil, fmt.Errorf("unable to stat %s: %w: %w", relWorkDir, errInvalidWorkDir, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("working directory %s is not a directory: %w", relWorkDir, errInvalidWorkDirNotDir) } - return NewLinksFS(root, absWorkDir) + return &LinksFS{repositoryRoot: repositoryRoot, workDir: relWorkDir}, nil } var _ fs.FS = (*LinksFS)(nil) @@ -56,80 +74,63 @@ var _ fs.FS = (*LinksFS)(nil) // and its checksum. If the included file is up to date, it returns the included file. // Otherwise, it returns an error. type LinksFS struct { - repoRoot *os.Root // The root of the repository, used to check if paths are within the repository. - workDir string - inner fs.FS + repositoryRoot *os.Root // The root of the repository, used to check if paths are within the repository. + workDir string } -// NewLinksFS creates a new LinksFS. workDir must be an absolute path, or a path relative to -// the repository root. -func NewLinksFS(repoRoot *os.Root, workDir string) (*LinksFS, error) { - // Ensure workDir is absolute for os.DirFS - var absWorkDir string - if filepath.IsAbs(workDir) { - absWorkDir = workDir - relative, err := filepath.Rel(repoRoot.Name(), absWorkDir) - if err != nil { - return nil, fmt.Errorf("invalid working directory %s: %w", absWorkDir, err) - } - workDir = relative - } else { - absWorkDir = filepath.Clean(filepath.Join(repoRoot.Name(), workDir)) - } - - info, err := repoRoot.Stat(workDir) +// Open opens a file in the filesystem. +// +// - name can be an absolute path or a path relative to the workDir. +// If name is absolute, it must be within the workDir. +func (lfs *LinksFS) Open(name string) (fs.File, error) { + // innerRoot is the filesystem rooted at the workDir + // we use it to check if the file exists in the workDir + // and to open non-link files directly + innerRoot, err := lfs.repositoryRoot.OpenRoot(lfs.workDir) if err != nil { - return nil, fmt.Errorf("invalid working directory %s: %w", absWorkDir, err) + return nil, fmt.Errorf("could not open workDir in root: %w", err) } - if !info.IsDir() { - return nil, fmt.Errorf("working directory %s is not a directory", absWorkDir) - } - - return &LinksFS{repoRoot: repoRoot, workDir: absWorkDir, inner: os.DirFS(absWorkDir)}, nil -} + defer innerRoot.Close() -// Open opens a file in the filesystem. -func (lfs *LinksFS) Open(name string) (fs.File, error) { - // Ensure name is relative for os.DirFS compatibility - var relativeName string + relName := name if filepath.IsAbs(name) { var err error - relativeName, err = filepath.Rel(lfs.workDir, name) + relName, err = filepath.Rel(filepath.Join(lfs.repositoryRoot.Name(), lfs.workDir), name) if err != nil { return nil, fmt.Errorf("could not make name relative to workDir: %w", err) } - } else { - relativeName = name + } + _, err = innerRoot.Stat(relName) + if err != nil { + return nil, fmt.Errorf("file %s not found in workDir %s: %w", relName, lfs.workDir, err) } // For non-link files, use the inner filesystem - if filepath.Ext(relativeName) != linkExtension { - return lfs.inner.Open(relativeName) + if filepath.Ext(relName) != linkExtension { + return innerRoot.Open(relName) } - // For link files, construct the absolute path to the link file - // Since workDir is expected to be absolute, we can directly join - linkFilePath := filepath.Join(lfs.workDir, relativeName) - - l, err := newLinkedFile(lfs.repoRoot, linkFilePath) + linkFilePath := filepath.Join(lfs.repositoryRoot.Name(), lfs.workDir, relName) + l, err := newLinkedFile(lfs.repositoryRoot, linkFilePath) if err != nil { return nil, err } if !l.UpToDate { - return nil, fmt.Errorf("linked file %s is not up to date", relativeName) + return nil, fmt.Errorf("%w: file %s", errFileNotUpToDate, relName) } - // Calculate the included file path relative to the link file's directory - linkDir := filepath.Dir(linkFilePath) - includedPath := filepath.Join(linkDir, l.IncludedFilePath) + // includedPath is the absolute path to the included file referenced at the link file + // inside a link file, the path to the included file is relative to the link file location + // so we need to join the directory of the link file with the included file path + includedPath := filepath.Join(filepath.Dir(linkFilePath), filepath.FromSlash(l.IncludedFilePath)) // Convert to relative path from repository root for secure access of target file - relativePath, err := filepath.Rel(lfs.repoRoot.Name(), includedPath) + relativePath, err := filepath.Rel(lfs.repositoryRoot.Name(), includedPath) if err != nil { return nil, fmt.Errorf("could not get relative path: %w", err) } - return lfs.repoRoot.Open(relativePath) + return lfs.repositoryRoot.Open(relativePath) } // ReadFile reads a file from the filesystem. @@ -145,105 +146,127 @@ func (lfs *LinksFS) ReadFile(name string) ([]byte, error) { // CheckLinkedFiles checks if all linked files in the directory are up-to-date. // Returns a list of outdated links that need updating. func (lfs *LinksFS) CheckLinkedFiles() ([]Link, error) { - return areLinkedFilesUpToDate(lfs.repoRoot, lfs.workDir) + return areLinkedFilesUpToDate(lfs.repositoryRoot, lfs.workDir) } // UpdateLinkedFiles updates the checksums of all outdated linked files in the directory. // Returns a list of links that were updated. func (lfs *LinksFS) UpdateLinkedFiles() ([]Link, error) { - return updateLinkedFilesChecksums(lfs.repoRoot, lfs.workDir) + return updateLinkedFilesChecksums(lfs.repositoryRoot, lfs.workDir) } // IncludeLinkedFiles copies all linked files from the source directory to the target directory. // This is used during package building to include linked files in the build output. func (lfs *LinksFS) IncludeLinkedFiles(toDir string) ([]Link, error) { - return includeLinkedFiles(lfs.repoRoot, lfs.workDir, toDir) + return includeLinkedFiles(lfs.repositoryRoot, lfs.workDir, toDir) } // ListLinkedFilesByPackage returns a mapping of packages to their linked files that reference // files from the given directory. func (lfs *LinksFS) ListLinkedFilesByPackage() ([]PackageLinks, error) { - return linkedFilesByPackageFrom(lfs.repoRoot, lfs.workDir) + return linkedFilesByPackageFrom(lfs.repositoryRoot, lfs.workDir) } // A Link represents a linked file. -// It contains the path to the link file, the checksum of the link file, -// the path to the included file, and the checksum of the included file contents. -// It also contains a boolean indicating whether the link is up to date. +// A linked file is a file with the ".link" extension that contains a reference to another file in an other package. +// The link file contains the relative path to the included file and an optional checksum of the included file contents. type Link struct { - WorkDir string + WorkDir string // WorkDir is the path to the directory containing the link file. This is where the copy of the included file will be placed. - LinkFilePath string + LinkFilePath string // LinkFilePath is the absolute path of the linked file LinkChecksum string - LinkPackageName string + LinkPackageName string // Package where the link file is located + + TargetRelPath string // TargetRelPath is the relative path to the target file, this will be the path where the content of the file is copied to - IncludedFilePath string - IncludedFileContentsChecksum string - IncludedPackageName string + IncludedFilePath string // IncludedFilePath is the path to the included file, this is the content of the link file + IncludedFileContentsChecksum string // IncludedFileContentsChecksum is the checksum of the included file contents, this is the second field in the link file + IncludedPackageName string // IncludedPackageName is the package where the included file is located - UpToDate bool + UpToDate bool // UpToDate indicates whether the content of the included file matches the checksum in the link file } -// NewLinkedFile creates a new Link from the given link file path. -func newLinkedFile(root *os.Root, linkFilePath string) (Link, error) { - var l Link - l.WorkDir = filepath.Dir(linkFilePath) - if linkPackageRoot, _, _ := packages.FindPackageRootFrom(l.WorkDir); linkPackageRoot != "" { - l.LinkPackageName = filepath.Base(linkPackageRoot) - } +// newLinkedFile creates a new Link struct from the given absolute path to a link file. +// repositoryRoot is the repository root, used to validate paths and access files securely +func newLinkedFile(repositoryRoot *os.Root, linkFilePath string) (*Link, error) { - firstLine, err := readFirstLine(linkFilePath) + workDir := filepath.Dir(linkFilePath) + + var linkPackageName string + linkPackageRoot, err := packages.FindPackageRootFrom(workDir) if err != nil { - return Link{}, err + return nil, fmt.Errorf("could not find package root for link file %s: %w", linkFilePath, err) + } + if linkPackageRoot != "" { + linkPackageName = filepath.Base(linkPackageRoot) + } else { + // if the link file is not in a package, we consider the workdir as the package root + linkPackageRoot = workDir } - l.LinkFilePath, err = filepath.Rel(l.WorkDir, linkFilePath) + + linkFileRelativePath, err := filepath.Rel(linkPackageRoot, linkFilePath) if err != nil { - return Link{}, fmt.Errorf("could not get relative path: %w", err) + return nil, fmt.Errorf("could not get relative path: %w", err) } + // read the content of the .link file, extract the included file path and checksum + firstLine, err := readFirstLine(linkFilePath) + if err != nil { + return nil, err + } fields := strings.Fields(firstLine) if len(fields) == 0 { - return Link{}, fmt.Errorf("link file %s is empty or has no valid content", linkFilePath) - } - if len(fields) > 2 { - return Link{}, fmt.Errorf("link file %s has invalid format: expected 1 or 2 fields, got %d", linkFilePath, len(fields)) + return nil, fmt.Errorf("link file %s is empty or has no valid content", linkFilePath) + } else if len(fields) > 2 { + return nil, fmt.Errorf("link file %s has invalid format: expected 1 or 2 fields, got %d", linkFilePath, len(fields)) } - l.IncludedFilePath = fields[0] + includedFileRelPath := fields[0] + // having the checksum is optional + var linkfileChecksum string if len(fields) == 2 { - l.LinkChecksum = fields[1] + linkfileChecksum = fields[1] } - pathName := filepath.Clean(filepath.Join(l.WorkDir, filepath.FromSlash(l.IncludedFilePath))) + // includedFilePath represents the absolute path to the included file which content we want to copy to the link file + // resolves the path of the included file relative to the link file + includedFilePath := filepath.Clean(filepath.Join(workDir, filepath.FromSlash(includedFileRelPath))) - // Store the original absolute path for package root detection - originalAbsPath := pathName - - // Convert to relative path for secure access of target file - if filepath.IsAbs(pathName) { - pathName, err = filepath.Rel(root.Name(), pathName) - if err != nil { - return Link{}, fmt.Errorf("could not get relative path: %w", err) - } - } - - if _, err := root.Stat(pathName); err != nil { - return Link{}, err - } - - cs, err := getLinkedFileChecksumFromRoot(root, pathName) + // get relative path of the included file from the root (packages root or repository root) + includedFilePathRelFromRoot, err := filepath.Rel(repositoryRoot.Name(), includedFilePath) if err != nil { - return Link{}, fmt.Errorf("could not collect file %s: %w", l.IncludedFilePath, err) + return nil, fmt.Errorf("could not get relative path: %w", err) } - if l.LinkChecksum == cs { - l.UpToDate = true + // check the file exists + if _, err := repositoryRoot.Stat(includedFilePathRelFromRoot); err != nil { + return nil, err } - l.IncludedFileContentsChecksum = cs - if includedPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Dir(originalAbsPath)); includedPackageRoot != "" { - l.IncludedPackageName = filepath.Base(includedPackageRoot) + // check if checksum is updated + cs, err := getLinkedFileChecksumFromRoot(repositoryRoot, includedFilePathRelFromRoot) + if err != nil { + return nil, fmt.Errorf("could not collect file %s: %w", includedFilePathRelFromRoot, err) } - return l, nil + var includedPackageName string + includedPackageRoot, err := packages.FindPackageRootFrom(filepath.Dir(includedFilePath)) + if err != nil { + return nil, fmt.Errorf("could not find package root for included file %s: %w", includedFilePath, err) + } + if includedPackageRoot != "" { + includedPackageName = filepath.Base(includedPackageRoot) + } + + return &Link{ + WorkDir: workDir, + LinkFilePath: linkFilePath, + LinkChecksum: linkfileChecksum, + LinkPackageName: linkPackageName, + IncludedFilePath: includedFileRelPath, + IncludedFileContentsChecksum: cs, + IncludedPackageName: includedPackageName, + UpToDate: cs == linkfileChecksum, + TargetRelPath: strings.TrimSuffix(linkFileRelativePath, linkExtension), + }, nil } // updateChecksum function updates the checksum of the linked file. @@ -259,7 +282,7 @@ func (l *Link) updateChecksum() (bool, error) { return false, fmt.Errorf("checksum is empty for included file %s", l.IncludedFilePath) } newContent := fmt.Sprintf("%v %v", filepath.ToSlash(l.IncludedFilePath), l.IncludedFileContentsChecksum) - if err := writeFile(filepath.Join(l.WorkDir, l.LinkFilePath), []byte(newContent)); err != nil { + if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { return false, fmt.Errorf("could not update checksum for link file %s: %w", l.LinkFilePath, err) } l.LinkChecksum = l.IncludedFileContentsChecksum @@ -267,24 +290,11 @@ func (l *Link) updateChecksum() (bool, error) { return true, nil } -// TargetFilePath returns the path where the linked file should be written. -// If workDir is provided, it uses that as the base directory, otherwise uses the link's WorkDir. -func (l *Link) TargetFilePath(workDir ...string) string { - targetFilePath := filepath.FromSlash(strings.TrimSuffix(l.LinkFilePath, linkExtension)) - wd := l.WorkDir - if len(workDir) > 0 { - wd = workDir[0] - } - return filepath.Join(wd, targetFilePath) -} - -// includeLinkedFiles function includes linked files from the source -// directory to the target directory. +// includeLinkedFiles function includes linked files from the source directory to the target directory. // It returns a slice of Link structs representing the included files. // It also updates the checksum of the linked files. -// Both directories must be relative to the root. -func includeLinkedFiles(root *os.Root, fromDir, toDir string) ([]Link, error) { - links, err := listLinkedFiles(root, fromDir) +func includeLinkedFiles(repositoryRoot *os.Root, fromDir, toDir string) ([]Link, error) { + links, err := listLinkedFiles(repositoryRoot, fromDir) if err != nil { return nil, fmt.Errorf("including linked files failed: %w", err) } @@ -292,9 +302,12 @@ func includeLinkedFiles(root *os.Root, fromDir, toDir string) ([]Link, error) { if _, err := l.updateChecksum(); err != nil { return nil, fmt.Errorf("could not update checksum for file %s: %w", l.LinkFilePath, err) } - targetFilePath := l.TargetFilePath(toDir) + // targetFilePath is the path where the content of the file is copied to + targetFilePath := filepath.Join(toDir, l.TargetRelPath) + + // from l.IncludedFilePath, we just need the path name without .link suffix and to be relative to the toDir instead of the package root if err := copyFromRoot( - root, + repositoryRoot, filepath.Join(l.WorkDir, filepath.FromSlash(l.IncludedFilePath)), targetFilePath, ); err != nil { @@ -305,88 +318,91 @@ func includeLinkedFiles(root *os.Root, fromDir, toDir string) ([]Link, error) { return links, nil } -// listLinkedFiles function returns a slice of Link structs representing linked files. -func listLinkedFiles(root *os.Root, fromDir string) ([]Link, error) { - var linkFiles []string +// listLinkedFiles returns a slice of Link structs representing linked files +// within the given directory. +// +// - fromDir should be relative to the repository root. +func listLinkedFiles(repositoryRoot *os.Root, fromDir string) ([]Link, error) { + var linkFilesPaths []string if err := filepath.Walk( - filepath.FromSlash(fromDir), + filepath.Join(repositoryRoot.Name(), fromDir), func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() && strings.HasSuffix(info.Name(), linkExtension) { - linkFiles = append(linkFiles, path) + linkFilesPaths = append(linkFilesPaths, path) } return nil }); err != nil { return nil, err } - links := make([]Link, len(linkFiles)) + links := make([]Link, len(linkFilesPaths)) - for i, f := range linkFiles { - l, err := newLinkedFile(root, filepath.FromSlash(f)) + for i, f := range linkFilesPaths { + l, err := newLinkedFile(repositoryRoot, filepath.FromSlash(f)) if err != nil { return nil, fmt.Errorf("could not initialize linked file %s: %w", f, err) } - links[i] = l + links[i] = *l } return links, nil } // createDirInRoot function creates a directory and all its parents within the root. -func createDirInRoot(root *os.Root, dir string) error { +func createDirInRoot(repositoryRoot *os.Root, dir string) error { dir = filepath.Clean(dir) if dir == "." || dir == "/" { return nil } // Check if the directory already exists - if _, err := root.Stat(dir); err == nil { + if _, err := repositoryRoot.Stat(dir); err == nil { return nil } // Create parent directory first parent := filepath.Dir(dir) if parent != dir { // Avoid infinite recursion - if err := createDirInRoot(root, parent); err != nil { + if err := createDirInRoot(repositoryRoot, parent); err != nil { return err } } // Create the directory - return root.Mkdir(dir, 0700) + return repositoryRoot.Mkdir(dir, 0700) } // copyFromRoot function copies a file from to to inside the root. -func copyFromRoot(root *os.Root, from, to string) error { +func copyFromRoot(repositoryRoot *os.Root, from, to string) error { var err error if filepath.IsAbs(from) { - from, err = filepath.Rel(root.Name(), filepath.FromSlash(from)) + from, err = filepath.Rel(repositoryRoot.Name(), filepath.FromSlash(from)) if err != nil { return fmt.Errorf("could not get relative path: %w", err) } } - source, err := root.Open(from) + source, err := repositoryRoot.Open(from) if err != nil { return err } defer source.Close() if filepath.IsAbs(to) { - to, err = filepath.Rel(root.Name(), filepath.FromSlash(to)) + to, err = filepath.Rel(repositoryRoot.Name(), filepath.FromSlash(to)) if err != nil { return fmt.Errorf("could not get relative path: %w", err) } } dir := filepath.Dir(to) - if _, err := root.Stat(dir); os.IsNotExist(err) { - if err := createDirInRoot(root, dir); err != nil { + if _, err := repositoryRoot.Stat(dir); os.IsNotExist(err) { + if err := createDirInRoot(repositoryRoot, dir); err != nil { return err } } - destination, err := root.Create(to) + destination, err := repositoryRoot.Create(to) if err != nil { return err } @@ -408,8 +424,8 @@ func writeFile(to string, b []byte) error { } // areLinkedFilesUpToDate function checks if all the linked files are up-to-date. -func areLinkedFilesUpToDate(root *os.Root, fromDir string) ([]Link, error) { - links, err := listLinkedFiles(root, fromDir) +func areLinkedFilesUpToDate(repositoryRoot *os.Root, fromDir string) ([]Link, error) { + links, err := listLinkedFiles(repositoryRoot, fromDir) if err != nil { return nil, fmt.Errorf("checking linked files failed: %w", err) } @@ -428,8 +444,8 @@ func areLinkedFilesUpToDate(root *os.Root, fromDir string) ([]Link, error) { // updateLinkedFilesChecksums function updates the checksums of the linked files. // It returns a slice of updated links. // If no links were updated, it returns an empty slice. -func updateLinkedFilesChecksums(root *os.Root, fromDir string) ([]Link, error) { - links, err := listLinkedFiles(root, fromDir) +func updateLinkedFilesChecksums(repositoryRoot *os.Root, fromDir string) ([]Link, error) { + links, err := listLinkedFiles(repositoryRoot, fromDir) if err != nil { return nil, fmt.Errorf("updating linked files checksums failed: %w", err) } @@ -450,16 +466,22 @@ func updateLinkedFilesChecksums(root *os.Root, fromDir string) ([]Link, error) { // linkedFilesByPackageFrom function returns a slice of PackageLinks containing linked files grouped by package. // Each PackageLinks contains the package name and a slice of linked file paths. -func linkedFilesByPackageFrom(root *os.Root, fromDir string) ([]PackageLinks, error) { +// +// - fromDir should be relative to the repository root. +func linkedFilesByPackageFrom(repositoryRoot *os.Root, fromDir string) ([]PackageLinks, error) { // we list linked files from all the root directory // to check which ones are linked to the 'fromDir' package - links, err := listLinkedFiles(root, root.Name()) + links, err := listLinkedFiles(repositoryRoot, ".") if err != nil { return nil, fmt.Errorf("listing linked files failed: %w", err) } var packageName string - if packageRoot, _, _ := packages.FindPackageRootFrom(fromDir); packageRoot != "" { + packageRoot, err := packages.FindPackageRootFrom(filepath.Join(repositoryRoot.Name(), fromDir)) + if err != nil { + return nil, fmt.Errorf("finding package root failed: %w", err) + } + if packageRoot != "" { packageName = filepath.Base(packageRoot) } byPackageMap := map[string][]string{} @@ -468,7 +490,7 @@ func linkedFilesByPackageFrom(root *os.Root, fromDir string) ([]PackageLinks, er packageName != l.IncludedPackageName { continue } - byPackageMap[l.LinkPackageName] = append(byPackageMap[l.LinkPackageName], filepath.Join(l.WorkDir, l.LinkFilePath)) + byPackageMap[l.LinkPackageName] = append(byPackageMap[l.LinkPackageName], l.LinkFilePath) } var packages []string @@ -488,8 +510,8 @@ func linkedFilesByPackageFrom(root *os.Root, fromDir string) ([]PackageLinks, er } // getLinkedFileChecksumFromRoot calculates the SHA256 checksum of a file using root-relative access. -func getLinkedFileChecksumFromRoot(root *os.Root, relativePath string) (string, error) { - file, err := root.Open(filepath.FromSlash(relativePath)) +func getLinkedFileChecksumFromRoot(repositoryRoot *os.Root, relativePath string) (string, error) { + file, err := repositoryRoot.Open(filepath.FromSlash(relativePath)) if err != nil { return "", err } diff --git a/internal/files/linkedfiles_test.go b/internal/files/linkedfiles_test.go index f057f955ca..3900a425ab 100644 --- a/internal/files/linkedfiles_test.go +++ b/internal/files/linkedfiles_test.go @@ -90,24 +90,27 @@ func TestListLinkedFiles(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) + fromDir, err := filepath.Rel(root.Name(), basePath) + require.NoError(t, err) + // Use the private function directly for testing - linkedFiles, err := listLinkedFiles(root, basePath) + linkedFiles, err := listLinkedFiles(root, fromDir) require.NoError(t, err) require.NotEmpty(t, linkedFiles) require.Len(t, linkedFiles, 2) // Expect exactly 2 link files in testdata // Verify first file (outdated.yml.link) - should be outdated (no checksum) - assert.Equal(t, "outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.True(t, strings.HasSuffix(linkedFiles[0].LinkFilePath, "outdated.yml.link")) assert.Empty(t, linkedFiles[0].LinkChecksum) // No checksum = outdated - assert.Equal(t, "outdated.yml", linkedFiles[0].TargetFilePath("")) + assert.Equal(t, "outdated.yml", linkedFiles[0].TargetRelPath) assert.Equal(t, "./included.yml", linkedFiles[0].IncludedFilePath) assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) assert.False(t, linkedFiles[0].UpToDate) // Verify second file (uptodate.yml.link) - should be up-to-date (has matching checksum) - assert.Equal(t, "uptodate.yml.link", linkedFiles[1].LinkFilePath) + assert.True(t, strings.HasSuffix(linkedFiles[1].LinkFilePath, "uptodate.yml.link")) assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].LinkChecksum) - assert.Equal(t, "uptodate.yml", linkedFiles[1].TargetFilePath("")) + assert.Equal(t, "uptodate.yml", linkedFiles[1].TargetRelPath) assert.Equal(t, "./included.yml", linkedFiles[1].IncludedFilePath) assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].IncludedFileContentsChecksum) assert.True(t, linkedFiles[1].UpToDate) @@ -161,7 +164,7 @@ func TestAreLinkedFilesUpToDate(t *testing.T) { t.Cleanup(func() { _ = root.Close() }) // Create LinksFS - linksFS, err := NewLinksFS(root, basePath) + linksFS, err := CreateLinksFSFromPath(root, basePath) require.NoError(t, err) // Get all outdated linked files from the test directory @@ -171,9 +174,9 @@ func TestAreLinkedFilesUpToDate(t *testing.T) { assert.Len(t, linkedFiles, 1) // Expect exactly 1 outdated file (outdated.yml.link) // Verify the outdated file details - assert.Equal(t, "outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.True(t, strings.HasSuffix(linkedFiles[0].LinkFilePath, "outdated.yml.link")) assert.Empty(t, linkedFiles[0].LinkChecksum) // No checksum indicates outdated - assert.Equal(t, "outdated.yml", linkedFiles[0].TargetFilePath("")) + assert.Equal(t, "outdated.yml", linkedFiles[0].TargetRelPath) assert.Equal(t, "./included.yml", linkedFiles[0].IncludedFilePath) assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) assert.False(t, linkedFiles[0].UpToDate) @@ -202,7 +205,7 @@ func TestUpdateLinkedFilesChecksums(t *testing.T) { t.Cleanup(func() { _ = root.Close() }) // Create LinksFS - linksFS, err := NewLinksFS(root, basePath) + linksFS, err := CreateLinksFSFromPath(root, basePath) require.NoError(t, err) // Update checksums for all outdated linked files @@ -235,8 +238,11 @@ func TestLinkedFilesByPackageFrom(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) + fromDir, err := filepath.Rel(root.Name(), basePath) + require.NoError(t, err) + // Create LinksFS - linksFS, err := NewLinksFS(root, basePath) + linksFS, err := CreateLinksFSFromPath(root, fromDir) require.NoError(t, err) // Get linked files organized by package @@ -286,19 +292,19 @@ func TestIncludeLinkedFiles(t *testing.T) { t.Cleanup(func() { _ = root.Close() }) // Include (copy) all linked files from source to destination using LinksFS - linksFS, err := NewLinksFS(root, fromDir) + linksFS, err := CreateLinksFSFromPath(root, fromDir) assert.NoError(t, err) linkedFiles, err := linksFS.IncludeLinkedFiles(toDir) assert.NoError(t, err) require.Equal(t, 1, len(linkedFiles)) // Expect 1 linked file to be processed // Verify the target file was created in the destination directory - assert.FileExists(t, linkedFiles[0].TargetFilePath(toDir)) + assert.FileExists(t, filepath.Join(toDir, linkedFiles[0].TargetRelPath)) // Verify the copied file has identical content to the original included file equal, err := filesEqual( filepath.Join(linkedFiles[0].WorkDir, filepath.FromSlash(linkedFiles[0].IncludedFilePath)), - linkedFiles[0].TargetFilePath(toDir), + filepath.Join(toDir, linkedFiles[0].TargetRelPath), ) assert.NoError(t, err) assert.True(t, equal, "files should be equal after copying") @@ -492,6 +498,18 @@ func TestNewLinkedFileRejectsPathTraversal(t *testing.T) { } } +func createPackageStructure(t *testing.T, dirs ...string) string { + packageDir := filepath.Join(dirs...) + err := os.MkdirAll(packageDir, 0755) + require.NoError(t, err) + + manifestFile := filepath.Join(packageDir, "manifest.yml") + err = os.WriteFile(manifestFile, []byte("version: '1.0'\ntype: integration\n"), 0644) + require.NoError(t, err) + + return packageDir +} + func TestLinksFSSecurityIsolation(t *testing.T) { tempDir := t.TempDir() @@ -500,10 +518,7 @@ func TestLinksFSSecurityIsolation(t *testing.T) { err := os.MkdirAll(repoDir, 0755) require.NoError(t, err) - // Create a working directory inside repo - workDir := filepath.Join(repoDir, "work") - err = os.MkdirAll(workDir, 0755) - require.NoError(t, err) + workDir := createPackageStructure(t, repoDir, "testpackage") // Create a valid included file in the repo includedFile := filepath.Join(workDir, "included.txt") @@ -522,12 +537,13 @@ func TestLinksFSSecurityIsolation(t *testing.T) { // Create LinksFS root, err := os.OpenRoot(repoDir) require.NoError(t, err) + defer root.Close() // Get the relative path from repo root to work directory relWorkDir, err := filepath.Rel(repoDir, workDir) require.NoError(t, err) - lfs, err := NewLinksFS(root, relWorkDir) + lfs, err := CreateLinksFSFromPath(root, relWorkDir) require.NoError(t, err) // Test opening the linked file - this should work and use the repository root @@ -560,31 +576,40 @@ func TestLinksFS_Open(t *testing.T) { err := os.MkdirAll(repoDir, 0755) require.NoError(t, err) - workDir := filepath.Join(repoDir, "work") - err = os.MkdirAll(workDir, 0755) - require.NoError(t, err) + workDir := createPackageStructure(t, repoDir, "work") // Create test files - regularFile := filepath.Join(workDir, "regular.txt") - err = os.WriteFile(regularFile, []byte("regular content"), 0644) + regularFileRel := "regular.txt" + regularFileAbs := filepath.Join(workDir, regularFileRel) + err = os.WriteFile(regularFileAbs, []byte("regular content"), 0644) require.NoError(t, err) - includedFile := filepath.Join(workDir, "included.txt") + createPackageStructure(t, repoDir, "other") + + includedFileAbs := filepath.Join(repoDir, "other", "included.txt") + err = os.MkdirAll(filepath.Dir(includedFileAbs), 0755) + require.NoError(t, err) includedContent := "included content" - err = os.WriteFile(includedFile, []byte(includedContent), 0644) + err = os.WriteFile(includedFileAbs, []byte(includedContent), 0644) require.NoError(t, err) + createPackageStructure(t, repoDir, "packages", "packageB") + // Create link file with correct checksum - linkFile := filepath.Join(workDir, "linked.txt.link") + linkFileRel := filepath.Join("packages", "packageB", "linked.txt.link") + linkFileAbs := filepath.Join(workDir, linkFileRel) + err = os.MkdirAll(filepath.Dir(linkFileAbs), 0755) + require.NoError(t, err) hash := sha256.Sum256([]byte(includedContent)) checksum := hex.EncodeToString(hash[:]) - linkContent := fmt.Sprintf("./included.txt %s", checksum) - err = os.WriteFile(linkFile, []byte(linkContent), 0644) + linkContent := fmt.Sprintf("../../../other/included.txt %s", checksum) + err = os.WriteFile(linkFileAbs, []byte(linkContent), 0644) require.NoError(t, err) // Create outdated link file (no checksum) - outdatedLinkFile := filepath.Join(workDir, "outdated.txt.link") - err = os.WriteFile(outdatedLinkFile, []byte("./included.txt"), 0644) + outdatedLinkFileRel := "outdated.txt.link" + outdatedLinkFileAbs := filepath.Join(workDir, outdatedLinkFileRel) + err = os.WriteFile(outdatedLinkFileAbs, []byte("../other/included.txt"), 0644) require.NoError(t, err) // Setup LinksFS with absolute workDir @@ -592,46 +617,49 @@ func TestLinksFS_Open(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) - lfs, err := NewLinksFS(root, workDir) + lfs, err := CreateLinksFSFromPath(root, workDir) require.NoError(t, err) tests := []struct { name string - fileName string - expectError bool - errorMsg string + fileName string // filename should be relative to workDir or absolute + expectedErr error expectFile bool }{ { name: "open regular file with relative path", - fileName: "regular.txt", + fileName: regularFileRel, expectFile: true, }, { name: "open regular file with absolute path", - fileName: filepath.Join(workDir, "regular.txt"), + fileName: regularFileAbs, expectFile: true, }, { name: "open up-to-date link file", - fileName: "linked.txt.link", + fileName: linkFileRel, expectFile: true, }, { name: "open up-to-date link file with absolute path", - fileName: filepath.Join(workDir, "linked.txt.link"), + fileName: linkFileAbs, expectFile: true, }, { name: "open outdated link file should fail", - fileName: "outdated.txt.link", - expectError: true, - errorMsg: "not up to date", + fileName: outdatedLinkFileRel, + expectedErr: errFileNotUpToDate, + }, + { + name: "open outdated link file with absolute pathshould fail", + fileName: outdatedLinkFileAbs, + expectedErr: errFileNotUpToDate, }, { name: "open non-existent file should fail", fileName: "nonexistent.txt", - expectError: true, + expectedErr: os.ErrNotExist, }, } @@ -639,30 +667,27 @@ func TestLinksFS_Open(t *testing.T) { t.Run(tc.name, func(t *testing.T) { file, err := lfs.Open(tc.fileName) - if tc.expectError { - assert.Error(t, err) - if tc.errorMsg != "" { - assert.Contains(t, err.Error(), tc.errorMsg) - } - assert.Nil(t, file) + if tc.expectedErr != nil { + require.Error(t, err) + require.Nil(t, file) + assert.ErrorIs(t, err, tc.expectedErr) } else { + require.NoError(t, err) + require.NotNil(t, file) + + // Verify we can read from the file + content, err := io.ReadAll(file) assert.NoError(t, err) - assert.NotNil(t, file) - if file != nil { - // Verify we can read from the file - content, err := io.ReadAll(file) - assert.NoError(t, err) + // For link files, content should be from the included file + if strings.HasSuffix(tc.fileName, ".link") { + assert.Equal(t, includedContent, string(content)) + } else { + assert.Equal(t, "regular content", string(content)) + } - // For link files, content should be from the included file - if strings.HasSuffix(tc.fileName, ".link") { - assert.Equal(t, includedContent, string(content)) - } else { - assert.Equal(t, "regular content", string(content)) - } + file.Close() - file.Close() - } } }) } @@ -677,9 +702,7 @@ func TestLinksFS_RelativeWorkDir(t *testing.T) { err := os.MkdirAll(repoDir, 0755) require.NoError(t, err) - workDir := filepath.Join(repoDir, "work") - err = os.MkdirAll(workDir, 0755) - require.NoError(t, err) + workDir := createPackageStructure(t, repoDir, "work") // Create test files regularFile := filepath.Join(workDir, "regular.txt") @@ -706,7 +729,7 @@ func TestLinksFS_RelativeWorkDir(t *testing.T) { t.Cleanup(func() { _ = root.Close() }) // Use relative path "work" instead of absolute path - lfs, err := NewLinksFS(root, "work") + lfs, err := CreateLinksFSFromPath(root, "work") require.NoError(t, err) tests := []struct { @@ -756,9 +779,7 @@ func TestLinksFS_ErrorConditions(t *testing.T) { err := os.MkdirAll(repoDir, 0755) require.NoError(t, err) - workDir := filepath.Join(repoDir, "work") - err = os.MkdirAll(workDir, 0755) - require.NoError(t, err) + workDir := createPackageStructure(t, repoDir, "work") // Create link file that points to non-existent file (this will fail security check) brokenLinkFile := filepath.Join(workDir, "broken.txt.link") @@ -780,7 +801,7 @@ func TestLinksFS_ErrorConditions(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) - lfs, err := NewLinksFS(root, workDir) + lfs, err := CreateLinksFSFromPath(root, workDir) require.NoError(t, err) notFoundErrorMsg := "no such file or directory" @@ -873,7 +894,7 @@ func TestLinksFS_WorkDirValidation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - lfs, err := NewLinksFS(root, tc.workDir) + lfs, err := CreateLinksFSFromPath(root, tc.workDir) if tc.expectError { assert.Error(t, err) @@ -886,3 +907,75 @@ func TestLinksFS_WorkDirValidation(t *testing.T) { }) } } + +func TestNewLinkedFile(t *testing.T) { + + repositoryRoot := t.TempDir() + repositoryRootHandle, err := os.OpenRoot(repositoryRoot) + require.NoError(t, err) + t.Cleanup(func() { _ = repositoryRootHandle.Close() }) + + linkFileContent := fmt.Sprintf("%s d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", filepath.Join("..", "..", "B", "otherFolder", "included.txt")) + includedFileContent := "included file content" + + // /packages/A/folder/link.txt.link + // /packages/B/otherFolder/included.txt + // included relative to link file: ../../B/otherFolder/included.txt + + err = os.MkdirAll(filepath.Join(repositoryRoot, "packages", "A", "folder"), 0755) + require.NoError(t, err) + + // required to identify a package root + _, err = repositoryRootHandle.Create(filepath.Join("packages", "A", "manifest.yml")) + require.NoError(t, err) + fManifestA, err := repositoryRootHandle.Create(filepath.Join("packages", "A", "manifest.yml")) + require.NoError(t, err) + _, err = fManifestA.WriteString(`name: A +version: 1.0.0 +type: integration +`) + require.NoError(t, err) + require.NoError(t, fManifestA.Close()) + + fLink, err := repositoryRootHandle.Create(filepath.Join("packages", "A", "folder", "link.txt.link")) + require.NoError(t, err) + _, err = fLink.WriteString(linkFileContent) + require.NoError(t, err) + require.NoError(t, fLink.Close()) + + err = os.MkdirAll(filepath.Join(repositoryRoot, "packages", "B", "otherFolder"), 0755) + require.NoError(t, err) + + // required to identify a package root + _, err = repositoryRootHandle.Create(filepath.Join("packages", "B", "manifest.yml")) + require.NoError(t, err) + fManifestB, err := repositoryRootHandle.Create(filepath.Join("packages", "B", "manifest.yml")) + require.NoError(t, err) + _, err = fManifestB.WriteString(`name: B +version: 1.0.0 +type: integration +`) + require.NoError(t, err) + require.NoError(t, fManifestB.Close()) + + fIncluded, err := repositoryRootHandle.Create(filepath.Join("packages", "B", "otherFolder", "included.txt")) + require.NoError(t, err) + _, err = fIncluded.WriteString(includedFileContent) + require.NoError(t, err) + require.NoError(t, fIncluded.Close()) + + l, err := newLinkedFile(repositoryRootHandle, fLink.Name()) + require.NoError(t, err) + assert.NotNil(t, l) + + assert.Equal(t, filepath.Join(repositoryRoot, filepath.Join("packages", "A", "folder")), l.WorkDir) + + assert.True(t, strings.HasSuffix(l.LinkFilePath, filepath.Join("folder", "link.txt.link"))) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", l.LinkChecksum) + assert.Equal(t, "A", l.LinkPackageName) + + assert.Equal(t, filepath.Join("..", "..", "B", "otherFolder", "included.txt"), l.IncludedFilePath) + assert.Equal(t, "1c14ca1eae312a58c08085436fd5dcd0c02451d3cdc8e4e4a1cc1415a6b4c6d0", l.IncludedFileContentsChecksum) + assert.Equal(t, "B", l.IncludedPackageName) + assert.False(t, l.UpToDate) +} diff --git a/internal/files/testdata/links/manifest.yml b/internal/files/testdata/links/manifest.yml new file mode 100644 index 0000000000..852577ae19 --- /dev/null +++ b/internal/files/testdata/links/manifest.yml @@ -0,0 +1,2 @@ +version: '1.0' +type: integration \ No newline at end of file diff --git a/internal/packages/archetype/data_stream_test.go b/internal/packages/archetype/data_stream_test.go index 30b124f5ed..d6e2deaa25 100644 --- a/internal/packages/archetype/data_stream_test.go +++ b/internal/packages/archetype/data_stream_test.go @@ -5,6 +5,7 @@ package archetype import ( + "os" "path/filepath" "testing" @@ -19,21 +20,39 @@ func TestDataStream(t *testing.T) { dd := createDataStreamDescriptorForTest() dd.Manifest.Type = "logs" - createAndCheckDataStream(t, pd, dd, true) + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + + createAndCheckDataStream(t, pd, dd, true, repositoryRoot) + + err = repositoryRoot.Close() + require.NoError(t, err) }) t.Run("valid-metrics", func(t *testing.T) { pd := createPackageDescriptorForTest("integration", "^7.13.0") dd := createDataStreamDescriptorForTest() dd.Manifest.Type = "metrics" - createAndCheckDataStream(t, pd, dd, true) + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + + createAndCheckDataStream(t, pd, dd, true, repositoryRoot) + + err = repositoryRoot.Close() + require.NoError(t, err) }) t.Run("missing-type", func(t *testing.T) { pd := createPackageDescriptorForTest("integration", "^7.13.0") dd := createDataStreamDescriptorForTest() dd.Manifest.Type = "" - createAndCheckDataStream(t, pd, dd, false) + repositoryRoot, err := os.OpenRoot(t.TempDir()) + require.NoError(t, err) + + createAndCheckDataStream(t, pd, dd, false, repositoryRoot) + + err = repositoryRoot.Close() + require.NoError(t, err) }) } @@ -57,16 +76,20 @@ func createDataStreamDescriptorForTest() DataStreamDescriptor { } } -func createAndCheckDataStream(t *testing.T, pd PackageDescriptor, dd DataStreamDescriptor, valid bool) { - tempDir := makeInRepoBuildTempDir(t) - err := createPackageInDir(pd, tempDir) +func createAndCheckDataStream(t *testing.T, pd PackageDescriptor, dd DataStreamDescriptor, valid bool, repositoryRoot *os.Root) { + + packagesDir := filepath.Join(repositoryRoot.Name(), "packages") + err := os.MkdirAll(packagesDir, 0o755) + require.NoError(t, err) + + err = createPackageInDir(pd, packagesDir) require.NoError(t, err) - packageRoot := filepath.Join(tempDir, pd.Manifest.Name) + packageRoot := filepath.Join(packagesDir, pd.Manifest.Name) dd.PackageRoot = packageRoot err = CreateDataStream(dd) require.NoError(t, err) - checkPackage(t, packageRoot, valid) + checkPackage(t, repositoryRoot, packageRoot, valid) } diff --git a/internal/packages/archetype/package_test.go b/internal/packages/archetype/package_test.go index c3297757c6..5f739d91ac 100644 --- a/internal/packages/archetype/package_test.go +++ b/internal/packages/archetype/package_test.go @@ -38,11 +38,15 @@ func TestPackage(t *testing.T) { } func createAndCheckPackage(t *testing.T, pd PackageDescriptor, valid bool) { - tempDir := makeInRepoBuildTempDir(t) - err := createPackageInDir(pd, tempDir) + repositoryRoot, err := os.OpenRoot(t.TempDir()) require.NoError(t, err) + defer repositoryRoot.Close() - checkPackage(t, filepath.Join(tempDir, pd.Manifest.Name), valid) + packagesDir := filepath.Join(repositoryRoot.Name(), "packages") + err = createPackageInDir(pd, packagesDir) + require.NoError(t, err) + + checkPackage(t, repositoryRoot, filepath.Join(packagesDir, pd.Manifest.Name), valid) } func createPackageDescriptorForTest(packageType, kibanaVersion string) PackageDescriptor { @@ -92,29 +96,32 @@ func createPackageDescriptorForTest(packageType, kibanaVersion string) PackageDe } } -func buildPackage(t *testing.T, packageRoot string) error { - buildDir := makeInRepoBuildTempDir(t) - _, err := docs.UpdateReadmes(packageRoot, buildDir) +func buildPackage(t *testing.T, repositoryRoot *os.Root, packageRootPath string) error { + buildDir := filepath.Join(repositoryRoot.Name(), "build") + err := os.MkdirAll(buildDir, 0o755) + require.NoError(t, err) + _, err = docs.UpdateReadmes(repositoryRoot, packageRootPath, buildDir) if err != nil { return err } _, err = builder.BuildPackage(t.Context(), builder.BuildOptions{ - PackageRoot: packageRoot, - BuildDir: buildDir, + PackageRootPath: packageRootPath, + BuildDir: buildDir, + RepositoryRoot: repositoryRoot, }) return err } -func checkPackage(t *testing.T, packageRoot string, valid bool) { - err := buildPackage(t, packageRoot) +func checkPackage(t *testing.T, repositoryRoot *os.Root, packageRootPath string, valid bool) { + err := buildPackage(t, repositoryRoot, packageRootPath) if !valid { assert.Error(t, err) return } require.NoError(t, err) - manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRootPath) require.NoError(t, err) // Running in subtests because manifest subobjects can be pointers that can panic when dereferenced by assertions. @@ -128,7 +135,7 @@ func checkPackage(t *testing.T, packageRoot string, valid bool) { if manifest.Type == "integration" { t.Run("integration", func(t *testing.T) { - ds, err := filepath.Glob(filepath.Join(packageRoot, "data_stream", "*")) + ds, err := filepath.Glob(filepath.Join(packageRootPath, "data_stream", "*")) require.NoError(t, err) for _, d := range ds { manifest, err := packages.ReadDataStreamManifest(filepath.Join(d, "manifest.yml")) @@ -140,22 +147,3 @@ func checkPackage(t *testing.T, packageRoot string, valid bool) { }) } } - -// makeInRepoBuildTempDir mimicks t.TempDir(), but creates the directory inside the current -// directory. -// FIXME: It should be possible to use t.TempDir(), but conflicts with links resolution, as -// t.TempDir() creates the directory out of the repository. We should refactor links resolution -// so it can write files out of the repository. -// https://github.com/elastic/elastic-package/issues/2797 -func makeInRepoBuildTempDir(t *testing.T) string { - t.Helper() - cwd, err := os.Getwd() - require.NoError(t, err) - dir, err := os.MkdirTemp(cwd, "_build-test-*") - require.NoError(t, err) - t.Cleanup(func() { - err := os.RemoveAll(dir) - assert.NoError(t, err) - }) - return dir -} diff --git a/internal/packages/installer/factory.go b/internal/packages/installer/factory.go index 5229bcacb7..ec278e6c57 100644 --- a/internal/packages/installer/factory.go +++ b/internal/packages/installer/factory.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "os" "github.com/Masterminds/semver/v3" @@ -33,10 +34,11 @@ type Installer interface { // Options are the parameters used to build an installer. type Options struct { - Kibana *kibana.Client - RootPath string - ZipPath string - SkipValidation bool + Kibana *kibana.Client + PackageRootPath string // Root path of the package to be installed. + ZipPath string + SkipValidation bool + RepositoryRoot *os.Root // Root of the repository where package source code is located. } // NewForPackage creates a new installer for a package, given its root path, or its prebuilt zip. @@ -48,9 +50,12 @@ func NewForPackage(ctx context.Context, options Options) (Installer, error) { if options.Kibana == nil { return nil, errors.New("missing kibana client") } - if options.RootPath == "" && options.ZipPath == "" { + if options.PackageRootPath == "" && options.ZipPath == "" { return nil, errors.New("missing package root path or pre-built zip package") } + if options.RepositoryRoot == nil { + return nil, errors.New("missing repository root") + } version, err := kibanaVersion(options.Kibana) if err != nil { @@ -81,10 +86,11 @@ func NewForPackage(ctx context.Context, options Options) (Installer, error) { } target, err := builder.BuildPackage(ctx, builder.BuildOptions{ - PackageRoot: options.RootPath, - CreateZip: supportsUploadZip, - SignPackage: false, - SkipValidation: options.SkipValidation, + PackageRootPath: options.PackageRootPath, + CreateZip: supportsUploadZip, + SignPackage: false, + SkipValidation: options.SkipValidation, + RepositoryRoot: options.RepositoryRoot, }) if err != nil { return nil, fmt.Errorf("failed to build package: %v", err) diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 4e55f9dca7..1cdd157104 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -286,27 +286,30 @@ func (t *Transform) HasSource(name string) (bool, error) { // MustFindPackageRoot finds and returns the path to the root folder of a package. // It fails with an error if the package root can't be found. func MustFindPackageRoot() (string, error) { - root, found, err := FindPackageRoot() + root, err := FindPackageRoot() if err != nil { return "", fmt.Errorf("locating package root failed: %w", err) } - if !found { - return "", errors.New("package root not found") - } return root, nil } // FindPackageRoot finds and returns the path to the root folder of a package from the working directory. -func FindPackageRoot() (string, bool, error) { +func FindPackageRoot() (string, error) { workDir, err := os.Getwd() if err != nil { - return "", false, fmt.Errorf("locating working directory failed: %w", err) + return "", fmt.Errorf("locating working directory failed: %w", err) } return FindPackageRootFrom(workDir) } +var ( + ErrPackageRootNotFound = errors.New("package root not found") +) + // FindPackageRootFrom finds and returns the path to the root folder of a package from a given directory. -func FindPackageRootFrom(fromDir string) (string, bool, error) { +// +// - fromDir should be an absolute path to a directory. +func FindPackageRootFrom(fromDir string) (string, error) { // VolumeName() will return something like "C:" in Windows, and "" in other OSs // rootDir will be something like "C:\" in Windows, and "/" everywhere else. rootDir := filepath.VolumeName(fromDir) + string(filepath.Separator) @@ -318,10 +321,10 @@ func FindPackageRootFrom(fromDir string) (string, bool, error) { if err == nil && !fileInfo.IsDir() { ok, err := isPackageManifest(path) if err != nil { - return "", false, fmt.Errorf("verifying manifest file failed (path: %s): %w", path, err) + return "", fmt.Errorf("verifying manifest file failed (path: %s): %w", path, err) } if ok { - return dir, true, nil + return dir, nil } } @@ -330,7 +333,7 @@ func FindPackageRootFrom(fromDir string) (string, bool, error) { } dir = filepath.Dir(dir) } - return "", false, nil + return "", ErrPackageRootNotFound } // FindDataStreamRootForPath finds and returns the path to the root folder of a data stream. diff --git a/internal/resources/fleetpackage.go b/internal/resources/fleetpackage.go index de7b1eb9d8..7fab894ae2 100644 --- a/internal/resources/fleetpackage.go +++ b/internal/resources/fleetpackage.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "os" "github.com/Masterminds/semver/v3" "github.com/elastic/go-resource" @@ -21,8 +22,11 @@ type FleetPackage struct { // Provider is the name of the provider to use, defaults to "kibana". Provider string - // RootPath is the root of the package source to install. - RootPath string + // PackageRootPath is the root of the package source to install. + PackageRootPath string + + // RepositoryRoot is the root of the repository. + RepositoryRoot *os.Root // Absent is set to true to indicate that the package should not be installed. Absent bool @@ -33,7 +37,7 @@ type FleetPackage struct { } func (f *FleetPackage) String() string { - return fmt.Sprintf("[FleetPackage:%s:%s]", f.Provider, f.RootPath) + return fmt.Sprintf("[FleetPackage:%s:%s]", f.Provider, f.PackageRootPath) } func (f *FleetPackage) provider(ctx resource.Context) (*KibanaProvider, error) { @@ -56,9 +60,10 @@ func (f *FleetPackage) installer(ctx resource.Context) (installer.Installer, err } return installer.NewForPackage(ctx, installer.Options{ - Kibana: provider.Client, - RootPath: f.RootPath, - SkipValidation: true, + Kibana: provider.Client, + PackageRootPath: f.PackageRootPath, + SkipValidation: true, + RepositoryRoot: f.RepositoryRoot, }) } @@ -68,9 +73,9 @@ func (f *FleetPackage) Get(ctx resource.Context) (current resource.ResourceState return nil, err } - manifest, err := packages.ReadPackageManifestFromPackageRoot(f.RootPath) + manifest, err := packages.ReadPackageManifestFromPackageRoot(f.PackageRootPath) if err != nil { - return nil, fmt.Errorf("failed to read manifest from %s: %w", f.RootPath, err) + return nil, fmt.Errorf("failed to read manifest from %s: %w", f.PackageRootPath, err) } fleetPackage, err := provider.Client.GetPackage(ctx, manifest.Name) @@ -112,7 +117,7 @@ func (f *FleetPackage) Create(ctx resource.Context) error { } // Using uninstallPachage instead of f.uninstall because we want to pass a context without cancellation. - uninstallErr = uninstallPackage(context.WithoutCancel(ctx), provider.Client, f.RootPath) + uninstallErr = uninstallPackage(context.WithoutCancel(ctx), provider.Client, f.PackageRootPath) if uninstallErr != nil { return fmt.Errorf("failed to uninstall package (%w) after installation failed: %w", uninstallErr, err) } @@ -129,7 +134,7 @@ func (f *FleetPackage) uninstall(ctx resource.Context) error { return err } - return uninstallPackage(ctx, provider.Client, f.RootPath) + return uninstallPackage(ctx, provider.Client, f.PackageRootPath) } func uninstallPackage(ctx context.Context, client *kibana.Client, rootPath string) error { diff --git a/internal/resources/fleetpackage_test.go b/internal/resources/fleetpackage_test.go index f755807130..7749ad20e6 100644 --- a/internal/resources/fleetpackage_test.go +++ b/internal/resources/fleetpackage_test.go @@ -12,16 +12,24 @@ import ( "github.com/elastic/go-resource" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/kibana" kibanatest "github.com/elastic/elastic-package/internal/kibana/test" ) func TestRequiredProvider(t *testing.T) { manager := resource.NewManager() - _, err := manager.Apply(resource.Resources{ + + repositoryRoot, err := files.FindRepositoryRoot() + require.NoError(t, err) + t.Cleanup(func() { _ = repositoryRoot.Close() }) + + _, err = manager.Apply(resource.Resources{ &FleetPackage{ - RootPath: "../../test/packages/parallel/nginx", + PackageRootPath: "../../test/packages/parallel/nginx", + RepositoryRoot: repositoryRoot, }, }) if assert.Error(t, err) { @@ -46,12 +54,19 @@ func TestPackageLifecycle(t *testing.T) { t.FailNow() } + repositoryRoot, err := files.FindRepositoryRoot() + require.NoError(t, err) + defer repositoryRoot.Close() + + packageRootPath := filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", c.name) + fleetPackage := FleetPackage{ - RootPath: filepath.Join("..", "..", "test", "packages", "parallel", c.name), + PackageRootPath: packageRootPath, + RepositoryRoot: repositoryRoot, } manager := resource.NewManager() manager.RegisterProvider(DefaultKibanaProviderName, &KibanaProvider{Client: kibanaClient}) - _, err := manager.Apply(resource.Resources{&fleetPackage}) + _, err = manager.Apply(resource.Resources{&fleetPackage}) assert.NoError(t, err) assertPackageInstalled(t, kibanaClient, "installed", c.name) @@ -69,15 +84,20 @@ func TestSystemPackageIsNotRemoved(t *testing.T) { t.FailNow() } + repositoryRoot, err := files.FindRepositoryRoot() + require.NoError(t, err) + t.Cleanup(func() { _ = repositoryRoot.Close() }) + fleetPackage := FleetPackage{ - RootPath: "../../test/packages/parallel/system", - Absent: true, + PackageRootPath: "../../test/packages/parallel/system", + Absent: true, + RepositoryRoot: repositoryRoot, } manager := resource.NewManager() manager.RegisterProvider(DefaultKibanaProviderName, &KibanaProvider{Client: kibanaClient}) // Try to uninstall the package, it should not be installed. - _, err := manager.Apply(resource.Resources{&fleetPackage}) + _, err = manager.Apply(resource.Resources{&fleetPackage}) assert.NoError(t, err) assertPackageInstalled(t, kibanaClient, "installed", "system") diff --git a/internal/resources/fleetpolicy.go b/internal/resources/fleetpolicy.go index 61ca16774e..da6abc11a2 100644 --- a/internal/resources/fleetpolicy.go +++ b/internal/resources/fleetpolicy.go @@ -56,9 +56,9 @@ type FleetPackagePolicy struct { // TemplateName is the policy template to use from the package manifest. TemplateName string - // RootPath is the root of the source of the package to configure, from it we should + // PackageRootPath is the root of the source of the package to configure, from it we should // be able to read the manifest, the data stream manifests and the policy template to use. - RootPath string + PackageRootPath string // DataStreamName is the name of the data stream to configure, for integration packages. DataStreamName string @@ -156,9 +156,9 @@ func (f *FleetAgentPolicy) Create(ctx resource.Context) error { } func createPackagePolicy(policy FleetAgentPolicy, packagePolicy FleetPackagePolicy) (*kibana.PackageDataStream, error) { - manifest, err := packages.ReadPackageManifestFromPackageRoot(packagePolicy.RootPath) + manifest, err := packages.ReadPackageManifestFromPackageRoot(packagePolicy.PackageRootPath) if err != nil { - return nil, fmt.Errorf("could not read package manifest at %s: %w", packagePolicy.RootPath, err) + return nil, fmt.Errorf("could not read package manifest at %s: %w", packagePolicy.PackageRootPath, err) } switch manifest.Type { @@ -176,9 +176,9 @@ func createIntegrationPackagePolicy(policy FleetAgentPolicy, manifest packages.P return nil, fmt.Errorf("expected data stream for integration package policy %q", packagePolicy.Name) } - dsManifest, err := packages.ReadDataStreamManifestFromPackageRoot(packagePolicy.RootPath, packagePolicy.DataStreamName) + dsManifest, err := packages.ReadDataStreamManifestFromPackageRoot(packagePolicy.PackageRootPath, packagePolicy.DataStreamName) if err != nil { - return nil, fmt.Errorf("could not read %q data stream manifest for package at %s: %w", packagePolicy.DataStreamName, packagePolicy.RootPath, err) + return nil, fmt.Errorf("could not read %q data stream manifest for package at %s: %w", packagePolicy.DataStreamName, packagePolicy.PackageRootPath, err) } policyTemplateName := packagePolicy.TemplateName diff --git a/internal/resources/fleetpolicy_test.go b/internal/resources/fleetpolicy_test.go index 7c9536f3d0..4d6c547159 100644 --- a/internal/resources/fleetpolicy_test.go +++ b/internal/resources/fleetpolicy_test.go @@ -7,19 +7,26 @@ package resources import ( "errors" "fmt" + "os" "path/filepath" "testing" "github.com/elastic/go-resource" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/kibana" kibanatest "github.com/elastic/elastic-package/internal/kibana/test" ) func TestRequiredProviderFleetPolicy(t *testing.T) { + repositoryRoot, err := files.FindRepositoryRoot() + require.NoError(t, err) + t.Cleanup(func() { _ = repositoryRoot.Close() }) + manager := resource.NewManager() - _, err := manager.Apply(resource.Resources{ + _, err = manager.Apply(resource.Resources{ &FleetAgentPolicy{ Name: "test-policy", }, @@ -30,6 +37,10 @@ func TestRequiredProviderFleetPolicy(t *testing.T) { } func TestPolicyLifecycle(t *testing.T) { + repositoryRoot, err := files.FindRepositoryRoot() + require.NoError(t, err) + t.Cleanup(func() { _ = repositoryRoot.Close() }) + cases := []struct { title string packagePolicies []FleetPackagePolicy @@ -41,9 +52,9 @@ func TestPolicyLifecycle(t *testing.T) { title: "one-package", packagePolicies: []FleetPackagePolicy{ { - Name: "nginx-1", - RootPath: "../../test/packages/parallel/nginx", - DataStreamName: "stubstatus", + Name: "nginx-1", + PackageRootPath: filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", "nginx"), + DataStreamName: "stubstatus", }, }, }, @@ -51,14 +62,14 @@ func TestPolicyLifecycle(t *testing.T) { title: "multiple-packages", packagePolicies: []FleetPackagePolicy{ { - Name: "nginx-1", - RootPath: "../../test/packages/parallel/nginx", - DataStreamName: "stubstatus", + Name: "nginx-1", + PackageRootPath: filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", "nginx"), + DataStreamName: "stubstatus", }, { - Name: "system-1", - RootPath: "../../test/packages/parallel/system", - DataStreamName: "process", + Name: "system-1", + PackageRootPath: filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", "system"), + DataStreamName: "process", }, }, }, @@ -66,8 +77,8 @@ func TestPolicyLifecycle(t *testing.T) { title: "input-package", packagePolicies: []FleetPackagePolicy{ { - Name: "input-1", - RootPath: "../../test/packages/parallel/sql_input", + Name: "input-1", + PackageRootPath: filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", "sql_input"), }, }, }, @@ -89,14 +100,14 @@ func TestPolicyLifecycle(t *testing.T) { Namespace: "eptest", PackagePolicies: c.packagePolicies, } - t.Cleanup(func() { deletePolicy(t, manager, agentPolicy) }) + t.Cleanup(func() { deletePolicy(t, manager, agentPolicy, repositoryRoot) }) - _, err := manager.Apply(withPackageResources(&agentPolicy)) + _, err := manager.Apply(withPackageResources(&agentPolicy, repositoryRoot)) assert.NoError(t, err) assertPolicyPresent(t, kibanaClient, true, agentPolicy.ID) agentPolicy.Absent = true - _, err = manager.Apply(withPackageResources(&agentPolicy)) + _, err = manager.Apply(withPackageResources(&agentPolicy, repositoryRoot)) assert.NoError(t, err) assertPolicyPresent(t, kibanaClient, false, agentPolicy.ID) }) @@ -105,12 +116,13 @@ func TestPolicyLifecycle(t *testing.T) { // withPackageResources prepares a list of resources that ensures that all required packages are installed // before creating the policy. -func withPackageResources(agentPolicy *FleetAgentPolicy) resource.Resources { +func withPackageResources(agentPolicy *FleetAgentPolicy, repostoryRoot *os.Root) resource.Resources { var resources resource.Resources for _, policy := range agentPolicy.PackagePolicies { resources = append(resources, &FleetPackage{ - RootPath: policy.RootPath, - Absent: agentPolicy.Absent, + PackageRootPath: policy.PackageRootPath, + Absent: agentPolicy.Absent, + RepositoryRoot: repostoryRoot, }) } return append(resources, agentPolicy) @@ -131,10 +143,10 @@ func assertPolicyPresent(t *testing.T, client *kibana.Client, expected bool, pol return false } -func deletePolicy(t *testing.T, manager *resource.Manager, agentPolicy FleetAgentPolicy) { +func deletePolicy(t *testing.T, manager *resource.Manager, agentPolicy FleetAgentPolicy, repositoryRoot *os.Root) { t.Helper() agentPolicy.Absent = true - _, err := manager.Apply(withPackageResources(&agentPolicy)) + _, err := manager.Apply(withPackageResources(&agentPolicy, repositoryRoot)) assert.NoError(t, err, "cleanup execution") } diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go index 8f39f8e701..93f90c3508 100644 --- a/internal/testrunner/runners/asset/runner.go +++ b/internal/testrunner/runners/asset/runner.go @@ -6,6 +6,7 @@ package asset import ( "context" + "os" "path/filepath" "github.com/elastic/elastic-package/internal/kibana" @@ -23,6 +24,7 @@ type runner struct { globalTestConfig testrunner.GlobalRunnerTestConfig withCoverage bool coverageType string + repositoryRoot *os.Root } type AssetTestRunnerOptions struct { @@ -31,6 +33,7 @@ type AssetTestRunnerOptions struct { GlobalTestConfig testrunner.GlobalRunnerTestConfig WithCoverage bool CoverageType string + RepositoryRoot *os.Root } func NewAssetTestRunner(options AssetTestRunnerOptions) *runner { @@ -40,6 +43,7 @@ func NewAssetTestRunner(options AssetTestRunnerOptions) *runner { globalTestConfig: options.GlobalTestConfig, withCoverage: options.WithCoverage, coverageType: options.CoverageType, + repositoryRoot: options.RepositoryRoot, } return &runner } @@ -70,6 +74,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) { GlobalTestConfig: r.globalTestConfig, WithCoverage: r.withCoverage, CoverageType: r.coverageType, + RepositoryRoot: r.repositoryRoot, }), } return testers, nil diff --git a/internal/testrunner/runners/asset/tester.go b/internal/testrunner/runners/asset/tester.go index d6996e3609..b0eabb811a 100644 --- a/internal/testrunner/runners/asset/tester.go +++ b/internal/testrunner/runners/asset/tester.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "github.com/elastic/elastic-package/internal/kibana" @@ -25,6 +26,7 @@ type tester struct { globalTestConfig testrunner.GlobalRunnerTestConfig withCoverage bool coverageType string + repositoryRoot *os.Root } type AssetTesterOptions struct { @@ -34,6 +36,7 @@ type AssetTesterOptions struct { GlobalTestConfig testrunner.GlobalRunnerTestConfig WithCoverage bool CoverageType string + RepositoryRoot *os.Root } func NewAssetTester(options AssetTesterOptions) *tester { @@ -44,6 +47,7 @@ func NewAssetTester(options AssetTesterOptions) *tester { globalTestConfig: options.GlobalTestConfig, withCoverage: options.WithCoverage, coverageType: options.CoverageType, + repositoryRoot: options.RepositoryRoot, } manager := resources.NewManager() @@ -80,9 +84,10 @@ func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) { func (r *tester) resources(installedPackage bool) resources.Resources { return resources.Resources{ &resources.FleetPackage{ - RootPath: r.packageRootPath, - Absent: !installedPackage, - Force: installedPackage, // Force re-installation, in case there are code changes in the same package version. + PackageRootPath: r.packageRootPath, + Absent: !installedPackage, + Force: installedPackage, // Force re-installation, in case there are code changes in the same package version. + RepositoryRoot: r.repositoryRoot, }, } } diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 01de91c644..f556f67323 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -35,6 +35,8 @@ type runner struct { coverageType string deferCleanup time.Duration globalTestConfig testrunner.GlobalRunnerTestConfig + + repositoryRoot *os.Root } type PipelineTestRunnerOptions struct { @@ -48,6 +50,7 @@ type PipelineTestRunnerOptions struct { CoverageType string DeferCleanup time.Duration GlobalTestConfig testrunner.GlobalRunnerTestConfig + RepositoryRoot *os.Root } func NewPipelineTestRunner(options PipelineTestRunnerOptions) *runner { @@ -62,6 +65,7 @@ func NewPipelineTestRunner(options PipelineTestRunnerOptions) *runner { coverageType: options.CoverageType, deferCleanup: options.DeferCleanup, globalTestConfig: options.GlobalTestConfig, + repositoryRoot: options.RepositoryRoot, } return &runner } @@ -138,6 +142,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) { API: r.esAPI, TestCaseFile: caseFile, GlobalTestConfig: r.globalTestConfig, + RepositoryRoot: r.repositoryRoot, }) if err != nil { return nil, fmt.Errorf("failed to create pipeline tester: %w", err) diff --git a/internal/testrunner/runners/pipeline/tester.go b/internal/testrunner/runners/pipeline/tester.go index 0ab0c348e0..ea96741609 100644 --- a/internal/testrunner/runners/pipeline/tester.go +++ b/internal/testrunner/runners/pipeline/tester.go @@ -53,7 +53,8 @@ type tester struct { runCompareResults bool - provider stack.Provider + provider stack.Provider + repositoryRoot *os.Root } type PipelineTesterOptions struct { @@ -67,6 +68,7 @@ type PipelineTesterOptions struct { CoverageType string TestCaseFile string GlobalTestConfig testrunner.GlobalRunnerTestConfig + RepositoryRoot *os.Root } func NewPipelineTester(options PipelineTesterOptions) (*tester, error) { @@ -85,6 +87,7 @@ func NewPipelineTester(options PipelineTesterOptions) (*tester, error) { withCoverage: options.WithCoverage, coverageType: options.CoverageType, globalTestConfig: options.GlobalTestConfig, + repositoryRoot: options.RepositoryRoot, } stackConfig, err := stack.LoadConfig(r.profile) @@ -168,7 +171,7 @@ func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) { startTesting := time.Now() var entryPipeline string - entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(ctx, r.esAPI, dataStreamPath) + entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(ctx, r.esAPI, dataStreamPath, r.repositoryRoot) if err != nil { return nil, fmt.Errorf("installing ingest pipelines failed: %w", err) } diff --git a/internal/testrunner/runners/policy/runner.go b/internal/testrunner/runners/policy/runner.go index 30788323d6..0f8166981f 100644 --- a/internal/testrunner/runners/policy/runner.go +++ b/internal/testrunner/runners/policy/runner.go @@ -7,6 +7,7 @@ package policy import ( "context" "fmt" + "os" "path/filepath" "strings" @@ -34,6 +35,8 @@ type runner struct { resourcesManager *resources.Manager cleanup func(context.Context) error + + repositoryRoot *os.Root } // Ensures that runner implements testrunner.TestRunner interface @@ -48,6 +51,7 @@ type PolicyTestRunnerOptions struct { GlobalTestConfig testrunner.GlobalRunnerTestConfig WithCoverage bool CoverageType string + RepositoryRoot *os.Root } func NewPolicyTestRunner(options PolicyTestRunnerOptions) *runner { @@ -60,6 +64,7 @@ func NewPolicyTestRunner(options PolicyTestRunnerOptions) *runner { globalTestConfig: options.GlobalTestConfig, withCoverage: options.WithCoverage, coverageType: options.CoverageType, + repositoryRoot: options.RepositoryRoot, } runner.resourcesManager = resources.NewManager() runner.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: runner.kibanaClient}) @@ -156,7 +161,8 @@ func (r *runner) Type() testrunner.TestType { func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) { packageResource := resources.FleetPackage{ - RootPath: r.packageRootPath, + PackageRootPath: r.packageRootPath, + RepositoryRoot: r.repositoryRoot, } setupResources := resources.Resources{ &packageResource, diff --git a/internal/testrunner/runners/policy/tester.go b/internal/testrunner/runners/policy/tester.go index 9cf62b7f2b..40f93c0894 100644 --- a/internal/testrunner/runners/policy/tester.go +++ b/internal/testrunner/runners/policy/tester.go @@ -113,12 +113,12 @@ func (r *tester) runTest(ctx context.Context, manager *resources.Manager, testPa Namespace: "ep", PackagePolicies: []resources.FleetPackagePolicy{ { - Name: testName + "-" + r.testFolder.Package, - RootPath: r.packageRootPath, - DataStreamName: r.testFolder.DataStream, - InputName: testConfig.Input, - Vars: testConfig.Vars, - DataStreamVars: testConfig.DataStream.Vars, + Name: testName + "-" + r.testFolder.Package, + PackageRootPath: r.packageRootPath, + DataStreamName: r.testFolder.DataStream, + InputName: testConfig.Input, + Vars: testConfig.Vars, + DataStreamVars: testConfig.DataStream.Vars, }, }, } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 6ba7125117..669089f125 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -25,6 +25,7 @@ import ( type runner struct { profile *profile.Profile + repositoryRoot *os.Root packageRootPath string kibanaClient *kibana.Client esAPI *elasticsearch.API @@ -55,6 +56,7 @@ var _ testrunner.TestRunner = new(runner) type SystemTestRunnerOptions struct { Profile *profile.Profile PackageRootPath string + RepositoryRoot *os.Root KibanaClient *kibana.Client API *elasticsearch.API @@ -97,6 +99,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner { globalTestConfig: options.GlobalTestConfig, withCoverage: options.WithCoverage, coverageType: options.CoverageType, + repositoryRoot: options.RepositoryRoot, } r.resourcesManager = resources.NewManager() @@ -283,9 +286,10 @@ func (r *runner) Type() testrunner.TestType { func (r *runner) resources(opts resourcesOptions) resources.Resources { return resources.Resources{ &resources.FleetPackage{ - RootPath: r.packageRootPath, - Absent: !opts.installedPackage, - Force: opts.installedPackage, // Force re-installation, in case there are code changes in the same package version. + PackageRootPath: r.packageRootPath, + Absent: !opts.installedPackage, + Force: opts.installedPackage, // Force re-installation, in case there are code changes in the same package version. + RepositoryRoot: r.repositoryRoot, }, } } diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index 004858eaec..70bef635e6 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -230,7 +230,7 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_DATA_HOME`: Custom path to be used for `elastic-package` data directory. By default this is `~/.elastic-package`. - Related to the build process: - - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license. + - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license. This path should be relative to the repository root. - `ELASTIC_PACKAGE_LINKS_FILE_PATH`: Path to the links table file (e.g. `links_table.yml`) with the link definitions to be used in the build process of a package. - Related to signing packages: