Simple includes implementation

README.md

@@ -370,6 +370,30 @@ Use this command to install the package in Kibana.
 
 The command uses Kibana API to install the package in Kibana. The package must be exposed via the Package Registry or built locally in zip format so they can be installed using --zip parameter. Zip packages can be installed directly in Kibana >= 8.7.0. More details in this [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/install_package.md).
 
+### `elastic-package links`
+
+_Context: global_
+
+Use this command to manage linked files in the repository.
+
+### `elastic-package links check`
+
+_Context: global_
+
+Use this command to check if linked files references inside the current directory are up to date.
+
+### `elastic-package links list`
+
+_Context: global_
+
+Use this command to list all packages that have linked file references that include the current directory.
+
+### `elastic-package links update`
+
+_Context: global_
+
+Use this command to update all linked files references inside the current directory.
+
 ### `elastic-package lint`
 
 _Context: package_

cmd/links.go

@@ -0,0 +1,146 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/files"
+)
+
+const (
+	linksLongDescription       = `Use this command to manage linked files in the repository.`
+	linksCheckLongDescription  = `Use this command to check if linked files references inside the current directory are up to date.`
+	linksUpdateLongDescription = `Use this command to update all linked files references inside the current directory.`
+	linksListLongDescription   = `Use this command to list all packages that have linked file references that include the current directory.`
+)
+
+func setupLinksCommand() *cobraext.Command {
+	cmd := &cobra.Command{
+		Use:   "links",
+		Short: "Manage linked files",
+		Long:  linksLongDescription,
+		RunE: func(parent *cobra.Command, args []string) error {
+			return cobraext.ComposeCommandsParentContext(parent, args, parent.Commands()...)
+		},
+	}
+
+	cmd.AddCommand(getLinksCheckCommand())
+	cmd.AddCommand(getLinksUpdateCommand())
+	cmd.AddCommand(getLinksListCommand())
+
+	return cobraext.NewCommand(cmd, cobraext.ContextGlobal)
+}
+
+func getLinksCheckCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "check",
+		Short: "Check for linked files changes",
+		Long:  linksCheckLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  linksCheckCommandAction,
+	}
+	return cmd
+}
+
+func linksCheckCommandAction(cmd *cobra.Command, args []string) error {
+	cmd.Printf("Check for linked files changes\n")
+	pwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("reading current working directory failed: %w", err)
+	}
+
+	linkedFiles, err := files.AreLinkedFilesUpToDate(pwd)
+	if err != nil {
+		return fmt.Errorf("checking linked files are up-to-date failed: %w", err)
+	}
+	for _, f := range linkedFiles {
+		if !f.UpToDate {
+			cmd.Printf("%s is outdated.\n", filepath.Join(f.WorkDir, f.LinkFilePath))
+		}
+	}
+	if len(linkedFiles) > 0 {
+		return fmt.Errorf("linked files are outdated")
+	}
+	return nil
+}
+
+func getLinksUpdateCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "update",
+		Short: "Update linked files checksums if needed.",
+		Long:  linksUpdateLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  linksUpdateCommandAction,
+	}
+	return cmd
+}
+
+func linksUpdateCommandAction(cmd *cobra.Command, args []string) error {
+	cmd.Printf("Update linked files checksums if needed.\n")
+	pwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("reading current working directory failed: %w", err)
+	}
+
+	updatedLinks, err := files.UpdateLinkedFilesChecksums(pwd)
+	if err != nil {
+		return fmt.Errorf("updating linked files checksums failed: %w", err)
+	}
+
+	for _, l := range updatedLinks {
+		cmd.Printf("%s was updated.\n", l.LinkFilePath)
+	}
+
+	return nil
+}
+
+func getLinksListCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "list",
+		Short: "List packages linking files from this path",
+		Long:  linksListLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  linksListCommandAction,
+	}
+	cmd.Flags().BoolP(cobraext.PackagesFlagName, "", false, cobraext.PackagesFlagDescription)
+	return cmd
+}
+
+func linksListCommandAction(cmd *cobra.Command, args []string) error {
+	onlyPackages, err := cmd.Flags().GetBool(cobraext.PackagesFlagName)
+	if err != nil {
+		return cobraext.FlagParsingError(err, cobraext.PackagesFlagName)
+	}
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("reading current working directory failed: %w", err)
+	}
+
+	byPackage, err := files.LinkedFilesByPackageFrom(pwd)
+	if err != nil {
+		return fmt.Errorf("listing linked packages failed: %w", err)
+	}
+
+	for i := range byPackage {
+		for p, links := range byPackage[i] {
+			if onlyPackages {
+				cmd.Println(p)
+				continue
+			}
+			for _, l := range links {
+				cmd.Println(l)
+			}
+		}
+	}
+
+	return nil
+}

cmd/lint.go

@@ -45,7 +45,6 @@ func setupLintCommand() *cobraext.Command {
 
 func lintCommandAction(cmd *cobra.Command, args []string) error {
 	cmd.Println("Lint the package")
-
 	readmeFiles, err := docs.AreReadmesUpToDate()
 	if err != nil {
 		for _, f := range readmeFiles {

cmd/root.go

@@ -28,6 +28,7 @@ var commands = []*cobraext.Command{
 	setupExportCommand(),
 	setupFormatCommand(),
 	setupInstallCommand(),
+	setupLinksCommand(),
 	setupLintCommand(),
 	setupProfilesCommand(),
 	setupReportsCommand(),

docs/howto/links.md

@@ -0,0 +1,34 @@
+# HOWTO: Use links to reuse common files.
+
+## Introduction
+
+Many packages have files that are equal between them. This is more common in pipelines, 
+input configurations, and field definitions.
+
+In order to help developers, there is the ability to define links, so a file that might be reused needs to only be defined once, and can be reused from any other packages.
+
+
+# Links
+
+Currently, there are some specific places where links can be defined:
+
+- `elasticsearch/ingest_pipeline`
+- `data_stream/**/elasticsearch/ingest_pipeline`
+- `agent/input`
+- `data_stream/**/agent/stream`
+- `data_stream/**/fields`
+
+A link consists of a file with a `.link` extension that contains a path, relative to its location, to the file that it will be replaced with. It also consists of a checksum to validate the linked file is up to date with the package expectations.
+
+`data_stream/foo/elasticsearch/ingest_pipeline/default.yml.link`
+
+```
+../../../../../testpackage/data_stream/test/elasticsearch/ingest_pipeline/default.yml f7c5f0c03aca8ef68c379a62447bdafbf0dcf32b1ff2de143fd6878ee01a91ad
+```
+
+This will use the contents of the linked file during validation, tests, and building of the package, so functionally nothing changes from the package point of view.
+
+## The `_dev/shared` folder
+
+As a convenience, shared files can be placed under `_dev/shared` if they are going to be
+reused from several places. They can even be added outside of any package, in any place in the repository.

go.mod

@@ -2,6 +2,8 @@ module github.com/elastic/elastic-package
 
 go 1.24.2
 
+replace github.com/elastic/package-spec/v3 => github.com/elastic/package-spec/v3 v3.0.0-20250409140721-851b65d4339d
+
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Masterminds/semver/v3 v3.3.1

go.sum

@@ -84,8 +84,8 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul
 github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg=
 github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU=
 github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI=
-github.com/elastic/package-spec/v3 v3.3.5 h1:D0AXRiTNcF8Ue8gLIafF/BLOk7V2yqSFVUy/p0fwArM=
-github.com/elastic/package-spec/v3 v3.3.5/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg=
+github.com/elastic/package-spec/v3 v3.0.0-20250409140721-851b65d4339d h1:jg8qN/0ZAxbo65coqJUFx01OC2PMkWc+6kaf9labTkc=
+github.com/elastic/package-spec/v3 v3.0.0-20250409140721-851b65d4339d/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls=

internal/builder/packages.go

@@ -184,6 +184,15 @@ func BuildPackage(options BuildOptions) (string, error) {
 		return "", fmt.Errorf("adding dynamic mappings: %w", err)
 	}
 
+	logger.Debug("Include linked files")
+	links, err := files.IncludeLinkedFiles(options.PackageRoot, destinationDir)
+	if err != nil {
+		return "", fmt.Errorf("including linked files failed: %w", err)
+	}
+	for _, l := range links {
+		logger.Debugf("Linked file included (path: %s)", l.TargetFilePath(destinationDir))
+	}
+
 	if options.CreateZip {
 		return buildZippedPackage(options, destinationDir)
 	}

internal/cobraext/flags.go

@@ -138,6 +138,9 @@ const (
 	GenerateTestResultFlagName        = "generate"
 	GenerateTestResultFlagDescription = "generate test result file"
 
+	PackagesFlagName        = "packages"
+	PackagesFlagDescription = "whether to return packages names or complete paths for the linked files found"
+
 	ProfileFlagName        = "profile"
 	ProfileFlagDescription = "select a profile to use for the stack configuration. Can also be set with %s"
 

internal/elasticsearch/ingest/datastream.go

@@ -19,13 +19,16 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/packages"
 )
 
 var (
-	ingestPipelineTag   = regexp.MustCompile(`{{\s*IngestPipeline.+}}`)
-	defaultPipelineJSON = "default.json"
-	defaultPipelineYML  = "default.yml"
+	ingestPipelineTag       = regexp.MustCompile(`{{\s*IngestPipeline.+}}`)
+	defaultPipelineJSON     = "default.json"
+	defaultPipelineJSONLink = "default.json"
+	defaultPipelineYML      = "default.yml.link"
+	defaultPipelineYMLLink  = "default.yml.link"
 )
 
 type Rule struct {
@@ -71,17 +74,18 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er
 	elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline")
 
 	var pipelineFiles []string
-	for _, pattern := range []string{"*.json", "*.yml"} {
+	for _, pattern := range []string{"*.json", "*.yml", "*.link"} {
 		files, err := filepath.Glob(filepath.Join(elasticsearchPath, pattern))
 		if err != nil {
 			return nil, fmt.Errorf("listing '%s' in '%s': %w", pattern, elasticsearchPath, err)
 		}
 		pipelineFiles = append(pipelineFiles, files...)
 	}
 
+	linksFS := files.NewLinksFS(elasticsearchPath)
 	var pipelines []Pipeline
 	for _, path := range pipelineFiles {
-		c, err := os.ReadFile(path)
+		c, err := linksFS.ReadFile(path)
 		if err != nil {
 			return nil, fmt.Errorf("reading ingest pipeline failed (path: %s): %w", path, err)
 		}
@@ -108,7 +112,7 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er
 		pipelines = append(pipelines, Pipeline{
 			Path:            path,
 			Name:            getPipelineNameWithNonce(name[:strings.Index(name, ".")], nonce),
-			Format:          filepath.Ext(path)[1:],
+			Format:          filepath.Ext(strings.TrimSuffix(path, ".link"))[1:],
 			Content:         cWithRerouteProcessors,
 			ContentOriginal: c,
 		})
@@ -119,7 +123,8 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er
 func addRerouteProcessors(pipeline []byte, dataStreamPath, path string) ([]byte, error) {
 	// Only attach routing_rules.yml reroute processors after the default pipeline
 	filename := filepath.Base(path)
-	if filename != defaultPipelineJSON && filename != defaultPipelineYML {
+	if filename != defaultPipelineJSON && filename != defaultPipelineYML &&
+		filename != defaultPipelineJSONLink && filename != defaultPipelineYMLLink {
 		return pipeline, nil
 	}
 

internal/files/copy.go

@@ -13,7 +13,7 @@ import (
 
 var (
 	defaultFoldersToSkip   = []string{"_dev", "build", ".git"}
-	defaultFileGlobsToSkip = []string{".DS_Store", ".*.swp"}
+	defaultFileGlobsToSkip = []string{".DS_Store", ".*.swp", "*.link"}
 )
 
 // CopyAll method copies files from the source to the destination skipping empty directories.