Add wolfi ess docker image

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -33,7 +33,10 @@ public enum DockerBase {
         "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
         "-wolfi",
         "apk"
-    );
+    ),
+
+    // Chainguard based wolfi image with latest jdk
+    WOLFI_ESS(null, "-wolfi-ess", "apk");
 
     private final String image;
     private final String suffix;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+
+public class DockerWolfiEssElasticsearchDistributionType implements ElasticsearchDistributionType {
+
+    DockerWolfiEssElasticsearchDistributionType() {}
+
+    @Override
+    public String getName() {
+        return "dockerWolfiEss";
+    }
+
+    @Override
+    public boolean isDocker() {
+        return true;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -22,6 +22,7 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
+    public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -31,6 +32,7 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_IRONBANK,
         DOCKER_CLOUD,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI
+        DOCKER_WOLFI,
+        DOCKER_WOLFI_ESS
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -54,6 +54,7 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
+import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 
 /**
@@ -152,6 +153,7 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
+        lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

distribution/docker/README.md

@@ -7,6 +7,13 @@ the [DockerBase] enum.
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
    * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
+   * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins
+     that the ES build generates in an archive directory. It also sets an
+     environment variable that points at this directory. This allows plugins to
+     be installed from the archive instead of the internet, speeding up
+     deployment times. Furthermore this image has
+     * `filebeat` and `metricbeat` included
+     * `wget` included
    * Iron Bank - this is the US Department of Defence's repository of digitally
      signed, binary container images including both Free and Open-Source
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is

distribution/docker/build.gradle

@@ -1,4 +1,3 @@
-import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.DockerBase
@@ -8,8 +7,10 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
 import org.elasticsearch.gradle.internal.docker.DockerSupportService
 import org.elasticsearch.gradle.internal.docker.ShellRetry
 import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
+import org.elasticsearch.gradle.internal.docker.*
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
+import org.elasticsearch.gradle.Architecture
 import java.nio.file.Path
 import java.time.temporal.ChronoUnit
 
@@ -99,9 +100,9 @@ String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'am
 
 dependencies {
   aarch64DockerSource project(":distribution:archives:linux-aarch64-tar")
-  aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default")
+  aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration: "default")
   dockerSource project(":distribution:archives:linux-tar")
-  dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration:"default")
+  dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration: "default")
   log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini "krallin:tini:0.19.0:${tiniArch}"
   allPlugins project(path: ':plugins', configuration: 'allPlugins')
@@ -112,7 +113,7 @@ dependencies {
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
-  def (major,minor) = VersionProperties.elasticsearch.split("\\.")
+  def (major, minor) = VersionProperties.elasticsearch.split("\\.")
 
   // We tag our Docker images with various pieces of information, including a timestamp
   // for when the image was built. However, this makes it impossible completely cache
@@ -216,7 +217,8 @@ elasticsearch_distributions {
 }
 
 interface Injected {
-  @Inject FileSystemOperations getFs()
+  @Inject
+  FileSystemOperations getFs()
 }
 
 tasks.named("preProcessFixture").configure {
@@ -300,7 +302,10 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         // For some reason, the artifact name can differ depending on what repository we used.
         rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
       }
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
+        project.gradle.sharedServices,
+        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+      )
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
     }
 
@@ -337,9 +342,9 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     into "${project.buildDir}/docker-context/${archiveName}"
 
     // Since we replaced the remote URL in the Dockerfile, copy in the required file
-    if(base == DockerBase.IRON_BANK) {
+    if (base == DockerBase.IRON_BANK) {
       from(architecture == Architecture.AARCH64 ? configurations.aarch64DockerSourceTar : configurations.dockerSourceTar)
-      from (configurations.tini) {
+      from(configurations.tini) {
         rename { _ -> 'tini' }
       }
     } else {
@@ -349,7 +354,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v ->
       inputs.property(k, { v.toString() })
     }
-    Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+    Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
+      project.gradle.sharedServices,
+      DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+    )
     onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
   }
 
@@ -423,7 +431,10 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
         baseImages = [base.image]
       }
 
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
+        project.gradle.sharedServices,
+        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+      )
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
 
     }
@@ -435,13 +446,12 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture) {
-  DockerBase base = DockerBase.CLOUD_ESS
+void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) {
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
-  String contextDir = "${project.buildDir}/docker-context/elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
+  String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
 
   final TaskProvider<Sync> buildContextTask =
-    tasks.register(taskName('build', architecture, base, 'DockerContext'), Sync) {
+    tasks.register(taskName('build', architecture, dockerBase, 'DockerContext'), Sync) {
       into contextDir
 
       final Path projectDir = project.projectDir.toPath()
@@ -450,28 +460,54 @@ void addBuildEssDockerImageTask(Architecture architecture) {
         from configurations.allPlugins
       }
 
-      from(projectDir.resolve("src/docker/Dockerfile.cloud-ess")) {
-        expand([
-          base_image: "elasticsearch${DockerBase.CLOUD.suffix}:${architecture.classifier}"
-        ])
+      if (dockerBase == DockerBase.WOLFI_ESS) {
+        // If we're performing a release build, but `build.id` hasn't been set, we can
+        // infer that we're not at the Docker building stage of the build, and therefore
+        // we should skip the beats part of the build.
+        String buildId = providers.systemProperty('build.id').getOrNull()
+        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+
+        if (includeBeats) {
+          from configurations.getByName("filebeat_${architecture.classifier}")
+          from configurations.getByName("metricbeat_${architecture.classifier}")
+        }
+        // For some reason, the artifact name can differ depending on what repository we used.
+        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
+      }
+
+      String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix
+      from(projectDir.resolve("src/docker/Dockerfile.ess")) {
+        expand(
+          [
+            base_image: "elasticsearch${baseSuffix}:${architecture.classifier}",
+            docker_base: "${dockerBase.name().toLowerCase()}",
+            version: "${VersionProperties.elasticsearch}",
+            retry: ShellRetry
+          ]
+        )
         filter SquashNewlinesFilter
-        rename ~/Dockerfile\.cloud-ess$/, 'Dockerfile'
+        rename ~/Dockerfile\.ess$/, 'Dockerfile'
       }
     }
 
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
-    tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) {
+    tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
 
-      TaskProvider<DockerBuildTask> buildCloudTask = tasks.named(taskName("build", architecture, DockerBase.CLOUD, "DockerImage"))
-      inputs.files(buildCloudTask)
+      DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI
+
+      TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
+      inputs.files(buildBaseTask)
 
       dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() })
 
       noCache = BuildParams.isCi()
       baseImages = []
-      tags = generateTags(base, architecture)
+      tags = generateTags(dockerBase, architecture)
       platforms.add(architecture.dockerPlatform)
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
+        project.gradle.sharedServices,
+        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+      )
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
 
     }
@@ -483,15 +519,16 @@ void addBuildEssDockerImageTask(Architecture architecture) {
 
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
-    if (base == DockerBase.CLOUD_ESS) {
+    if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
       continue
     }
     addBuildDockerContextTask(architecture, base)
     addTransformDockerContextTask(architecture, base)
     addBuildDockerImageTask(architecture, base)
   }
 
-  addBuildEssDockerImageTask(architecture)
+  addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS)
+  addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -515,6 +552,8 @@ subprojects { Project subProject ->
       base = DockerBase.CLOUD_ESS
     } else if (subProject.name.contains('cloud-')) {
       base = DockerBase.CLOUD
+    } else if (subProject.name.contains('wolfi-ess')) {
+      base = DockerBase.WOLFI_ESS
     } else if (subProject.name.contains('wolfi-')) {
       base = DockerBase.WOLFI
     }
@@ -525,7 +564,8 @@ subprojects { Project subProject ->
         (base == DockerBase.CLOUD ? 'cloud.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
-              'docker.tar'))))
+              (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' :
+                'docker.tar')))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')
@@ -541,7 +581,10 @@ subprojects { Project subProject ->
         tarFile,
         "elasticsearch${base.suffix}:${architecture.classifier}"
       dependsOn(parent.path + ":" + buildTaskName)
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
+        project.gradle.sharedServices,
+        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+      )
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
     }
 

distribution/docker/src/docker/Dockerfile.ess

@@ -0,0 +1,52 @@
+FROM ${base_image} AS builder
+
+USER root
+
+<% if (docker_base == "wolfi_ess") { %>
+    # Add plugins infrastructure
+    RUN mkdir -p /opt/plugins/archive
+    RUN chmod -R 0555 /opt/plugins
+
+    COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
+    RUN set -eux ; \\
+        for beat in filebeat metricbeat ; do \\
+          if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
+            echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
+            exit 1 ; \\
+          fi ; \\
+          if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
+            echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
+            exit 1 ; \\
+          fi ; \\
+          mkdir -p /opt/\$beat ; \\
+          tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
+        done
+<% } %>
+
+COPY plugins/*.zip /opt/plugins/archive/
+
+RUN chown root.root /opt/plugins/archive/*
+RUN chmod 0444 /opt/plugins/archive/*
+
+FROM ${base_image}
+RUN echo "${docker_base}"
+<% if (docker_base == "wolfi_ess") { %>
+USER root
+
+RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %>
+
+# tweak entry point for ESS specific wolfi image
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD ["/app/elasticsearch.sh"]
+# Generate a stub command that will be overwritten at runtime
+RUN mkdir /app && \\
+    echo -e '#!/bin/bash\\nexec /usr/local/bin/docker-entrypoint.sh eswrapper' > /app/elasticsearch.sh && \\
+    chmod 0555 /app/elasticsearch.sh
+
+COPY --from=builder --chown=0:0 /opt /opt
+USER 1000:0
+<% } else { %>
+COPY --from=builder /opt/plugins /opt/plugins
+<% } %>
+
+ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java

@@ -79,7 +79,8 @@ public enum Packaging {
         DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()),
         DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()),
         DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()),
-        DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker());
+        DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()),
+        DOCKER_WOLFI_ESS(".wolfi-ess.tar", Platforms.isDocker());
 
         /** The extension of this distribution's file */
         public final String extension;

settings.gradle

@@ -75,6 +75,8 @@ List projects = [
   'distribution:docker:ubi-docker-export',
   'distribution:docker:wolfi-docker-aarch64-export',
   'distribution:docker:wolfi-docker-export',
+  'distribution:docker:wolfi-ess-docker-aarch64-export',
+  'distribution:docker:wolfi-ess-docker-export',
   'distribution:packages:aarch64-deb',
   'distribution:packages:deb',
   'distribution:packages:aarch64-rpm',