Merge 7cff27a into openjdk23-bundle

elastic · Sep 29, 2024 · efdaa15 · efdaa15
2 parents d2260fd + 7cff27a
commit efdaa15
Show file tree

Hide file tree

Showing 8 changed files with 39 additions and 335 deletions.
diff --git a/...ip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java b/...ip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java
@@ -44,32 +44,36 @@ public class DatabaseNodeServiceIT extends AbstractGeoIpIT {
      * This test makes sure that if we index an ordinary mmdb file into the .geoip_databases index, it is correctly handled upon retrieval.
      */
     public void testNonGzippedDatabase() throws Exception {
-        String databaseName = "GeoLite2-Country";
-        String databaseFileName = databaseName + ".mmdb";
+        String databaseType = "GeoLite2-Country";
+        String databaseFileName = databaseType + ".mmdb";
+        // making the dabase name unique so we know we're not using another one:
+        String databaseName = randomAlphaOfLength(20) + "-" + databaseFileName;
         byte[] mmdbBytes = getBytesForFile(databaseFileName);
         final DatabaseNodeService databaseNodeService = internalCluster().getInstance(DatabaseNodeService.class);
-        assertNull(databaseNodeService.getDatabase(databaseFileName));
-        int numChunks = indexData(databaseFileName, mmdbBytes);
-        retrieveDatabase(databaseNodeService, databaseFileName, mmdbBytes, numChunks);
-        assertBusy(() -> assertNotNull(databaseNodeService.getDatabase(databaseFileName)));
-        assertValidDatabase(databaseNodeService, databaseFileName, databaseName);
+        assertNull(databaseNodeService.getDatabase(databaseName));
+        int numChunks = indexData(databaseName, mmdbBytes);
+        retrieveDatabase(databaseNodeService, databaseName, mmdbBytes, numChunks);
+        assertBusy(() -> assertNotNull(databaseNodeService.getDatabase(databaseName)));
+        assertValidDatabase(databaseNodeService, databaseName, databaseType);
     }
 
     /*
      * This test makes sure that if we index a gzipped tar file wrapping an mmdb file into the .geoip_databases index, it is correctly
      * handled upon retrieval.
      */
     public void testGzippedDatabase() throws Exception {
-        String databaseName = "GeoLite2-Country";
-        String databaseFileName = databaseName + ".mmdb";
+        String databaseType = "GeoLite2-Country";
+        String databaseFileName = databaseType + ".mmdb";
+        // making the dabase name unique so we know we're not using another one:
+        String databaseName = randomAlphaOfLength(20) + "-" + databaseFileName;
         byte[] mmdbBytes = getBytesForFile(databaseFileName);
-        byte[] gzipBytes = gzipFileBytes(databaseFileName, mmdbBytes);
+        byte[] gzipBytes = gzipFileBytes(databaseName, mmdbBytes);
         final DatabaseNodeService databaseNodeService = internalCluster().getInstance(DatabaseNodeService.class);
-        assertNull(databaseNodeService.getDatabase(databaseFileName));
-        int numChunks = indexData(databaseFileName, gzipBytes);
-        retrieveDatabase(databaseNodeService, databaseFileName, gzipBytes, numChunks);
-        assertBusy(() -> assertNotNull(databaseNodeService.getDatabase(databaseFileName)));
-        assertValidDatabase(databaseNodeService, databaseFileName, databaseName);
+        assertNull(databaseNodeService.getDatabase(databaseName));
+        int numChunks = indexData(databaseName, gzipBytes);
+        retrieveDatabase(databaseNodeService, databaseName, gzipBytes, numChunks);
+        assertBusy(() -> assertNotNull(databaseNodeService.getDatabase(databaseName)));
+        assertValidDatabase(databaseNodeService, databaseName, databaseType);
     }
 
     /*

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -329,8 +329,11 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113722
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/113730
+- class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanosTests
+  issue: https://github.com/elastic/elasticsearch/issues/113661
+- class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
+  method: testDeploymentSurvivesRestart {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/112980
 
 # Examples:
 #

diff --git a/...gsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/...gsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
@@ -40,31 +40,7 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            if (randomBoolean()) {
-                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
-            } else if (randomBoolean()) {
-                String mapping = """
-                    {
-                        "properties": {
-                            "field1": {
-                                "type": "keyword",
-                                "time_series_dimension": true
-                            }
-                        }
-                    }
-                    """;
-                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
-                createIndex("test-index", settings, mapping);
-            } else {
-                String mapping = """
-                    {
-                        "_source": {
-                            "mode": "synthetic"
-                        }
-                    }
-                    """;
-                createIndex("test-index", Settings.EMPTY, mapping);
-            }
+            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
@@ -42,31 +42,7 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            if (randomBoolean()) {
-                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
-            } else if (randomBoolean()) {
-                String mapping = """
-                    {
-                        "properties": {
-                            "field1": {
-                                "type": "keyword",
-                                "time_series_dimension": true
-                            }
-                        }
-                    }
-                    """;
-                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
-                createIndex("test-index", settings, mapping);
-            } else {
-                String mapping = """
-                    {
-                        "_source": {
-                            "mode": "synthetic"
-                        }
-                    }
-                    """;
-                createIndex("test-index", Settings.EMPTY, mapping);
-            }
+            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -51,10 +51,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
         if (DiscoveryNode.isStateless(settings)) {
             return List.of(logsdbIndexModeSettingsProvider);
         }
-        return List.of(
-            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
-            logsdbIndexModeSettingsProvider
-        );
+        return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider);
     }
 
     @Override

diff --git a/...db/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/...db/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
@@ -9,41 +9,28 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.mapper.MapperService;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.time.Instant;
 import java.util.List;
-
-import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
+import java.util.Locale;
 
 /**
  * An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
  */
-final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
+public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
 
     private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
 
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
-    private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
 
-    SyntheticSourceIndexSettingsProvider(
-        SyntheticSourceLicenseService syntheticSourceLicenseService,
-        CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
-    ) {
+    public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
         this.syntheticSourceLicenseService = syntheticSourceLicenseService;
-        this.mapperServiceFactory = mapperServiceFactory;
     }
 
     @Override
@@ -56,68 +43,19 @@ public Settings getAdditionalIndexSettings(
         Settings indexTemplateAndCreateRequestSettings,
         List<CompressedXContent> combinedTemplateMappings
     ) {
-        if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
+        if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
             && syntheticSourceLicenseService.fallbackToStoredSource()) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
             // TODO: handle falling back to stored source
         }
         return Settings.EMPTY;
     }
 
-    boolean newIndexHasSyntheticSourceUsage(
-        String indexName,
-        boolean isTimeSeries,
-        Settings indexTemplateAndCreateRequestSettings,
-        List<CompressedXContent> combinedTemplateMappings
-    ) {
-        if ("validate-index-name".equals(indexName)) {
-            // This index name is used when validating component and index templates, we should skip this check in that case.
-            // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
-            return false;
-        }
-
-        var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
-        try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
-            // combinedTemplateMappings can be null when creating system indices
-            // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
-            if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
-                combinedTemplateMappings = List.of(new CompressedXContent("{}"));
-            }
-            mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
-            return mapperService.documentMapper().sourceMapper().isSynthetic();
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
-    private IndexMetadata buildIndexMetadataForMapperService(
-        String indexName,
-        boolean isTimeSeries,
-        Settings indexTemplateAndCreateRequestSettings
-    ) {
-        var tmpIndexMetadata = IndexMetadata.builder(indexName);
-
-        int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
-        int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
-            IndexMetadata.SETTING_NUMBER_OF_SHARDS,
-            dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
-        );
-        int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
-        var finalResolvedSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-            .put(indexTemplateAndCreateRequestSettings)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
-            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
-
-        if (isTimeSeries) {
-            finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
-            // Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
-            finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
-        }
-
-        tmpIndexMetadata.settings(finalResolvedSettings);
-        return tmpIndexMetadata.build();
+    boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
+        // TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
+        // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
+        var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
+        IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
+        return indexMode != null && indexMode.isSyntheticSourceEnabled();
     }
 }
diff --git a/...in/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/...in/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
@@ -16,7 +16,7 @@
 /**
  * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
  */
-final class SyntheticSourceLicenseService {
+public final class SyntheticSourceLicenseService {
 
     private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
 
@@ -39,7 +39,7 @@ final class SyntheticSourceLicenseService {
     private XPackLicenseState licenseState;
     private volatile boolean syntheticSourceFallback;
 
-    SyntheticSourceLicenseService(Settings settings) {
+    public SyntheticSourceLicenseService(Settings settings) {
         syntheticSourceFallback = FALLBACK_SETTING.get(settings);
     }