Reapply "synthetic source index setting provider should check source …

…field mapper" (elastic#113759)

Originally added via elastic#113522, but then reverted via elastic#113745, because of mixed cluster test failures (elastic#113730).

This PR is a clean revert of the commit the reverted elastic#113522 and one additional commit that should address the build failures report in elastic#113730 : c7bd242

Basically create index invocation that would fail anyway should be ignored. If mapper service creation now fails, then we just assume that there is no synthetic source usage. This is ok, because the index creation would fail anyway later one.

Closes elastic#113730

x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -40,7 +40,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -42,7 +42,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

@@ -51,7 +51,10 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
         if (DiscoveryNode.isStateless(settings)) {
             return List.of(logsdbIndexModeSettingsProvider);
         }
-        return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider);
+        return List.of(
+            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
+            logsdbIndexModeSettingsProvider
+        );
     }
 
     @Override

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java

@@ -9,28 +9,41 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.MapperService;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.util.List;
-import java.util.Locale;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
 
 /**
  * An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
  */
-public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
+final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
 
     private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
 
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
+    private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
 
-    public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
+    SyntheticSourceIndexSettingsProvider(
+        SyntheticSourceLicenseService syntheticSourceLicenseService,
+        CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
+    ) {
         this.syntheticSourceLicenseService = syntheticSourceLicenseService;
+        this.mapperServiceFactory = mapperServiceFactory;
     }
 
     @Override
@@ -46,19 +59,71 @@ public Settings getAdditionalIndexSettings(
         // This index name is used when validating component and index templates, we should skip this check in that case.
         // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
         boolean isTemplateValidation = "validate-index-name".equals(indexName);
-        if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
+        if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
             && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
             // TODO: handle falling back to stored source
         }
         return Settings.EMPTY;
     }
 
-    boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
-        // TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
-        // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
-        var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
-        IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
-        return indexMode != null && indexMode.isSyntheticSourceEnabled();
+    boolean newIndexHasSyntheticSourceUsage(
+        String indexName,
+        boolean isTimeSeries,
+        Settings indexTemplateAndCreateRequestSettings,
+        List<CompressedXContent> combinedTemplateMappings
+    ) {
+        if ("validate-index-name".equals(indexName)) {
+            // This index name is used when validating component and index templates, we should skip this check in that case.
+            // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
+            return false;
+        }
+
+        var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
+        try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
+            // combinedTemplateMappings can be null when creating system indices
+            // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
+            if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
+                combinedTemplateMappings = List.of(new CompressedXContent("{}"));
+            }
+            mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
+            return mapperService.documentMapper().sourceMapper().isSynthetic();
+        } catch (AssertionError | Exception e) {
+            // In case invalid mappings or setting are provided, then mapper service creation can fail.
+            // In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source.
+            LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e);
+            return false;
+        }
+    }
+
+    // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
+    private IndexMetadata buildIndexMetadataForMapperService(
+        String indexName,
+        boolean isTimeSeries,
+        Settings indexTemplateAndCreateRequestSettings
+    ) {
+        var tmpIndexMetadata = IndexMetadata.builder(indexName);
+
+        int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
+        int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
+            IndexMetadata.SETTING_NUMBER_OF_SHARDS,
+            dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
+        );
+        int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+        var finalResolvedSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(indexTemplateAndCreateRequestSettings)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
+            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
+
+        if (isTimeSeries) {
+            finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
+            // Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
+            finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
+        }
+
+        tmpIndexMetadata.settings(finalResolvedSettings);
+        return tmpIndexMetadata.build();
     }
 }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java

@@ -16,7 +16,7 @@
 /**
  * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
  */
-public final class SyntheticSourceLicenseService {
+final class SyntheticSourceLicenseService {
 
     private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
 
@@ -39,7 +39,7 @@ public final class SyntheticSourceLicenseService {
     private XPackLicenseState licenseState;
     private volatile boolean syntheticSourceFallback;
 
-    public SyntheticSourceLicenseService(Settings settings) {
+    SyntheticSourceLicenseService(Settings settings) {
         syntheticSourceFallback = FALLBACK_SETTING.get(settings);
     }