From 75e5c0e39f78600655ef8173f63c5119e5ca7b14 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 19 Sep 2024 15:50:37 +0100 Subject: [PATCH] Use deprecation logger for CLDR date format specifiers (#112917) The addition of the logger requires several updates to tests to deal with the possible warning, or muting if there is not way to specify an allowed (but not mandatory) warning --- .../cat/component-templates.asciidoc | 7 ++-- docs/reference/esql/esql-rest.asciidoc | 1 + .../indices/index-templates.asciidoc | 3 +- .../indices/put-component-template.asciidoc | 3 +- .../indices/put-index-template-v1.asciidoc | 13 ++++---- .../indices/simulate-template.asciidoc | 23 +++++++------ .../ingest/common-log-format-example.asciidoc | 3 +- modules/ingest-common/build.gradle | 14 ++++++++ .../test/ingest/30_date_processor.yml | 7 ++++ .../test/ingest/20_combine_processors.yml | 11 +++++++ .../search/180_locale_dependent_mapping.yml | 5 +++ .../ComponentTemplatesFileSettingsIT.java | 24 +++++--------- .../elasticsearch/common/time/DateUtils.java | 11 ++++--- ...vedComposableIndexTemplateActionTests.java | 33 +++++++------------ .../src/main/resources/date.csv-spec | 1 + .../sql/qa/CustomDateFormatTestCase.java | 4 +-- .../rest-api-spec/test/esql/70_locale.yml | 2 ++ 17 files changed, 93 insertions(+), 72 deletions(-) diff --git a/docs/reference/cat/component-templates.asciidoc b/docs/reference/cat/component-templates.asciidoc index 596c86befd1b7..f87f400bb8885 100644 --- a/docs/reference/cat/component-templates.asciidoc +++ b/docs/reference/cat/component-templates.asciidoc @@ -6,8 +6,8 @@ [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application +cat APIs are only intended for human consumption using the command line or {kib} +console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== @@ -82,8 +82,7 @@ PUT _component_template/my-template-2 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } } diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc index 2c8c5e81e273d..b2493dc32a211 100644 --- a/docs/reference/esql/esql-rest.asciidoc +++ b/docs/reference/esql/esql-rest.asciidoc @@ -233,6 +233,7 @@ POST /_query } ---- // TEST[setup:library] +// TEST[skip:This can output a warning, and asciidoc doesn't support allowed_warnings] [discrete] [[esql-rest-params]] diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 66911716ffee2..69936df7feff8 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -134,8 +134,7 @@ PUT _index_template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 0a0e36b63e6cd..6fd54f81d1222 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -26,8 +26,7 @@ PUT _component_template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } } diff --git a/docs/reference/indices/put-index-template-v1.asciidoc b/docs/reference/indices/put-index-template-v1.asciidoc index 86a8a54edd97f..75325edf77696 100644 --- a/docs/reference/indices/put-index-template-v1.asciidoc +++ b/docs/reference/indices/put-index-template-v1.asciidoc @@ -4,8 +4,8 @@ Create or update index template (legacy) ++++ -IMPORTANT: This documentation is about legacy index templates, -which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. +IMPORTANT: This documentation is about legacy index templates, +which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. For information about composable templates, see <>. Creates or updates an index template. @@ -27,8 +27,7 @@ PUT _template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } } @@ -67,8 +66,8 @@ that you can automatically apply when creating new indices. {es} applies templates to new indices based on an index pattern that matches the index name. -NOTE: Composable templates always take precedence over legacy templates. -If no composable template matches a new index, +NOTE: Composable templates always take precedence over legacy templates. +If no composable template matches a new index, matching legacy templates are applied according to their order. Index templates are only applied during index creation. @@ -263,4 +262,4 @@ The API returns the following response: "version" : 123 } } --------------------------------------------------- \ No newline at end of file +-------------------------------------------------- diff --git a/docs/reference/indices/simulate-template.asciidoc b/docs/reference/indices/simulate-template.asciidoc index c7397ace97886..fd825883dc20c 100644 --- a/docs/reference/indices/simulate-template.asciidoc +++ b/docs/reference/indices/simulate-template.asciidoc @@ -26,8 +26,7 @@ PUT _index_template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -74,19 +73,19 @@ privilege>> to use this API. ``:: (Optional, string) Name of the index template to simulate. -To test a template configuration before you add it to the cluster, +To test a template configuration before you add it to the cluster, omit this parameter and specify the template configuration in the request body. [[simulate-template-api-query-params]] ==== {api-query-parms-title} //// `cause`:: -(Optional, string) The reason for using the specified template for the simulation. +(Optional, string) The reason for using the specified template for the simulation. //// `create`:: -(Optional, Boolean) If `true`, the template passed in the body is -only used if no existing templates match the same index patterns. +(Optional, Boolean) If `true`, the template passed in the body is +only used if no existing templates match the same index patterns. If `false`, the simulation uses the template with the highest priority. Note that the template is not permanently added or updated in either case; it is only used for the simulation. @@ -114,7 +113,7 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=index-template-api .Properties of `overlapping` [%collapsible%open] ==== -`index_patterns`:: +`index_patterns`:: (array) Index patterns that the superseded template applies to. `name`:: @@ -123,7 +122,7 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=index-template-api `template`:: (object) -The settings, mappings, and aliases that would be applied to matching indices. +The settings, mappings, and aliases that would be applied to matching indices. + .Properties of `template` [%collapsible%open] @@ -227,8 +226,8 @@ The response shows the index settings, mappings, and aliases applied by the `fin [[simulate-template-config-ex]] ===== Simulating an arbitrary template configuration -To see what settings will be applied by a template before you add it to the cluster, -you can pass a template configuration in the request body. +To see what settings will be applied by a template before you add it to the cluster, +you can pass a template configuration in the request body. The specified template is used for the simulation if it has a higher priority than existing templates. [source,console] @@ -266,8 +265,8 @@ The response shows any overlapping templates with a lower priority. } }, "mappings" : { - "properties" : { - "@timestamp" : { + "properties" : { + "@timestamp" : { "type" : "date" } } diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc index 3fd296a167a6a..be20b49cf39f7 100644 --- a/docs/reference/ingest/common-log-format-example.asciidoc +++ b/docs/reference/ingest/common-log-format-example.asciidoc @@ -28,7 +28,7 @@ Pipelines**. image::images/ingest/ingest-pipeline-list.png[Kibana's Ingest Pipelines list view,align="center"] . Click **Create pipeline > New pipeline**. -. Set **Name** to `my-pipeline` and optionally add a description for the +. Set **Name** to `my-pipeline` and optionally add a description for the pipeline. . Add a <> to parse the log message: @@ -128,6 +128,7 @@ PUT _ingest/pipeline/my-pipeline // end::common-log-pipeline[] } ---- +// TEST[skip:This can output a warning, and asciidoc doesn't support allowed_warnings] //// -- diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index d7100745680ba..2279bb4acc1a8 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -50,6 +50,20 @@ tasks.named("thirdPartyAudit").configure { ) } +tasks.named("yamlRestTest").configure { + systemProperty 'tests.rest.blacklist', [ + // for some reason, allowed_warnings on the test isn't working here + 'ingest/30_date_processor/Test date processor with no timezone configured', + ].join(',') +} + +tasks.named("yamlRestTestV7CompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + // for some reason, allowed_warnings on the test isn't working here + 'ingest/30_date_processor/Test date processor with no timezone configured', + ].join(',') +} + tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.addAllowedWarningRegex("\\[types removal\\].*") } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml index 78fb895ea791c..8508534614c4b 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -1,3 +1,6 @@ +setup: + - requires: + test_runner_features: allowed_warnings --- teardown: - do: @@ -97,6 +100,8 @@ teardown: "Test date processor with no timezone configured": - do: + allowed_warnings: + - 'Date format [dd/MMM/yyyy:H:m:s Z] contains textual field specifiers that could change in JDK 23' ingest.put_pipeline: id: "my_pipeline" # sample formats from beats, featuring mongodb, icinga, apache @@ -165,6 +170,8 @@ teardown: - match: { acknowledged: true } - do: + allowed_warnings: + - 'Date format [dd/MMM/yyyy:H:m:s Z] contains textual field specifiers that could change in JDK 23' index: index: test id: "1" diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml index 9a7444c4ffc6c..301ff636d72be 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml @@ -1,6 +1,11 @@ +setup: + - requires: + test_runner_features: allowed_warnings --- "Test with date processor": - do: + allowed_warnings: + - 'Date format [dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23' ingest.put_pipeline: id: "_id" body: > @@ -41,6 +46,8 @@ - match: { acknowledged: true } - do: + allowed_warnings: + - 'Date format [dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23' index: index: test id: "1" @@ -70,6 +77,8 @@ --- "Test with date processor and ECS-v1": - do: + allowed_warnings: + - 'Date format [dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23' ingest.put_pipeline: id: "_id" body: > @@ -99,6 +108,8 @@ - match: { acknowledged: true } - do: + allowed_warnings: + - 'Date format [dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23' index: index: test id: "1" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 7c345b7d4d3ac..079b109d93044 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -1,6 +1,11 @@ +setup: + - requires: + test_runner_features: allowed_warnings --- "Test Index and Search locale dependent mappings / dates": - do: + allowed_warnings: + - 'Date format [E, d MMM yyyy HH:mm:ss Z] contains textual field specifiers that could change in JDK 23' indices.create: index: test_index body: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java index 4ce92610eff17..f2e662fb2d589 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java @@ -128,8 +128,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -159,8 +158,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -190,8 +188,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -259,8 +256,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -290,8 +286,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -335,8 +330,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -662,8 +656,7 @@ private TransportPutComposableIndexTemplateAction.Request sampleIndexTemplateRes "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -704,8 +697,7 @@ private TransportPutComposableIndexTemplateAction.Request sampleIndexTemplateRes "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index d475906be6b40..91baaedf4a669 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.logging.LogManager; import java.time.Clock; import java.time.Duration; @@ -391,13 +390,17 @@ public static ZonedDateTime nowWithMillisResolution(Clock clock) { // check for all textual fields, and localized zone offset // the weird thing with Z is to ONLY match 4 in a row, with no Z before or after (but those groups can also be empty) private static final Predicate CONTAINS_CHANGING_TEXT_SPECIFIERS = System.getProperty("java.locale.providers", "") - .contains("COMPAT") ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|ccc|eee|(? ", " <= ", " >= ", " = ", " != " }; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index e181f77f2bcef..7c6ea434c12df 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -29,6 +29,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" + - "Date format \\[MMMM] contains textual field specifiers that could change in JDK 23" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' @@ -50,6 +51,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" + - "Date format \\[MMMM] contains textual field specifiers that could change in JDK 23" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format'