From 55078d4c5ea6e4ca9425eae114c4b7864f64371d Mon Sep 17 00:00:00 2001
From: Liam Thompson <32779855+leemthompo@users.noreply.github.com>
Date: Mon, 30 Sep 2024 16:11:46 +0200
Subject: [PATCH 1/7] [DOCS] Fix heading level (#113800)
---
.../docs/connectors-postgresql.asciidoc | 360 ++++++++++++++++--
.../connector/docs/connectors-redis.asciidoc | 43 +--
2 files changed, 353 insertions(+), 50 deletions(-)
diff --git a/docs/reference/connector/docs/connectors-postgresql.asciidoc b/docs/reference/connector/docs/connectors-postgresql.asciidoc
index 861140cbd7b03..1fe28f867337c 100644
--- a/docs/reference/connector/docs/connectors-postgresql.asciidoc
+++ b/docs/reference/connector/docs/connectors-postgresql.asciidoc
@@ -1,5 +1,5 @@
[#es-connectors-postgresql]
-==== Elastic PostgreSQL connector reference
+=== Elastic PostgreSQL connector reference
++++
PostgreSQL
++++
@@ -13,8 +13,312 @@ This connector is written in Python using the {connectors-python}[Elastic connec
This connector uses the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
View the specific {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
+
+.Choose your connector reference
+*******************************
+Are you using an Elastic managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method.
+*******************************
+
+// //////// //// //// //// //// //// //// ////////
+// //////// NATIVE CONNECTOR REFERENCE ///////
+// //////// //// //// //// //// //// //// ////////
+
+[discrete#connectors-postgresql-native-connector-reference]
+=== *Elastic managed connector (Elastic Cloud)*
+
+.View *Elastic managed connector* reference
+
+[%collapsible]
+===============
+
+[discrete#connectors-postgresql-availability-prerequisites]
+==== Availability and prerequisites
+
+This connector is available as an *Elastic managed connector* in Elastic versions *8.8.0 and later*.
+To use this connector natively in Elastic Cloud, satisfy all <>.
+
+[discrete#connectors-postgresql-create-native-connector]
+==== Create a {service-name} connector
+include::_connectors-create-native.asciidoc[]
+
+[discrete#connectors-postgresql-usage]
+==== Usage
+
+To use this connector as an *Elastic managed connector*, use the *Connector* workflow.
+See <>.
+
+[TIP]
+====
+Users must set `track_commit_timestamp` to `on`.
+To do this, run `ALTER SYSTEM SET track_commit_timestamp = on;` in PostgreSQL server.
+====
+
+For additional operations, see <<-esconnectors-usage>>.
+
+[NOTE]
+====
+For an end-to-end example of the connector client workflow, see <>.
+====
+
+[discrete#connectors-postgresql-compatibility]
+==== Compatibility
+
+PostgreSQL versions 11 to 15 are compatible with the Elastic connector.
+
+[discrete#connectors-postgresql-configuration]
+==== Configuration
+
+Set the following configuration fields:
+
+Host::
+The server host address where the PostgreSQL instance is hosted.
+Examples:
++
+* `192.158.1.38`
+* `demo.instance.demo-region.demo.service.com`
+
+Port::
+The port where the PostgreSQL instance is hosted.
+Examples:
++
+* `5432` (default)
+
+Username::
+The username of the PostgreSQL account.
+
+Password::
+The password of the PostgreSQL account.
+
+Database::
+Name of the PostgreSQL database.
+Examples:
++
+* `employee_database`
+* `customer_database`
+
+Schema::
+The schema of the PostgreSQL database.
+
+Comma-separated List of Tables::
+A list of tables separated by commas.
+The PostgreSQL connector will fetch data from all tables present in the configured database, if the value is `*` .
+Default value is `*`.
+Examples:
++
+* `table_1, table_2`
+* `*`
++
+[WARNING]
+====
+This field can be bypassed when using advanced sync rules.
+====
+
+Enable SSL::
+Toggle to enable SSL verification.
+Disabled by default.
+
+SSL Certificate::
+Content of SSL certificate.
+If SSL is disabled, the `ssl_ca` value will be ignored.
++
+.*Expand* to see an example certificate
+[%collapsible]
+====
+```
+-----BEGIN CERTIFICATE-----
+MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT
+AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV
+BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ
+BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa
+MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO
+b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z
+dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV
+z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW
+CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj
+hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko
+i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t
+njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea
+MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr
+BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2
+MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv
+Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0
+gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4
+CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA
+iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS
+hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk
+xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr
+7RhLQyWn2u00L7/9Omw=
+-----END CERTIFICATE-----
+```
+====
+
+[discrete#connectors-postgresql-documents-syncs]
+==== Documents and syncs
+
+* Tables must be owned by a PostgreSQL user.
+* Tables with no primary key defined are skipped.
+* To fetch the last updated time in PostgreSQL, `track_commit_timestamp` must be set to `on`.
+Otherwise, all data will be indexed in every sync.
+
+[NOTE]
+====
+* Files bigger than 10 MB won't be extracted.
+* Permissions are not synced.
+**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment.
+====
+
+[discrete#connectors-postgresql-sync-rules]
+==== Sync rules
+
+<> are identical for all connectors and are available by default.
+
+[discrete#connectors-postgresql-sync-rules-advanced]
+===== Advanced sync rules
+
+[NOTE]
+====
+A <> is required for advanced sync rules to take effect.
+====
+
+Advanced sync rules are defined through a source-specific DSL JSON snippet.
+
+[discrete#connectors-postgresql-sync-rules-advanced-example-data]
+====== Example data
+
+Here is some example data that will be used in the following examples.
+
+[discrete#connectors-postgresql-sync-rules-advanced-example-data-1]
+======= `employee` table
+
+[cols="3*", options="header"]
+|===
+| emp_id | name | age
+| 3 | John | 28
+| 10 | Jane | 35
+| 14 | Alex | 22
+|===
+
+[discrete#connectors-postgresql-sync-rules-advanced-example-2]
+======= `customer` table
+
+[cols="3*", options="header"]
+|===
+| c_id | name | age
+| 2 | Elm | 24
+| 6 | Pine | 30
+| 9 | Oak | 34
+|===
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples]
+====== Advanced sync rules examples
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-1]
+======= Multiple table queries
+
+[source,js]
+----
+[
+ {
+ "tables": [
+ "employee"
+ ],
+ "query": "SELECT * FROM employee"
+ },
+ {
+ "tables": [
+ "customer"
+ ],
+ "query": "SELECT * FROM customer"
+ }
+]
+----
+// NOTCONSOLE
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-1-id-columns]
+======= Multiple table queries with `id_columns`
+
+In 8.15.0, we added a new optional `id_columns` field in our advanced sync rules for the PostgreSQL connector.
+Use the `id_columns` field to ingest tables which do not have a primary key. Include the names of unique fields so that the connector can use them to generate unique IDs for documents.
+
+[source,js]
+----
+[
+ {
+ "tables": [
+ "employee"
+ ],
+ "query": "SELECT * FROM employee",
+ "id_columns": ["emp_id"]
+ },
+ {
+ "tables": [
+ "customer"
+ ],
+ "query": "SELECT * FROM customer",
+ "id_columns": ["c_id"]
+ }
+]
+----
+// NOTCONSOLE
+
+This example uses the `id_columns` field to specify the unique fields `emp_id` and `c_id` for the `employee` and `customer` tables, respectively.
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-2]
+======= Filtering data with `WHERE` clause
+
+[source,js]
+----
+[
+ {
+ "tables": ["employee"],
+ "query": "SELECT * FROM employee WHERE emp_id > 5"
+ }
+]
+----
+// NOTCONSOLE
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-3]
+======= `JOIN` operations
+
+[source,js]
+----
+[
+ {
+ "tables": ["employee", "customer"],
+ "query": "SELECT * FROM employee INNER JOIN customer ON employee.emp_id = customer.c_id"
+ }
+]
+----
+// NOTCONSOLE
+
+[WARNING]
+====
+When using advanced rules, a query can bypass the configuration field `tables`.
+This will happen if the query specifies a table that doesn't appear in the configuration.
+This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables.
+====
+
+[discrete#connectors-postgresql-known-issues]
+==== Known issues
+
+There are no known issues for this connector.
+Refer to <> for a list of known issues for all connectors.
+
+[discrete#connectors-postgresql-troubleshooting]
+==== Troubleshooting
+
+See <>.
+
+[discrete#connectors-postgresql-security]
+==== Security
+
+See <>.
+
+// Closing the collapsible section
+===============
+
[discrete#es-connectors-postgresql-connector-client-reference]
-==== *Self-managed connector*
+=== *Self-managed connector*
.View *self-managed connector* reference
@@ -22,19 +326,19 @@ View the specific {connectors-python}/connectors/sources/{service-name-stub}.py[
===============
[discrete#es-connectors-postgresql-client-availability-prerequisites]
-===== Availability and prerequisites
+==== Availability and prerequisites
This connector is available as a self-managed *self-managed connector*.
-To use this connector, satisfy all //build-connector,self-managed connector requirements.
+To use this connector, satisfy all <>.
[discrete#es-connectors-postgresql-create-connector-client]
-===== Create a {service-name} connector
+==== Create a {service-name} connector
include::_connectors-create-client.asciidoc[]
[discrete#es-connectors-postgresql-client-usage]
-===== Usage
+==== Usage
-To use this connector as a *self-managed connector*, see //build-connector
+To use this connector as a *self-managed connector*, see <>.
[TIP]
====
Users must set `track_commit_timestamp` to `on`.
@@ -45,20 +349,20 @@ For additional operations, see.
[NOTE]
====
-For an end-to-end example of the self-managed connector workflow, see //postgresql-connector-client-tutorial.
+For an end-to-end example of the self-managed connector workflow, see <>.
====
[discrete#es-connectors-postgresql-client-compatibility]
-===== Compatibility
+==== Compatibility
PostgreSQL versions 11 to 15 are compatible with Elastic connector frameworks.
[discrete#es-connectors-postgresql-client-configuration]
-===== Configuration
+==== Configuration
[TIP]
====
-When using the //build-connector, self-managed connector workflow, initially these fields will use the default configuration set in the https://github.com/elastic/connectors-python/blob/{branch}/connectors/sources/postgresql.py[connector source code^].
+When using the <>, initially these fields will use the default configuration set in the https://github.com/elastic/connectors-python/blob/{branch}/connectors/sources/postgresql.py[connector source code^].
These configurable fields will be rendered with their respective *labels* in the Kibana UI.
Once connected, users will be able to update these values in Kibana.
@@ -150,12 +454,12 @@ xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr
====
[discrete#es-connectors-postgresql-client-docker]
-===== Deployment using Docker
+==== Deployment using Docker
include::_connectors-docker-instructions.asciidoc[]
[discrete#es-connectors-postgresql-client-documents-syncs]
-===== Documents and syncs
+==== Documents and syncs
* Tables must be owned by a PostgreSQL user.
* Tables with no primary key defined are skipped.
@@ -170,12 +474,12 @@ Otherwise, all data will be indexed in every sync.
====
[discrete#es-connectors-postgresql-client-sync-rules]
-===== Sync rules
+==== Sync rules
//sync-rules-basic,Basic sync rules are identical for all connectors and are available by default.
[discrete#es-connectors-postgresql-client-sync-rules-advanced]
-====== Advanced sync rules
+===== Advanced sync rules
[NOTE]
====
@@ -185,12 +489,12 @@ A //connectors-sync-types-full, full sync is required for advanced sync rules to
Advanced sync rules are defined through a source-specific DSL JSON snippet.
[discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data]
-======= Example data
+====== Example data
Here is some example data that will be used in the following examples.
[discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data-1]
-======== `employee` table
+======= `employee` table
[cols="3*", options="header"]
|===
@@ -201,7 +505,7 @@ Here is some example data that will be used in the following examples.
|===
[discrete#es-connectors-postgresql-client-sync-rules-advanced-example-2]
-======== `customer` table
+======= `customer` table
[cols="3*", options="header"]
|===
@@ -212,7 +516,7 @@ Here is some example data that will be used in the following examples.
|===
[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples]
-======= Advanced sync rules examples
+====== Advanced sync rules examples
[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-1]
======== Multiple table queries
@@ -301,10 +605,10 @@ This can also happen if the configuration specifies `*` to fetch all tables whil
====
[discrete#es-connectors-postgresql-client-client-operations-testing]
-===== End-to-end testing
+==== End-to-end testing
The connector framework enables operators to run functional tests against a real data source.
-Refer to //build-connector-testing for more details.
+Refer to <> for more details.
To perform E2E testing for the PostgreSQL connector, run the following command:
@@ -321,20 +625,20 @@ make ftest NAME=postgresql DATA_SIZE=small
----
[discrete#es-connectors-postgresql-client-known-issues]
-===== Known issues
+==== Known issues
There are no known issues for this connector.
-Refer to //connectors-known-issues for a list of known issues for all connectors.
+Refer to <> for a list of known issues for all connectors.
[discrete#es-connectors-postgresql-client-troubleshooting]
-===== Troubleshooting
+==== Troubleshooting
-See //connectors-troubleshooting.
+See <>.
[discrete#es-connectors-postgresql-client-security]
-===== Security
+==== Security
-See //connectors-security.
+See <>.
// Closing the collapsible section
-===============
+===============
\ No newline at end of file
diff --git a/docs/reference/connector/docs/connectors-redis.asciidoc b/docs/reference/connector/docs/connectors-redis.asciidoc
index 5dbd008ee5932..7aad7b0b41497 100644
--- a/docs/reference/connector/docs/connectors-redis.asciidoc
+++ b/docs/reference/connector/docs/connectors-redis.asciidoc
@@ -1,5 +1,5 @@
[#es-connectors-redis]
-==== Redis connector reference
+=== Redis connector reference
++++
Redis
++++
@@ -12,7 +12,7 @@ The Redis connector is built with the Elastic connectors Python framework and is
View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
[discrete#es-connectors-redis-connector-availability-and-prerequisites]
-===== Availability and prerequisites
+==== Availability and prerequisites
This connector was introduced in Elastic *8.13.0*, available as a *self-managed* self-managed connector.
@@ -29,19 +29,19 @@ This connector is in *technical preview* and is subject to change. The design an
====
[discrete#es-connectors-redis-connector-usage]
-===== Usage
+==== Usage
To set up this connector in the UI, select the *Redis* tile when creating a new connector under *Search -> Connectors*.
For additional operations, see <>.
[discrete#es-connectors-redis-connector-docker]
-===== Deploy with Docker
+==== Deploy with Docker
include::_connectors-docker-instructions.asciidoc[]
[discrete#es-connectors-redis-connector-configuration]
-===== Configuration
+==== Configuration
`host` (required)::
The IP of your Redis server/cloud. Example:
@@ -89,7 +89,7 @@ Specifies the client private key. The value of the key is used to validate the c
Depends on `mutual_tls_enabled`.
[discrete#es-connectors-redis-connector-documents-and-syncs]
-===== Documents and syncs
+==== Documents and syncs
The connector syncs the following objects and entities:
@@ -102,12 +102,12 @@ The connector syncs the following objects and entities:
====
[discrete#es-connectors-redis-connector-sync-rules]
-===== Sync rules
+==== Sync rules
<> are identical for all connectors and are available by default.
[discrete#es-connectors-redis-connector-advanced-sync-rules]
-===== Advanced Sync Rules
+==== Advanced Sync Rules
<> are defined through a source-specific DSL JSON snippet.
@@ -134,10 +134,10 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
====
[discrete#es-connectors-redis-connector-advanced-sync-rules-examples]
-====== Advanced sync rules examples
+===== Advanced sync rules examples
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-1]
-======= Example 1
+====== Example 1
*Fetch database records where keys start with `alpha`*:
@@ -153,7 +153,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
// NOTCONSOLE
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-2]
-======= Example 2
+====== Example 2
*Fetch database records with exact match by specifying the full key name:*
@@ -169,7 +169,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
// NOTCONSOLE
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-3]
-======= Example 3
+====== Example 3
*Fetch database records where keys start with `test1`, `test2` or `test3`:*
@@ -180,13 +180,12 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
"database": 0,
"key_pattern": "test[123]"
}
-]
----
// NOTCONSOLE
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-4]
-======= Example 4
+====== Example 4
*Exclude database records where keys start with `test1`, `test2` or `test3`:*
@@ -202,7 +201,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
// NOTCONSOLE
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-5]
-======= Example 5
+====== Example 5
*Fetch all database records:*
@@ -218,7 +217,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
// NOTCONSOLE
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-6]
-======= Example 6
+====== Example 6
*Fetch all database records where type is `SET`:*
@@ -235,7 +234,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
// NOTCONSOLE
[discrete#es-connectors-redis-connector-advanced-sync-rules-example-7]
-======= Example 7
+====== Example 7
*Fetch database records where type is `SET`*:
@@ -251,10 +250,10 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
// NOTCONSOLE
[discrete#es-connectors-redis-connector-connector-client-operations]
-===== Connector Client operations
+==== Connector Client operations
[discrete#es-connectors-redis-connector-end-to-end-testing]
-====== End-to-end Testing
+===== End-to-end Testing
The connector framework enables operators to run functional tests against a real data source, using Docker Compose.
You don't need a running Elasticsearch instance or Redis source to run this test.
@@ -276,7 +275,7 @@ make ftest NAME=redis DATA_SIZE=small
By default, `DATA_SIZE=MEDIUM`.
[discrete#es-connectors-redis-connector-known-issues]
-===== Known issues
+==== Known issues
* The last modified time is unavailable when retrieving keys/values from the Redis database.
As a result, *all objects* are indexed each time an advanced sync rule query is executed.
@@ -284,11 +283,11 @@ As a result, *all objects* are indexed each time an advanced sync rule query is
Refer to <> for a list of known issues for all connectors.
[discrete#es-connectors-redis-connector-troubleshooting]
-===== Troubleshooting
+==== Troubleshooting
See <>.
[discrete#es-connectors-redis-connector-security]
-===== Security
+==== Security
See <>.
\ No newline at end of file
From bf329e2c484c94de44e4b37e0c11b26689443a41 Mon Sep 17 00:00:00 2001
From: Tim Grein
Date: Mon, 30 Sep 2024 16:58:23 +0200
Subject: [PATCH 2/7] [Inference API] Propagate infer trace context to EIS
(#113407)
---
.../ElasticInferenceServiceActionCreator.java | 8 +++--
...ServiceSparseEmbeddingsRequestManager.java | 10 +++++--
...ferenceServiceSparseEmbeddingsRequest.java | 30 +++++++++++++++++--
.../elastic/ElasticInferenceService.java | 18 ++++++++++-
.../inference/telemetry/TraceContext.java | 10 +++++++
...ticInferenceServiceActionCreatorTests.java | 13 +++++---
...ceServiceSparseEmbeddingsRequestTests.java | 22 +++++++++++++-
7 files changed, 99 insertions(+), 12 deletions(-)
create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
index ea2295979c480..c8ada6e535b63 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
@@ -13,6 +13,7 @@
import org.elasticsearch.xpack.inference.external.http.sender.Sender;
import org.elasticsearch.xpack.inference.services.ServiceComponents;
import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
import java.util.Objects;
@@ -24,14 +25,17 @@ public class ElasticInferenceServiceActionCreator implements ElasticInferenceSer
private final ServiceComponents serviceComponents;
- public ElasticInferenceServiceActionCreator(Sender sender, ServiceComponents serviceComponents) {
+ private final TraceContext traceContext;
+
+ public ElasticInferenceServiceActionCreator(Sender sender, ServiceComponents serviceComponents, TraceContext traceContext) {
this.sender = Objects.requireNonNull(sender);
this.serviceComponents = Objects.requireNonNull(serviceComponents);
+ this.traceContext = traceContext;
}
@Override
public ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model) {
- var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents);
+ var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents, traceContext);
var errorMessage = constructFailedToSendRequestMessage(model.uri(), "Elastic Inference Service sparse embeddings");
return new SenderExecutableAction(sender, requestManager, errorMessage);
}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
index b59ac54d5cbb6..e7ee41525f07d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
@@ -19,6 +19,7 @@
import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceSparseEmbeddingsResponseEntity;
import org.elasticsearch.xpack.inference.services.ServiceComponents;
import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
import java.util.List;
import java.util.function.Supplier;
@@ -35,6 +36,8 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestManager extends Elast
private final Truncator truncator;
+ private final TraceContext traceContext;
+
private static ResponseHandler createSparseEmbeddingsHandler() {
return new ElasticInferenceServiceResponseHandler(
"Elastic Inference Service sparse embeddings",
@@ -44,11 +47,13 @@ private static ResponseHandler createSparseEmbeddingsHandler() {
public ElasticInferenceServiceSparseEmbeddingsRequestManager(
ElasticInferenceServiceSparseEmbeddingsModel model,
- ServiceComponents serviceComponents
+ ServiceComponents serviceComponents,
+ TraceContext traceContext
) {
super(serviceComponents.threadPool(), model);
this.model = model;
this.truncator = serviceComponents.truncator();
+ this.traceContext = traceContext;
}
@Override
@@ -64,7 +69,8 @@ public void execute(
ElasticInferenceServiceSparseEmbeddingsRequest request = new ElasticInferenceServiceSparseEmbeddingsRequest(
truncator,
truncatedInput,
- model
+ model,
+ traceContext
);
execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
index 41a2ef1c3ccda..d445a779f8230 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
@@ -12,11 +12,13 @@
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.message.BasicHeader;
import org.elasticsearch.common.Strings;
+import org.elasticsearch.tasks.Task;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xpack.inference.common.Truncator;
import org.elasticsearch.xpack.inference.external.request.HttpRequest;
import org.elasticsearch.xpack.inference.external.request.Request;
import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
import java.net.URI;
import java.nio.charset.StandardCharsets;
@@ -31,15 +33,19 @@ public class ElasticInferenceServiceSparseEmbeddingsRequest implements ElasticIn
private final Truncator.TruncationResult truncationResult;
private final Truncator truncator;
+ private final TraceContext traceContext;
+
public ElasticInferenceServiceSparseEmbeddingsRequest(
Truncator truncator,
Truncator.TruncationResult truncationResult,
- ElasticInferenceServiceSparseEmbeddingsModel model
+ ElasticInferenceServiceSparseEmbeddingsModel model,
+ TraceContext traceContext
) {
this.truncator = truncator;
this.truncationResult = truncationResult;
this.model = Objects.requireNonNull(model);
this.uri = model.uri();
+ this.traceContext = traceContext;
}
@Override
@@ -50,6 +56,10 @@ public HttpRequest createHttpRequest() {
ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8));
httpPost.setEntity(byteEntity);
+ if (traceContext != null) {
+ propagateTraceContext(httpPost);
+ }
+
httpPost.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()));
return new HttpRequest(httpPost, getInferenceEntityId());
@@ -65,11 +75,15 @@ public URI getURI() {
return this.uri;
}
+ public TraceContext getTraceContext() {
+ return traceContext;
+ }
+
@Override
public Request truncate() {
var truncatedInput = truncator.truncate(truncationResult.input());
- return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model);
+ return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model, traceContext);
}
@Override
@@ -77,4 +91,16 @@ public boolean[] getTruncationInfo() {
return truncationResult.truncated().clone();
}
+ private void propagateTraceContext(HttpPost httpPost) {
+ var traceParent = traceContext.traceParent();
+ var traceState = traceContext.traceState();
+
+ if (traceParent != null) {
+ httpPost.setHeader(Task.TRACE_PARENT_HTTP_HEADER, traceParent);
+ }
+
+ if (traceState != null) {
+ httpPost.setHeader(Task.TRACE_STATE, traceState);
+ }
+ }
}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
index 103ddd4c5c5ea..abbe893823b96 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
@@ -23,6 +23,7 @@
import org.elasticsearch.inference.ModelSecrets;
import org.elasticsearch.inference.TaskType;
import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
@@ -34,6 +35,7 @@
import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
import org.elasticsearch.xpack.inference.services.SenderService;
import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
import java.util.List;
import java.util.Map;
@@ -75,8 +77,13 @@ protected void doInfer(
return;
}
+ // We extract the trace context here as it's sufficient to propagate the trace information of the REST request,
+ // which handles the request to the inference API overall (including the outgoing request, which is started in a new thread
+ // generating a different "traceparent" as every task and every REST request creates a new span).
+ var currentTraceInfo = getCurrentTraceInfo();
+
ElasticInferenceServiceModel elasticInferenceServiceModel = (ElasticInferenceServiceModel) model;
- var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents());
+ var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents(), currentTraceInfo);
var action = elasticInferenceServiceModel.accept(actionCreator, taskSettings);
action.execute(inputs, timeout, listener);
@@ -258,4 +265,13 @@ private ElasticInferenceServiceSparseEmbeddingsModel updateModelWithEmbeddingDet
return new ElasticInferenceServiceSparseEmbeddingsModel(model, serviceSettings);
}
+
+ private TraceContext getCurrentTraceInfo() {
+ var threadPool = getServiceComponents().threadPool();
+
+ var traceParent = threadPool.getThreadContext().getHeader(Task.TRACE_PARENT);
+ var traceState = threadPool.getThreadContext().getHeader(Task.TRACE_STATE);
+
+ return new TraceContext(traceParent, traceState);
+ }
}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java
new file mode 100644
index 0000000000000..05654ed146f16
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.telemetry;
+
+public record TraceContext(String traceParent, String traceState) {}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
index 1081a60ba6866..02b09917d0065 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
@@ -25,6 +25,7 @@
import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests;
import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
import org.junit.After;
import org.junit.Before;
@@ -89,7 +90,7 @@ public void testExecute_ReturnsSuccessfulResponse_ForElserAction() throws IOExce
webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
- var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+ var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
var action = actionCreator.create(model);
PlainActionFuture listener = new PlainActionFuture<>();
@@ -145,7 +146,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOEx
webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
- var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+ var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
var action = actionCreator.create(model);
PlainActionFuture listener = new PlainActionFuture<>();
@@ -197,7 +198,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating() throws IOExc
webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
- var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+ var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
var action = actionCreator.create(model);
PlainActionFuture listener = new PlainActionFuture<>();
@@ -257,7 +258,7 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException {
// truncated to 1 token = 3 characters
var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer), 1);
- var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+ var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
var action = actionCreator.create(model);
PlainActionFuture listener = new PlainActionFuture<>();
@@ -286,4 +287,8 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException {
}
}
+ private TraceContext createTraceContext() {
+ return new TraceContext(randomAlphaOfLength(10), randomAlphaOfLength(10));
+ }
+
}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
index 0f2c859fb62d5..9d3bbe2ed12ae 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
@@ -9,11 +9,13 @@
import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xpack.inference.common.Truncator;
import org.elasticsearch.xpack.inference.common.TruncatorTests;
import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
import java.io.IOException;
import java.util.List;
@@ -42,6 +44,23 @@ public void testCreateHttpRequest() throws IOException {
assertThat(requestMap.get("input"), is(List.of(input)));
}
+ public void testTraceContextPropagatedThroughHTTPHeaders() {
+ var url = "http://eis-gateway.com";
+ var input = "input";
+
+ var request = createRequest(url, input);
+ var httpRequest = request.createHttpRequest();
+
+ assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+ var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+ var traceParent = request.getTraceContext().traceParent();
+ var traceState = request.getTraceContext().traceState();
+
+ assertThat(httpPost.getLastHeader(Task.TRACE_PARENT_HTTP_HEADER).getValue(), is(traceParent));
+ assertThat(httpPost.getLastHeader(Task.TRACE_STATE).getValue(), is(traceState));
+ }
+
public void testTruncate_ReducesInputTextSizeByHalf() throws IOException {
var url = "http://eis-gateway.com";
var input = "abcd";
@@ -75,7 +94,8 @@ public ElasticInferenceServiceSparseEmbeddingsRequest createRequest(String url,
return new ElasticInferenceServiceSparseEmbeddingsRequest(
TruncatorTests.createTruncator(),
new Truncator.TruncationResult(List.of(input), new boolean[] { false }),
- embeddingsModel
+ embeddingsModel,
+ new TraceContext(randomAlphaOfLength(10), randomAlphaOfLength(10))
);
}
}
From 9365efb970ce8ddec8e2f1c1f4f198a3f9755248 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Mon, 30 Sep 2024 18:12:24 +0300
Subject: [PATCH 3/7] Restore node feature (#113805)
This got rolled back as part of #113692, but the change had already
rolled out to QA.
---
.../main/java/org/elasticsearch/index/mapper/MapperFeatures.java | 1 +
.../main/java/org/elasticsearch/index/mapper/ObjectMapper.java | 1 +
2 files changed, 2 insertions(+)
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
index 2f665fd5d1e6a..31df558492b35 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -36,6 +36,7 @@ public Set getFeatures() {
NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS,
BooleanFieldMapper.BOOLEAN_DIMENSION,
ObjectMapper.SUBOBJECTS_AUTO,
+ ObjectMapper.SUBOBJECTS_AUTO_FIXES,
KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE,
SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX,
Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE,
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index f9c854749e885..40019566adaa8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -45,6 +45,7 @@ public class ObjectMapper extends Mapper {
public static final String CONTENT_TYPE = "object";
static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source";
static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto");
+ static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes");
/**
* Enhances the previously boolean option for subobjects support with an intermediate mode `auto` that uses
From a6b104d8433c28b1a35747fbad9a26ac9a789d3d Mon Sep 17 00:00:00 2001
From: Patrick Doyle <810052+prdoyle@users.noreply.github.com>
Date: Mon, 30 Sep 2024 11:23:32 -0400
Subject: [PATCH 4/7] Fix max file size check to use getMaxFileSize (#113723)
* Fix max file size check to use getMaxFileSize
* Update docs/changelog/113723.yaml
* CURSE YOU SPOTLESS
---
docs/changelog/113723.yaml | 6 ++++++
.../java/org/elasticsearch/bootstrap/BootstrapChecks.java | 8 ++++----
.../org/elasticsearch/bootstrap/BootstrapChecksTests.java | 4 ++--
3 files changed, 12 insertions(+), 6 deletions(-)
create mode 100644 docs/changelog/113723.yaml
diff --git a/docs/changelog/113723.yaml b/docs/changelog/113723.yaml
new file mode 100644
index 0000000000000..2cbcf49102719
--- /dev/null
+++ b/docs/changelog/113723.yaml
@@ -0,0 +1,6 @@
+pr: 113723
+summary: Fix max file size check to use `getMaxFileSize`
+area: Infra/Core
+type: bug
+issues:
+ - 113705
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 566c8001dea56..021ad8127a2d0 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -412,12 +412,12 @@ static class MaxFileSizeCheck implements BootstrapCheck {
@Override
public BootstrapCheckResult check(BootstrapContext context) {
- final long maxFileSize = getMaxFileSize();
+ final long maxFileSize = getProcessLimits().maxFileSize();
if (maxFileSize != Long.MIN_VALUE && maxFileSize != ProcessLimits.UNLIMITED) {
final String message = String.format(
Locale.ROOT,
"max file size [%d] for user [%s] is too low, increase to [unlimited]",
- getMaxFileSize(),
+ maxFileSize,
BootstrapInfo.getSystemProperties().get("user.name")
);
return BootstrapCheckResult.failure(message);
@@ -426,8 +426,8 @@ public BootstrapCheckResult check(BootstrapContext context) {
}
}
- long getMaxFileSize() {
- return NativeAccess.instance().getProcessLimits().maxVirtualMemorySize();
+ protected ProcessLimits getProcessLimits() {
+ return NativeAccess.instance().getProcessLimits();
}
@Override
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
index 9a51757189f8b..8c3749dbd3a45 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
@@ -389,8 +389,8 @@ public void testMaxFileSizeCheck() throws NodeValidationException {
final AtomicLong maxFileSize = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
final BootstrapChecks.MaxFileSizeCheck check = new BootstrapChecks.MaxFileSizeCheck() {
@Override
- long getMaxFileSize() {
- return maxFileSize.get();
+ protected ProcessLimits getProcessLimits() {
+ return new ProcessLimits(ProcessLimits.UNKNOWN, ProcessLimits.UNKNOWN, maxFileSize.get());
}
};
From e57fc24af5a452cd5617e62f34e507da14894b1d Mon Sep 17 00:00:00 2001
From: Nhat Nguyen
Date: Mon, 30 Sep 2024 10:31:24 -0700
Subject: [PATCH 5/7] Add indices metrics for each index mode (#113737)
This change introduces index metrics per node, grouped by by index mode.
For each index mode, we track the number of indices, document count, and
store size. These metrics will help compare the usage of logsdb and
time_series indices to standard indices.
Other metrics, such as index longevity and newly created indices, could
be added in a follow-up.
Here is the list of 9 metrics introduced in this PR:
es.indices.standard.total
es.indices.standard.docs.total
es.indices.standard.bytes.total
es.indices.time_series.total
es.indices.time_series.docs.total
es.indices.time_series.bytes.total
es.indices.logsdb.total
es.indices.logsdb.docs.total
es.indices.logsdb.bytes.total
---
.../monitor/metrics/IndicesMetricsIT.java | 245 ++++++++++++++++++
.../monitor/metrics/IndicesMetrics.java | 177 +++++++++++++
.../java/org/elasticsearch/node/Node.java | 4 +
.../elasticsearch/node/NodeConstruction.java | 3 +
4 files changed, 429 insertions(+)
create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
create mode 100644 server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
new file mode 100644
index 0000000000000..b72257b884f08
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.monitor.metrics;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
+public class IndicesMetricsIT extends ESIntegTestCase {
+
+ public static class TestAPMInternalSettings extends Plugin {
+ @Override
+ public List> getSettings() {
+ return List.of(
+ Setting.timeSetting("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0), Setting.Property.NodeScope)
+ );
+ }
+ }
+
+ @Override
+ protected Collection> nodePlugins() {
+ return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class);
+ }
+
+ @Override
+ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+ return Settings.builder()
+ .put(super.nodeSettings(nodeOrdinal, otherSettings))
+ .put("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0)) // disable metrics cache refresh delay
+ .build();
+ }
+
+ static final String STANDARD_INDEX_COUNT = "es.indices.standard.total";
+ static final String STANDARD_DOCS_COUNT = "es.indices.standard.docs.total";
+ static final String STANDARD_BYTES_SIZE = "es.indices.standard.bytes.total";
+
+ static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total";
+ static final String TIME_SERIES_DOCS_COUNT = "es.indices.time_series.docs.total";
+ static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.bytes.total";
+
+ static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total";
+ static final String LOGSDB_DOCS_COUNT = "es.indices.logsdb.docs.total";
+ static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.bytes.total";
+
+ public void testIndicesMetrics() {
+ String node = internalCluster().startNode();
+ ensureStableCluster(1);
+ final TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, node)
+ .filterPlugins(TestTelemetryPlugin.class)
+ .findFirst()
+ .orElseThrow();
+ telemetry.resetMeter();
+ long numStandardIndices = randomIntBetween(1, 5);
+ long numStandardDocs = populateStandardIndices(numStandardIndices);
+ collectThenAssertMetrics(
+ telemetry,
+ 1,
+ Map.of(
+ STANDARD_INDEX_COUNT,
+ equalTo(numStandardIndices),
+ STANDARD_DOCS_COUNT,
+ equalTo(numStandardDocs),
+ STANDARD_BYTES_SIZE,
+ greaterThan(0L),
+
+ TIME_SERIES_INDEX_COUNT,
+ equalTo(0L),
+ TIME_SERIES_DOCS_COUNT,
+ equalTo(0L),
+ TIME_SERIES_BYTES_SIZE,
+ equalTo(0L),
+
+ LOGSDB_INDEX_COUNT,
+ equalTo(0L),
+ LOGSDB_DOCS_COUNT,
+ equalTo(0L),
+ LOGSDB_BYTES_SIZE,
+ equalTo(0L)
+ )
+ );
+
+ long numTimeSeriesIndices = randomIntBetween(1, 5);
+ long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices);
+ collectThenAssertMetrics(
+ telemetry,
+ 2,
+ Map.of(
+ STANDARD_INDEX_COUNT,
+ equalTo(numStandardIndices),
+ STANDARD_DOCS_COUNT,
+ equalTo(numStandardDocs),
+ STANDARD_BYTES_SIZE,
+ greaterThan(0L),
+
+ TIME_SERIES_INDEX_COUNT,
+ equalTo(numTimeSeriesIndices),
+ TIME_SERIES_DOCS_COUNT,
+ equalTo(numTimeSeriesDocs),
+ TIME_SERIES_BYTES_SIZE,
+ greaterThan(20L),
+
+ LOGSDB_INDEX_COUNT,
+ equalTo(0L),
+ LOGSDB_DOCS_COUNT,
+ equalTo(0L),
+ LOGSDB_BYTES_SIZE,
+ equalTo(0L)
+ )
+ );
+
+ long numLogsdbIndices = randomIntBetween(1, 5);
+ long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices);
+ collectThenAssertMetrics(
+ telemetry,
+ 3,
+ Map.of(
+ STANDARD_INDEX_COUNT,
+ equalTo(numStandardIndices),
+ STANDARD_DOCS_COUNT,
+ equalTo(numStandardDocs),
+ STANDARD_BYTES_SIZE,
+ greaterThan(0L),
+
+ TIME_SERIES_INDEX_COUNT,
+ equalTo(numTimeSeriesIndices),
+ TIME_SERIES_DOCS_COUNT,
+ equalTo(numTimeSeriesDocs),
+ TIME_SERIES_BYTES_SIZE,
+ greaterThan(20L),
+
+ LOGSDB_INDEX_COUNT,
+ equalTo(numLogsdbIndices),
+ LOGSDB_DOCS_COUNT,
+ equalTo(numLogsdbDocs),
+ LOGSDB_BYTES_SIZE,
+ greaterThan(0L)
+ )
+ );
+ }
+
+ void collectThenAssertMetrics(TestTelemetryPlugin telemetry, int times, Map> matchers) {
+ telemetry.collect();
+ for (Map.Entry> e : matchers.entrySet()) {
+ String name = e.getKey();
+ List measurements = telemetry.getLongGaugeMeasurement(name);
+ assertThat(name, measurements, hasSize(times));
+ assertThat(name, measurements.getLast().getLong(), e.getValue());
+ }
+ }
+
+ int populateStandardIndices(long numIndices) {
+ int totalDocs = 0;
+ for (int i = 0; i < numIndices; i++) {
+ String indexName = "standard-" + i;
+ createIndex(indexName);
+ int numDocs = between(1, 5);
+ for (int d = 0; d < numDocs; d++) {
+ indexDoc(indexName, Integer.toString(d), "f", Integer.toString(d));
+ }
+ totalDocs += numDocs;
+ flush(indexName);
+ }
+ return totalDocs;
+ }
+
+ int populateTimeSeriesIndices(long numIndices) {
+ int totalDocs = 0;
+ for (int i = 0; i < numIndices; i++) {
+ String indexName = "time_series-" + i;
+ Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host")).build();
+ client().admin()
+ .indices()
+ .prepareCreate(indexName)
+ .setSettings(settings)
+ .setMapping(
+ "@timestamp",
+ "type=date",
+ "host",
+ "type=keyword,time_series_dimension=true",
+ "cpu",
+ "type=long,time_series_metric=gauge"
+ )
+ .get();
+ long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
+ int numDocs = between(1, 5);
+ for (int d = 0; d < numDocs; d++) {
+ timestamp += between(1, 5) * 1000L;
+ client().prepareIndex(indexName)
+ .setSource("@timestamp", timestamp, "host", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100))
+ .get();
+ }
+ totalDocs += numDocs;
+ flush(indexName);
+ }
+ return totalDocs;
+ }
+
+ int populateLogsdbIndices(long numIndices) {
+ int totalDocs = 0;
+ for (int i = 0; i < numIndices; i++) {
+ String indexName = "logsdb-" + i;
+ Settings settings = Settings.builder().put("mode", "logsdb").build();
+ client().admin()
+ .indices()
+ .prepareCreate(indexName)
+ .setSettings(settings)
+ .setMapping("@timestamp", "type=date", "host.name", "type=keyword", "cpu", "type=long")
+ .get();
+ long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
+ int numDocs = between(1, 5);
+ for (int d = 0; d < numDocs; d++) {
+ timestamp += between(1, 5) * 1000L;
+ client().prepareIndex(indexName)
+ .setSource("@timestamp", timestamp, "host.name", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100))
+ .get();
+ }
+ totalDocs += numDocs;
+ flush(indexName);
+ }
+ return totalDocs;
+ }
+}
diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
new file mode 100644
index 0000000000000..17e290283d5e0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.monitor.metrics;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.util.SingleObjectCache;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.shard.IllegalIndexShardStateException;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.telemetry.metric.LongWithAttributes;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * {@link IndicesMetrics} monitors index statistics on an Elasticsearch node and exposes them as metrics
+ * through the provided {@link MeterRegistry}. It tracks the current total number of indices, document count, and
+ * store size (in bytes) for each index mode.
+ */
+public class IndicesMetrics extends AbstractLifecycleComponent {
+ private final Logger logger = LogManager.getLogger(IndicesMetrics.class);
+ private final MeterRegistry registry;
+ private final List metrics = new ArrayList<>();
+ private final IndicesStatsCache stateCache;
+
+ public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService, TimeValue metricsInterval) {
+ this.registry = meterRegistry;
+ // Use half of the update interval to ensure that results aren't cached across updates,
+ // while preventing the cache from expiring when reading different gauges within the same update.
+ var cacheExpiry = new TimeValue(metricsInterval.getMillis() / 2);
+ this.stateCache = new IndicesStatsCache(indicesService, cacheExpiry);
+ }
+
+ private static List registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) {
+ List metrics = new ArrayList<>(IndexMode.values().length * 3);
+ assert IndexMode.values().length == 3 : "index modes have changed";
+ for (IndexMode indexMode : IndexMode.values()) {
+ String name = indexMode.getName();
+ metrics.add(
+ registry.registerLongGauge(
+ "es.indices." + name + ".total",
+ "total number of " + name + " indices",
+ "unit",
+ () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numIndices)
+ )
+ );
+ metrics.add(
+ registry.registerLongGauge(
+ "es.indices." + name + ".docs.total",
+ "total documents of " + name + " indices",
+ "unit",
+ () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numDocs)
+ )
+ );
+ metrics.add(
+ registry.registerLongGauge(
+ "es.indices." + name + ".bytes.total",
+ "total size in bytes of " + name + " indices",
+ "unit",
+ () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numBytes)
+ )
+ );
+ }
+ return metrics;
+ }
+
+ @Override
+ protected void doStart() {
+ metrics.addAll(registerAsyncMetrics(registry, stateCache));
+ }
+
+ @Override
+ protected void doStop() {
+ stateCache.stopRefreshing();
+ }
+
+ @Override
+ protected void doClose() throws IOException {
+ metrics.forEach(metric -> {
+ try {
+ metric.close();
+ } catch (Exception e) {
+ logger.warn("metrics close() method should not throw Exception", e);
+ }
+ });
+ }
+
+ static class IndexStats {
+ int numIndices = 0;
+ long numDocs = 0;
+ long numBytes = 0;
+ }
+
+ private static class IndicesStatsCache extends SingleObjectCache