Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Limit prometheus exemplar labels #6791

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Limit prometheus exemplar labels #6791

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2bd6126
Limit prometheus exemplar labels (open-telemetry#6770)
harshitrjpt Oct 15, 2024
965a8fa
Use codePointCount instead of length (open-telemetry#6770)
harshitrjpt Oct 15, 2024
5a536f2
Merge branch 'open-telemetry:main' into bug/6770-limit-prometheus-exe…
harshitrjpt Oct 17, 2024
280c548
Better warn log phrasing (open-telemetry#6770)
harshitrjpt Oct 17, 2024
55a1bac
Better var name label
harshitrjpt Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 29 additions & 13 deletions ...ometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.InfoSnapshot;
import io.prometheus.metrics.model.snapshots.InfoSnapshot.InfoDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.Label;
import io.prometheus.metrics.model.snapshots.Labels;
import io.prometheus.metrics.model.snapshots.MetricMetadata;
import io.prometheus.metrics.model.snapshots.MetricSnapshot;
Expand All @@ -51,6 +52,7 @@
import io.prometheus.metrics.model.snapshots.SummarySnapshot;
import io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.Unit;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -77,6 +79,7 @@
private static final String OTEL_SCOPE_VERSION = "otel_scope_version";
private static final long NANOS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1);
static final int MAX_CACHE_SIZE = 10;
static final int EXEMPLAR_MAX_RUNES = 128;

private final boolean otelScopeEnabled;
@Nullable private final Predicate<String> allowedResourceAttributesFilter;
Expand Down Expand Up @@ -400,29 +403,42 @@
return Exemplars.of(result);
}

@Nullable
private Exemplar convertExemplar(double value, ExemplarData exemplar) {
SpanContext spanContext = exemplar.getSpanContext();
Labels labels = Labels.EMPTY;
if (spanContext.isValid()) {
return new Exemplar(
value,
labels =
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
null,
null,
laurit marked this conversation as resolved.
Show resolved Hide resolved
exemplar.getFilteredAttributes(),
"trace_id",
spanContext.getTraceId(),
"span_id",
spanContext.getSpanId()),
exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
spanContext.getSpanId());
} else {
return new Exemplar(
value,
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
exemplar.getFilteredAttributes()),
exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
labels = convertAttributes(null, null, exemplar.getFilteredAttributes());

Check warning on line 421 in exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java

View check run for this annotation

Codecov / codecov/patch

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java#L421 

Added line #L421 was not covered by tests
}
int runes = getRunes(labels);
if (runes > EXEMPLAR_MAX_RUNES) {
THROTTLING_LOGGER.log(
Level.WARNING,
"exemplar labels have " + runes + " runes, exceeding the limit of " + EXEMPLAR_MAX_RUNES);
return null;
}
return new Exemplar(value, labels, exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
}

private static int getRunes(Labels labels) {
laurit marked this conversation as resolved.
Show resolved Hide resolved
int runes = 0;
for (Label l : labels) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

strongly suggest you rename l -> label. Otherwise it looks like the number '1'

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jkwatson i don't think it's a big deal as the loop is too small, but i suppose it does help in quicker reading. I addressed it in latest commit.

runes +=
new String(l.getName().getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8).length()
laurit marked this conversation as resolved.
Show resolved Hide resolved
+ new String(l.getValue().getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8)
.length();
}
return runes;
}

private InfoSnapshot makeTargetInfo(Resource resource) {
Expand Down
133 changes: 133 additions & 0 deletions ...heus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
import static org.assertj.core.api.Assertions.assertThatCode;

import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.api.trace.TraceFlags;
import io.opentelemetry.api.trace.TraceState;
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
import io.opentelemetry.sdk.metrics.data.MetricData;
Expand All @@ -21,6 +24,7 @@
import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableHistogramData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableHistogramPointData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableLongExemplarData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableLongPointData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableSumData;
Expand Down Expand Up @@ -422,6 +426,42 @@ static MetricData createSampleMetricData(
throw new IllegalArgumentException("Unsupported metric data type: " + metricDataType);
}

static MetricData createLongMetricDataWithExemplar(
String metricName,
String metricUnit,
@Nullable Attributes attributes,
@Nullable Resource resource,
Attributes exemplarFilteredAttributes) {
Attributes attributesToUse = attributes == null ? Attributes.empty() : attributes;
Resource resourceToUse = resource == null ? Resource.getDefault() : resource;

return ImmutableMetricData.createLongSum(
resourceToUse,
InstrumentationScopeInfo.create("scope"),
metricName,
"description",
metricUnit,
ImmutableSumData.create(
true,
AggregationTemporality.CUMULATIVE,
Collections.singletonList(
ImmutableLongPointData.create(
0,
100000,
attributesToUse,
1L,
Collections.singletonList(
ImmutableLongExemplarData.create(
exemplarFilteredAttributes,
1L,
SpanContext.create(
"0669315b30dbe08683c19ed9bd24068b",
"049178b29912fdb4",
TraceFlags.getDefault(),
TraceState.getDefault()),
2))))));
}

@Test
void validateCacheIsBounded() {
AtomicInteger predicateCalledCount = new AtomicInteger();
Expand Down Expand Up @@ -478,4 +518,97 @@ void validateCacheIsBounded() {
// it never saw those resources before.
assertThat(predicateCalledCount.get()).isEqualTo(2);
}

@Test
void exemplarLabelsWithinLimit() throws IOException {

Otel2PrometheusConverter converter = new Otel2PrometheusConverter(true, null);
Attributes exemplarfilteredAttributes =
Attributes.of(
stringKey("client_address"),
"127.0.0.6",
stringKey("network_peer_address"),
"127.0.0.6");

MetricData metricDataWithExemplar =
createLongMetricDataWithExemplar(
"metric_hertz",
"hertz",
Attributes.of(stringKey("foo1"), "bar1", stringKey("foo2"), "bar2"),
Resource.create(
Attributes.of(stringKey("host"), "localhost", stringKey("cluster"), "mycluster")),
exemplarfilteredAttributes);
String expectedExemplarLabels =
"client_address=\"127.0.0.6\""
+ ",network_peer_address=\"127.0.0.6\",span_id=\"049178b29912fdb4\""
+ ",trace_id=\"0669315b30dbe08683c19ed9bd24068b\"";
ByteArrayOutputStream out = new ByteArrayOutputStream();
MetricSnapshots snapshots =
converter.convert(Collections.singletonList(metricDataWithExemplar));
ExpositionFormats.init().getOpenMetricsTextFormatWriter().write(out, snapshots);
String expositionFormat = new String(out.toByteArray(), StandardCharsets.UTF_8);

// extract the only metric line
List<String> metricLines =
Arrays.stream(expositionFormat.split("\n"))
.filter(line -> line.startsWith("metric_hertz"))
.collect(Collectors.toList());
assertThat(metricLines).hasSize(1);

// metric_hertz_total{foo1="bar1",foo2="bar2",otel_scope_name="scope"} 1.0 #
// {client_address="127.0.0.6",network_peer_address="127.0.0.6",span_id="0002",trace_id="0001"}
// 2.0
String metricLine = metricLines.get(0);
String exemplarPart = metricLine.substring(metricLine.indexOf("#") + 2);

String exemplarLabels =
exemplarPart.substring(exemplarPart.indexOf("{") + 1, exemplarPart.indexOf("}"));
assertThat(exemplarLabels).isEqualTo(expectedExemplarLabels);
}

@Test
void exemplarLabelsAboveLimit() throws IOException {

Otel2PrometheusConverter converter = new Otel2PrometheusConverter(true, null);
Attributes exemplarfilteredAttributes =
Attributes.of(
stringKey("client_address"),
"127.0.0.6",
stringKey("network_peer_address"),
"127.0.0.6",
stringKey("network_peer_port"),
"55579",
stringKey("server_address"),
"10.3.17.168",
stringKey("server_port"),
"8081",
stringKey("url_path"),
"/foo/bar");
MetricData metricDataWithExemplar =
createLongMetricDataWithExemplar(
"metric_hertz",
"hertz",
Attributes.of(stringKey("foo1"), "bar1", stringKey("foo2"), "bar2"),
Resource.create(
Attributes.of(stringKey("host"), "localhost", stringKey("cluster"), "mycluster")),
exemplarfilteredAttributes);
ByteArrayOutputStream out = new ByteArrayOutputStream();
MetricSnapshots snapshots =
converter.convert(Collections.singletonList(metricDataWithExemplar));
ExpositionFormats.init().getOpenMetricsTextFormatWriter().write(out, snapshots);
String expositionFormat = new String(out.toByteArray(), StandardCharsets.UTF_8);

// extract the only metric line
List<String> metricLines =
Arrays.stream(expositionFormat.split("\n"))
.filter(line -> line.startsWith("metric_hertz"))
.collect(Collectors.toList());
assertThat(metricLines).hasSize(1);

// metric_hertz_total{foo1="bar1",foo2="bar2",otel_scope_name="scope"} 1.0
// no exemplar data as runes limit was reached
String metricLine = metricLines.get(0);
int exemplarDelimitterPos = metricLine.indexOf("#");
assertThat(exemplarDelimitterPos).isEqualTo(-1);
}
}
Loading