From 2f192e8e0b32987fe2aea6b110ae7fee7c6011c0 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 27 Sep 2024 22:58:57 -0400 Subject: [PATCH] Refactor ip database downloading (#113725) --- .../geoip/EnterpriseGeoIpDownloader.java | 283 ++++++++++++------ ...EnterpriseGeoIpDownloaderTaskExecutor.java | 27 +- .../geoip/EnterpriseGeoIpDownloaderTests.java | 84 ++---- 3 files changed, 231 insertions(+), 163 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java index cd2649c210500..acc51c1bb0b53 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java @@ -20,9 +20,9 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -40,9 +40,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.net.PasswordAuthentication; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.Arrays; @@ -67,32 +67,18 @@ public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask { private static final Logger logger = LogManager.getLogger(EnterpriseGeoIpDownloader.class); - private static final Pattern CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)"); + + // a sha256 checksum followed by two spaces followed by an (ignored) file name + private static final Pattern SHA256_CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)"); // for overriding in tests static String DEFAULT_MAXMIND_ENDPOINT = System.getProperty( - MAXMIND_SETTINGS_PREFIX + "endpoint.default", + MAXMIND_SETTINGS_PREFIX + "endpoint.default", // "https://download.maxmind.com/geoip/databases" ); // n.b. a future enhancement might be to allow for a MAXMIND_ENDPOINT_SETTING, but // at the moment this is an unsupported system property for use in tests (only) - static String downloadUrl(final String name, final String suffix) { - String endpointPattern = DEFAULT_MAXMIND_ENDPOINT; - if (endpointPattern.contains("%")) { - throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); - } - if (endpointPattern.endsWith("/") == false) { - endpointPattern += "/"; - } - endpointPattern += "%s/download?suffix=%s"; - - // at this point the pattern looks like this (in the default case): - // https://download.maxmind.com/geoip/databases/%s/download?suffix=%s - - return Strings.format(endpointPattern, name, suffix); - } - static final String DATABASES_INDEX = ".geoip_databases"; static final int MAX_CHUNK_SIZE = 1024 * 1024; @@ -105,7 +91,7 @@ static String downloadUrl(final String name, final String suffix) { protected volatile EnterpriseGeoIpTaskState state; private volatile Scheduler.ScheduledCancellable scheduled; private final Supplier pollIntervalSupplier; - private final Function credentialsBuilder; + private final Function tokenProvider; EnterpriseGeoIpDownloader( Client client, @@ -119,7 +105,7 @@ static String downloadUrl(final String name, final String suffix) { TaskId parentTask, Map headers, Supplier pollIntervalSupplier, - Function credentialsBuilder + Function tokenProvider ) { super(id, type, action, description, parentTask, headers); this.client = client; @@ -127,7 +113,7 @@ static String downloadUrl(final String name, final String suffix) { this.clusterService = clusterService; this.threadPool = threadPool; this.pollIntervalSupplier = pollIntervalSupplier; - this.credentialsBuilder = credentialsBuilder; + this.tokenProvider = tokenProvider; } void setState(EnterpriseGeoIpTaskState state) { @@ -156,7 +142,7 @@ void updateDatabases() throws IOException { } } - logger.trace("Updating geoip databases"); + logger.trace("Updating databases"); IngestGeoIpMetadata geoIpMeta = clusterState.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); // if there are entries in the cs that aren't in the persistent task state, @@ -174,15 +160,8 @@ void updateDatabases() throws IOException { DatabaseConfiguration database = entry.getValue().database(); if (existingDatabaseNames.contains(database.name() + ".mmdb") == false) { logger.debug("A new database appeared [{}]", database.name()); - - final String accountId = database.maxmind().accountId(); - try (HttpClient.PasswordAuthenticationHolder holder = credentialsBuilder.apply(accountId)) { - if (holder == null) { - logger.warn("No credentials found to download database [{}], skipping download...", id); - } else { - processDatabase(holder.get(), database); - addedSomething = true; - } + if (processDatabase(id, database)) { + addedSomething = true; } } } @@ -225,14 +204,8 @@ void updateDatabases() throws IOException { for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { final String id = entry.getKey(); DatabaseConfiguration database = entry.getValue().database(); - - final String accountId = database.maxmind().accountId(); - try (HttpClient.PasswordAuthenticationHolder holder = credentialsBuilder.apply(accountId)) { - if (holder == null) { - logger.warn("No credentials found to download database [{}], skipping download...", id); - } else { - processDatabase(holder.get(), database); - } + try { + processDatabase(id, database); } catch (Exception e) { accumulator = ExceptionsHelper.useOrSuppress(accumulator, ExceptionsHelper.convertToRuntime(e)); } @@ -244,68 +217,69 @@ void updateDatabases() throws IOException { } /** - * This method fetches the sha256 file and tar.gz file for the given database from the Maxmind endpoint, then indexes that tar.gz - * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database tar.gz from the index if they exist. - * If the computed sha256 does not match the expected sha256, an error will be logged and the database will not be put into the - * Elasticsearch index. + * This method fetches the checksum and database for the given database from the Maxmind endpoint, then indexes that database + * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database from the index if they exist. + * If the computed checksum does not match the expected checksum, an error will be logged and the database will not be put into + * the Elasticsearch index. *

- * As an implementation detail, this method retrieves the sha256 checksum of the database to download and then invokes - * {@link EnterpriseGeoIpDownloader#processDatabase(PasswordAuthentication, String, String, String)} with that checksum, deferring to - * that method to actually download and process the tar.gz itself. + * As an implementation detail, this method retrieves the checksum of the database to download and then invokes + * {@link EnterpriseGeoIpDownloader#processDatabase(String, Checksum, CheckedSupplier)} with that checksum, + * deferring to that method to actually download and process the database file itself. * - * @param auth The credentials to use to download from the Maxmind endpoint - * @param database The database to be downloaded from Maxmind and indexed into an Elasticsearch index - * @throws IOException If there is an error fetching the sha256 file + * @param id The identifier for this database (just for logging purposes) + * @param database The database to be downloaded and indexed into an Elasticsearch index + * @return true if the file was processed, false if the file wasn't processed (for example if credentials haven't been configured) + * @throws IOException If there is an error fetching the checksum or database file */ - void processDatabase(PasswordAuthentication auth, DatabaseConfiguration database) throws IOException { + boolean processDatabase(String id, DatabaseConfiguration database) throws IOException { final String name = database.name(); logger.debug("Processing database [{}] for configuration [{}]", name, database.id()); - final String sha256Url = downloadUrl(name, "tar.gz.sha256"); - final String tgzUrl = downloadUrl(name, "tar.gz"); - - String result = new String(httpClient.getBytes(auth, sha256Url), StandardCharsets.UTF_8).trim(); // this throws if the auth is bad - var matcher = CHECKSUM_PATTERN.matcher(result); - boolean match = matcher.matches(); - if (match == false) { - throw new RuntimeException("Unexpected sha256 response from [" + sha256Url + "]"); + try (ProviderDownload downloader = downloaderFor(database)) { + if (downloader.validCredentials()) { + // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, + // but the downloading and indexing of database code expects it to be there, so we add it on here before continuing + final String fileName = name + ".mmdb"; + processDatabase(fileName, downloader.checksum(), downloader.download()); + return true; + } else { + logger.warn("No credentials found to download database [{}], skipping download...", id); + return false; + } } - final String sha256 = matcher.group(1); - // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, - // but the downloading and indexing of database code expects it to be there, so we add it on here before further processing - processDatabase(auth, name + ".mmdb", sha256, tgzUrl); } /** - * This method fetches the tar.gz file for the given database from the Maxmind endpoint, then indexes that tar.gz - * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database tar.gz from the index if they exist. + * This method fetches the database file for the given database from the passed-in source, then indexes that database + * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database from the index if they exist. * - * @param auth The credentials to use to download from the Maxmind endpoint - * The name of the database to be downloaded from Maxmind and indexed into an Elasticsearch index - * @param sha256 The sha256 to compare to the computed sha256 of the downloaded tar.gz file - * @param url The URL for the Maxmind endpoint from which the database's tar.gz will be downloaded + * @param name The name of the database to be downloaded and indexed into an Elasticsearch index + * @param checksum The checksum to compare to the computed checksum of the downloaded file + * @param source The supplier of an InputStream that will actually download the file */ - private void processDatabase(PasswordAuthentication auth, String name, String sha256, String url) { + private void processDatabase(final String name, final Checksum checksum, final CheckedSupplier source) { Metadata metadata = state.getDatabases().getOrDefault(name, Metadata.EMPTY); - if (Objects.equals(metadata.sha256(), sha256)) { + if (checksum.matches(metadata)) { updateTimestamp(name, metadata); return; } - logger.debug("downloading geoip database [{}]", name); + logger.debug("downloading database [{}]", name); long start = System.currentTimeMillis(); - try (InputStream is = httpClient.get(auth, url)) { + try (InputStream is = source.get()) { int firstChunk = metadata.lastChunk() + 1; // if there is no metadata, then Metadata.EMPTY + 1 = 0 - Tuple tuple = indexChunks(name, is, firstChunk, MessageDigests.sha256(), sha256, start); + Tuple tuple = indexChunks(name, is, firstChunk, checksum, start); int lastChunk = tuple.v1(); - String md5 = tuple.v2(); + String md5 = tuple.v2(); // the md5 of the bytes as they passed through indexChunks if (lastChunk > firstChunk) { + // if there is a sha256 for this download, then record it (otherwise record null for it, which is also fine) + String sha256 = checksum.type == Checksum.Type.SHA256 ? checksum.checksum : null; state = state.put(name, new Metadata(start, firstChunk, lastChunk - 1, md5, start, sha256)); updateTaskState(); - logger.info("successfully downloaded geoip database [{}]", name); + logger.info("successfully downloaded database [{}]", name); deleteOldChunks(name, firstChunk); } } catch (Exception e) { - logger.error(() -> "error downloading geoip database [" + name + "]", e); + logger.error(() -> "error downloading database [" + name + "]", e); } } @@ -319,13 +293,13 @@ void deleteOldChunks(String name, int firstChunk) { client.execute( DeleteByQueryAction.INSTANCE, request, - ActionListener.wrap(r -> {}, e -> logger.warn("could not delete old chunks for geoip database [" + name + "]", e)) + ActionListener.wrap(r -> {}, e -> logger.warn("could not delete old chunks for database [" + name + "]", e)) ); } // visible for testing protected void updateTimestamp(String name, Metadata old) { - logger.debug("geoip database [{}] is up to date, updated timestamp", name); + logger.debug("database [{}] is up to date, updated timestamp", name); state = state.put( name, new Metadata(old.lastUpdate(), old.firstChunk(), old.lastChunk(), old.md5(), System.currentTimeMillis(), old.sha256()) @@ -340,15 +314,11 @@ void updateTaskState() { } // visible for testing - Tuple indexChunks( - String name, - InputStream is, - int chunk, - @Nullable MessageDigest digest, - String expectedChecksum, - long timestamp - ) throws IOException { + Tuple indexChunks(String name, InputStream is, int chunk, final Checksum checksum, long timestamp) throws IOException { + // we have to calculate and return md5 sums as a matter of course (see actualMd5 being return below), + // but we don't have to do it *twice* -- so if the passed-in checksum is also md5, then we'll get null here MessageDigest md5 = MessageDigests.md5(); + MessageDigest digest = checksum.digest(); // this returns null for md5 for (byte[] buf = getChunk(is); buf.length != 0; buf = getChunk(is)) { md5.update(buf); if (digest != null) { @@ -371,6 +341,7 @@ Tuple indexChunks( String actualMd5 = MessageDigests.toHexString(md5.digest()); String actualChecksum = digest == null ? actualMd5 : MessageDigests.toHexString(digest.digest()); + String expectedChecksum = checksum.checksum; if (Objects.equals(expectedChecksum, actualChecksum) == false) { throw new IOException("checksum mismatch, expected [" + expectedChecksum + "], actual [" + actualChecksum + "]"); } @@ -418,12 +389,12 @@ synchronized void runDownloader() { try { updateDatabases(); // n.b. this downloads bytes from the internet, it can take a while } catch (Exception e) { - logger.error("exception during geoip databases update", e); + logger.error("exception during databases update", e); } try { cleanDatabases(); } catch (Exception e) { - logger.error("exception during geoip databases cleanup", e); + logger.error("exception during databases cleanup", e); } } @@ -472,4 +443,136 @@ private void scheduleNextRun(TimeValue time) { } } + private ProviderDownload downloaderFor(DatabaseConfiguration database) { + return new MaxmindDownload(database.name(), database.maxmind()); + } + + class MaxmindDownload implements ProviderDownload { + + final String name; + final DatabaseConfiguration.Maxmind maxmind; + HttpClient.PasswordAuthenticationHolder auth; + + MaxmindDownload(String name, DatabaseConfiguration.Maxmind maxmind) { + this.name = name; + this.maxmind = maxmind; + this.auth = buildCredentials(); + } + + @Override + public HttpClient.PasswordAuthenticationHolder buildCredentials() { + // if the username is missing, empty, or blank, return null as 'no auth' + final String username = maxmind.accountId(); + if (username == null || username.isEmpty() || username.isBlank()) { + return null; + } + + // likewise if the password chars array is missing or empty, return null as 'no auth' + final char[] passwordChars = tokenProvider.apply("maxmind"); + if (passwordChars == null || passwordChars.length == 0) { + return null; + } + + return new HttpClient.PasswordAuthenticationHolder(username, passwordChars); + } + + @Override + public boolean validCredentials() { + return auth.get() != null; + } + + @Override + public String url(String suffix) { + String endpointPattern = DEFAULT_MAXMIND_ENDPOINT; + if (endpointPattern.contains("%")) { + throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); + } + if (endpointPattern.endsWith("/") == false) { + endpointPattern += "/"; + } + endpointPattern += "%s/download?suffix=%s"; + + // at this point the pattern looks like this (in the default case): + // https://download.maxmind.com/geoip/databases/%s/download?suffix=%s + + return Strings.format(endpointPattern, name, suffix); + } + + @Override + public Checksum checksum() throws IOException { + final String sha256Url = this.url("tar.gz.sha256"); + var result = new String(httpClient.getBytes(auth.get(), sha256Url), StandardCharsets.UTF_8).trim(); // throws if the auth is bad + var matcher = SHA256_CHECKSUM_PATTERN.matcher(result); + boolean match = matcher.matches(); + if (match == false) { + throw new RuntimeException("Unexpected sha256 response from [" + sha256Url + "]"); + } + final String sha256 = matcher.group(1); + return Checksum.sha256(sha256); + } + + @Override + public CheckedSupplier download() { + final String tgzUrl = this.url("tar.gz"); + return () -> httpClient.get(auth.get(), tgzUrl); + } + + @Override + public void close() throws IOException { + auth.close(); + } + } + + interface ProviderDownload extends Closeable { + // note: buildCredentials and url are inherently just implementation details of checksum() and download(), + // but it's useful to have unit tests for this logic and to keep it separate + HttpClient.PasswordAuthenticationHolder buildCredentials(); + + String url(String suffix); + + boolean validCredentials(); + + Checksum checksum() throws IOException; + + CheckedSupplier download(); + + @Override + void close() throws IOException; + } + + record Checksum(Type type, String checksum) { + + // use the static factory methods, though, rather than this + public Checksum { + Objects.requireNonNull(type); + Objects.requireNonNull(checksum); + } + + static Checksum md5(String checksum) { + return new Checksum(Type.MD5, checksum); + } + + static Checksum sha256(String checksum) { + return new Checksum(Type.SHA256, checksum); + } + + enum Type { + MD5, + SHA256 + } + + MessageDigest digest() { + return switch (type) { + case MD5 -> null; // a leaky implementation detail, we don't need to calculate two md5s + case SHA256 -> MessageDigests.sha256(); + }; + } + + boolean matches(Metadata metadata) { + return switch (type) { + case MD5 -> checksum.equals(metadata.md5()); + case SHA256 -> checksum.equals(metadata.sha256()); + }; + } + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java index 6f04ad4422c66..5214c0e4a6a51 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java @@ -100,25 +100,14 @@ private void setPollInterval(TimeValue pollInterval) { } } - private HttpClient.PasswordAuthenticationHolder buildCredentials(final String username) { - final char[] passwordChars; - if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) { - passwordChars = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars(); - } else { - passwordChars = null; - } - - // if the username is missing, empty, or blank, return null as 'no auth' - if (username == null || username.isEmpty() || username.isBlank()) { - return null; - } - - // likewise if the password chars array is missing or empty, return null as 'no auth' - if (passwordChars == null || passwordChars.length == 0) { - return null; + private char[] getSecureToken(final String type) { + char[] token = null; + if (type.equals("maxmind")) { + if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) { + token = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars(); + } } - - return new HttpClient.PasswordAuthenticationHolder(username, passwordChars); + return token; } @Override @@ -142,7 +131,7 @@ protected EnterpriseGeoIpDownloader createTask( parentTaskId, headers, () -> pollInterval, - this::buildCredentials + this::getSecureToken ); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java index c254c54ae983c..88c37409713ac 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java @@ -29,11 +29,11 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ReferenceDocs; -import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.ingest.EnterpriseGeoIpTask; +import org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloader.Checksum; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; import org.elasticsearch.node.Node; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -51,11 +51,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.net.PasswordAuthentication; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -111,7 +107,7 @@ public void setup() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { { EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams geoIpTaskParams = mock(EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams.class); @@ -206,8 +202,7 @@ public void testIndexChunksNoData() throws IOException { "test", empty, 0, - MessageDigests.sha256(), - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + Checksum.sha256("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), 0 ) ); @@ -228,7 +223,7 @@ public void testIndexChunksMd5Mismatch() { IOException exception = expectThrows( IOException.class, - () -> geoIpDownloader.indexChunks("test", new ByteArrayInputStream(new byte[0]), 0, MessageDigests.sha256(), "123123", 0) + () -> geoIpDownloader.indexChunks("test", new ByteArrayInputStream(new byte[0]), 0, Checksum.sha256("123123"), 0) ); assertEquals( "checksum mismatch, expected [123123], actual [e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]", @@ -279,8 +274,7 @@ public void testIndexChunks() throws IOException { "test", big, 15, - MessageDigests.sha256(), - "f2304545f224ff9ffcc585cb0a993723f911e03beb552cc03937dd443e931eab", + Checksum.sha256("f2304545f224ff9ffcc585cb0a993723f911e03beb552cc03937dd443e931eab"), 0 ) ); @@ -304,7 +298,7 @@ public void testProcessDatabaseNew() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { @@ -312,18 +306,11 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { } @Override - Tuple indexChunks( - String name, - InputStream is, - int chunk, - MessageDigest digest, - String expectedMd5, - long start - ) { + Tuple indexChunks(String name, InputStream is, int chunk, Checksum checksum, long start) { assertSame(bais, is); assertEquals(0, chunk); indexedChunks.set(true); - return Tuple.tuple(11, expectedMd5); + return Tuple.tuple(11, checksum.checksum()); } @Override @@ -340,10 +327,9 @@ void deleteOldChunks(String name, int firstChunk) { }; geoIpDownloader.setState(EnterpriseGeoIpTaskState.EMPTY); - PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); String id = randomIdentifier(); DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); - geoIpDownloader.processDatabase(auth, databaseConfiguration); + geoIpDownloader.processDatabase(id, databaseConfiguration); assertThat(indexedChunks.get(), equalTo(true)); } @@ -363,7 +349,7 @@ public void testProcessDatabaseUpdate() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { @@ -371,18 +357,11 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { } @Override - Tuple indexChunks( - String name, - InputStream is, - int chunk, - MessageDigest digest, - String expectedMd5, - long start - ) { + Tuple indexChunks(String name, InputStream is, int chunk, Checksum checksum, long start) { assertSame(bais, is); assertEquals(9, chunk); indexedChunks.set(true); - return Tuple.tuple(1, expectedMd5); + return Tuple.tuple(1, checksum.checksum()); } @Override @@ -399,10 +378,9 @@ void deleteOldChunks(String name, int firstChunk) { }; geoIpDownloader.setState(EnterpriseGeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0", 0))); - PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); String id = randomIdentifier(); DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); - geoIpDownloader.processDatabase(auth, databaseConfiguration); + geoIpDownloader.processDatabase(id, databaseConfiguration); assertThat(indexedChunks.get(), equalTo(true)); } @@ -431,7 +409,7 @@ public void testProcessDatabaseSame() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) { @@ -440,16 +418,9 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) } @Override - Tuple indexChunks( - String name, - InputStream is, - int chunk, - MessageDigest digest, - String expectedChecksum, - long start - ) { + Tuple indexChunks(String name, InputStream is, int chunk, Checksum checksum, long start) { fail(); - return Tuple.tuple(0, expectedChecksum); + return Tuple.tuple(0, checksum.checksum()); } @Override @@ -463,10 +434,9 @@ void deleteOldChunks(String name, int firstChunk) { } }; geoIpDownloader.setState(taskState); - PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); String id = randomIdentifier(); DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); - geoIpDownloader.processDatabase(auth, databaseConfiguration); + geoIpDownloader.processDatabase(id, databaseConfiguration); } public void testUpdateDatabasesWriteBlock() { @@ -502,14 +472,20 @@ public void testUpdateDatabasesIndexNotReady() throws IOException { verifyNoInteractions(httpClient); } - private GeoIpTaskState.Metadata newGeoIpTaskStateMetadata(boolean expired) { - Instant lastChecked; - if (expired) { - lastChecked = Instant.now().minus(randomIntBetween(31, 100), ChronoUnit.DAYS); - } else { - lastChecked = Instant.now().minus(randomIntBetween(0, 29), ChronoUnit.DAYS); + public void testMaxmindUrls() { + // non-static classes have fun syntax, but it's nice to be able to test this behavior by itself + final EnterpriseGeoIpDownloader.MaxmindDownload download = geoIpDownloader.new MaxmindDownload( + "GeoLite2-City", new DatabaseConfiguration.Maxmind("account_id") + ); + + { + String url = "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"; + assertThat(download.url("tar.gz"), equalTo(url)); + } + { + String url = "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz.sha256"; + assertThat(download.url("tar.gz.sha256"), equalTo(url)); } - return new GeoIpTaskState.Metadata(0, 0, 0, randomAlphaOfLength(20), lastChecked.toEpochMilli()); } private static class MockClient extends NoOpClient {