Skip to content

Commit

Permalink
Add rolling upgrade BWC tests for N-2 index compatibility (#119468)
Browse files Browse the repository at this point in the history
This change adds rolling upgrade BWC tests for searchable 
snapshots in N-2 version, so that they are tested in a mixed 
versions cluster.

The tests do not use replicas because it may or may not be 
assigned depending on which node the primary is assigned 
on (ie, if the cluster has 1 new node and 2 old node and the 
primary shard is assigned to the new node, then a replica 
cannot be assigned until at least another node is upgraded). 

It also renames the existing full cluster restart BWC tests and 
mutualizes some common code in AbstractIndexCompatibilityTestCase.

Relates ES-10432
  • Loading branch information
tlrx authored Jan 3, 2025
1 parent d2d0636 commit c62ee88
Show file tree
Hide file tree
Showing 6 changed files with 380 additions and 72 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@

package org.elasticsearch.lucene;

import com.carrotsearch.randomizedtesting.TestMethodAndParams;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;

import org.elasticsearch.client.Request;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Strings;
Expand All @@ -22,16 +17,18 @@
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.cluster.util.Version;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestRule;

import java.util.Comparator;
import java.io.IOException;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static org.elasticsearch.test.cluster.util.Version.CURRENT;
import static org.elasticsearch.test.cluster.util.Version.fromString;
Expand All @@ -41,24 +38,21 @@
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;

/**
* Test suite for Lucene indices backward compatibility with N-2 versions. The test suite creates a cluster in N-2 version, then upgrades it
* to N-1 version and finally upgrades it to the current version. Test methods are executed after each upgrade.
*/
@TestCaseOrdering(AbstractLuceneIndexCompatibilityTestCase.TestCaseOrdering.class)
public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTestCase {
public abstract class AbstractIndexCompatibilityTestCase extends ESRestTestCase {

protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
protected static final Version VERSION_CURRENT = CURRENT;

protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
protected static final int NODES = 3;

private static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();

protected static LocalClusterConfigProvider clusterConfig = c -> {};
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.distribution(DistributionType.DEFAULT)
.version(VERSION_MINUS_2)
.nodes(2)
.nodes(NODES)
.setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
.setting("xpack.security.enabled", "false")
.setting("xpack.ml.enabled", "false")
Expand All @@ -71,15 +65,44 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes

private static boolean upgradeFailed = false;

private final Version clusterVersion;
@Before
public final void maybeUpgradeBeforeTest() throws Exception {
// We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
// in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));

if (upgradeFailed == false) {
try {
maybeUpgrade();
} catch (Exception e) {
upgradeFailed = true;
throw e;
}
}

public AbstractLuceneIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
this.clusterVersion = clusterVersion;
// Skip remaining tests if upgrade failed
assumeFalse("Cluster upgrade failed", upgradeFailed);
}

@ParametersFactory
public static Iterable<Object[]> parameters() {
return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
protected abstract void maybeUpgrade() throws Exception;

@After
public final void deleteSnapshotBlobCache() throws IOException {
// TODO ES-10475: The .snapshot-blob-cache created in legacy version can block upgrades, we should probably delete it automatically
try {
var request = new Request("DELETE", "/.snapshot-blob-cache");
request.setOptions(
expectWarnings(
"this request accesses system indices: [.snapshot-blob-cache], but in a future major version, "
+ "direct access to system indices will be prevented by default"
)
);
adminClient().performRequest(request);
} catch (IOException e) {
if (isNotFoundResponseException(e) == false) {
throw e;
}
}
}

@Override
Expand All @@ -92,26 +115,8 @@ protected boolean preserveClusterUponCompletion() {
return true;
}

@Before
public void maybeUpgrade() throws Exception {
// We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
// in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));

var currentVersion = clusterVersion();
if (currentVersion.before(clusterVersion)) {
try {
cluster.upgradeToVersion(clusterVersion);
closeClients();
initClient();
} catch (Exception e) {
upgradeFailed = true;
throw e;
}
}

// Skip remaining tests if upgrade failed
assumeFalse("Cluster upgrade failed", upgradeFailed);
protected ElasticsearchCluster cluster() {
return cluster;
}

protected String suffix(String name) {
Expand All @@ -124,12 +129,18 @@ protected Settings repositorySettings() {
.build();
}

protected static Version clusterVersion() throws Exception {
var response = assertOK(client().performRequest(new Request("GET", "/")));
var responseBody = createFromResponse(response);
var version = Version.fromString(responseBody.evaluate("version.number").toString());
assertThat("Failed to retrieve cluster version", version, notNullValue());
return version;
protected static Map<String, Version> nodesVersions() throws Exception {
var nodesInfos = getNodesInfo(adminClient());
assertThat(nodesInfos.size(), equalTo(NODES));
var versions = new HashMap<String, Version>();
for (var nodeInfos : nodesInfos.values()) {
versions.put((String) nodeInfos.get("name"), Version.fromString((String) nodeInfos.get("version")));
}
return versions;
}

protected static boolean isFullyUpgradedTo(Version version) throws Exception {
return nodesVersions().values().stream().allMatch(v -> v.equals(version));
}

protected static Version indexVersion(String indexName) throws Exception {
Expand Down Expand Up @@ -181,16 +192,4 @@ protected static void restoreIndex(String repository, String snapshot, String in
assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed")));
assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0));
}

/**
* Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
*/
public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
@Override
public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
var version1 = (Version) o1.getInstanceArguments().get(0);
var version2 = (Version) o2.getInstanceArguments().get(0);
return version1.compareTo(version2);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.lucene;

import com.carrotsearch.randomizedtesting.TestMethodAndParams;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;

import org.elasticsearch.test.cluster.util.Version;

import java.util.Comparator;
import java.util.stream.Stream;

import static org.elasticsearch.test.cluster.util.Version.CURRENT;
import static org.hamcrest.Matchers.equalTo;

/**
* Test suite for Lucene indices backward compatibility with N-2 versions after full cluster restart upgrades. The test suite creates a
* cluster in N-2 version, then upgrades it to N-1 version and finally upgrades it to the current version. Test methods are executed after
* each upgrade.
*/
@TestCaseOrdering(FullClusterRestartIndexCompatibilityTestCase.TestCaseOrdering.class)
public abstract class FullClusterRestartIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase {

private final Version clusterVersion;

public FullClusterRestartIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
this.clusterVersion = clusterVersion;
}

@ParametersFactory
public static Iterable<Object[]> parameters() {
return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
}

@Override
protected void maybeUpgrade() throws Exception {
if (nodesVersions().values().stream().anyMatch(version -> version.before(clusterVersion))) {
cluster().upgradeToVersion(clusterVersion);
closeClients();
initClient();
}
assertThat(isFullyUpgradedTo(clusterVersion), equalTo(true));
}

/**
* Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
*/
public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
@Override
public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
var version1 = (Version) o1.getInstanceArguments().get(0);
var version2 = (Version) o2.getInstanceArguments().get(0);
return version1.compareTo(version2);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.equalTo;

public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {

static {
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
}

public LuceneCompatibilityIT(Version version) {
public FullClusterRestartLuceneIndexCompatibilityIT(Version version) {
super(version);
}

Expand All @@ -42,7 +42,7 @@ public void testRestoreIndex() throws Exception {
final String index = suffix("index");
final int numDocs = 1234;

if (VERSION_MINUS_2.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
logger.debug("--> registering repository [{}]", repository);
registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());

Expand All @@ -65,7 +65,7 @@ public void testRestoreIndex() throws Exception {
return;
}

if (VERSION_MINUS_1.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
ensureGreen(index);

assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
Expand All @@ -76,7 +76,7 @@ public void testRestoreIndex() throws Exception {
return;
}

if (VERSION_CURRENT.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_CURRENT)) {
var restoredIndex = suffix("index-restored");
logger.debug("--> restoring index [{}] as [{}]", index, restoredIndex);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@

import static org.hamcrest.Matchers.equalTo;

public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {

static {
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial")
.setting("xpack.searchable.snapshot.shared_cache.size", "16MB")
.setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB");
}

public SearchableSnapshotCompatibilityIT(Version version) {
public FullClusterRestartSearchableSnapshotIndexCompatibilityIT(Version version) {
super(version);
}

Expand All @@ -38,7 +38,7 @@ public void testSearchableSnapshot() throws Exception {
final String index = suffix("index");
final int numDocs = 1234;

if (VERSION_MINUS_2.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
logger.debug("--> registering repository [{}]", repository);
registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());

Expand All @@ -61,7 +61,7 @@ public void testSearchableSnapshot() throws Exception {
return;
}

if (VERSION_MINUS_1.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
ensureGreen(index);

assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
Expand All @@ -72,7 +72,7 @@ public void testSearchableSnapshot() throws Exception {
return;
}

if (VERSION_CURRENT.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_CURRENT)) {
var mountedIndex = suffix("index-mounted");
logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
Expand All @@ -98,7 +98,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
final String index = suffix("index");
final int numDocs = 4321;

if (VERSION_MINUS_2.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
logger.debug("--> registering repository [{}]", repository);
registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());

Expand All @@ -124,7 +124,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
return;
}

if (VERSION_MINUS_1.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);

Expand All @@ -135,7 +135,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
return;
}

if (VERSION_CURRENT.equals(clusterVersion())) {
if (isFullyUpgradedTo(VERSION_CURRENT)) {
ensureGreen(mountedIndex);

assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
Expand Down
Loading

0 comments on commit c62ee88

Please sign in to comment.