diff --git a/framework/db/pom.xml b/framework/db/pom.xml
index ced0f64c663d..1e7ac57de1e3 100644
--- a/framework/db/pom.xml
+++ b/framework/db/pom.xml
@@ -75,7 +75,17 @@
com/cloud/utils/testcase/*TestCase*
- com/cloud/utils/db/*Test*
+
+ com/cloud/utils/db/DbTest*
+ com/cloud/utils/db/ElementCollectionTest*
+ com/cloud/utils/db/FilterTest*
+ com/cloud/utils/db/GenericDaoBaseTest*
+ com/cloud/utils/db/GlobalLockTest*
+ com/cloud/utils/db/GroupByTest*
+ com/cloud/utils/db/Merovingian2Test*
+ com/cloud/utils/db/TestTransaction*
+ com/cloud/utils/db/TransactionContextBuilderTest*
diff --git a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
index 42a8b27fa854..3d54da78f491 100644
--- a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
+++ b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
@@ -31,6 +31,7 @@
import java.util.Properties;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
import javax.sql.DataSource;
@@ -87,6 +88,17 @@ public class TransactionLegacy implements Closeable {
public static final short CONNECTED_DB = -1;
public static final String CONNECTION_PARAMS = "scrollTolerantForwardOnly=true";
+ /**
+ * Format of the pattern that matches a parameter of a connection URI: the name must be at the start of the URI
+ * parameters or right after a parameter separator, and must be followed by "=". Searching for the bare name would
+ * consider the parameter as configured when it is just part of a host, of a database name or of the value of
+ * another parameter.
+ */
+ private static final String URI_PARAM_PATTERN_FORMAT = "(?:^|[?&])%s=";
+ private static final String CONNECTION_COLLATION_PARAM = "connectionCollation";
+ private static final String CHARACTER_ENCODING_PARAM = "characterEncoding";
+ private static final String DEFAULT_CONNECTION_COLLATION = "utf8mb4_general_ci";
+
private static AtomicLong s_id = new AtomicLong();
private static final TransactionMBeanImpl s_mbean = new TransactionMBeanImpl();
static {
@@ -1196,6 +1208,9 @@ protected static Pair getConnectionUriAndDriver(Properties dbPro
connectionUri = propertyUri;
}
+
+ connectionUri = addDefaultConnectionCollation(connectionUri);
+
LOGGER.info("Using the following URI to connect to {} database [{}].", schema, connectionUri);
return new Pair<>(connectionUri, driver);
}
@@ -1260,6 +1275,48 @@ protected static String buildConnectionUri(String loadBalanceStrategy, String dr
return connectionUri.toString();
}
+ /**
+ * Informs whether {@link #DEFAULT_CONNECTION_COLLATION} should be added to a connection URI. It is only added for
+ * connections that do not already define the charset or the collation, either through {@code db..url.params}
+ * or directly in {@code db..uri}.
+ *
+ * @param connectionUri the connection URI configured by the operator.
+ */
+ protected static boolean shouldPinConnectionCollation(String connectionUri) {
+ return !containsUriParam(connectionUri, CONNECTION_COLLATION_PARAM)
+ && !containsUriParam(connectionUri, CHARACTER_ENCODING_PARAM);
+ }
+
+ /**
+ * Informs whether the given parameter is defined in the connection URI.
+ *
+ * @param connectionUri the connection URI configured by the operator; it also accepts only the parameters of a URI,
+ * as in {@code db..url.params};
+ * @param param the name of the parameter to look for.
+ */
+ protected static boolean containsUriParam(String connectionUri, String param) {
+ Pattern pattern = Pattern.compile(String.format(URI_PARAM_PATTERN_FORMAT, Pattern.quote(param)), Pattern.CASE_INSENSITIVE);
+ return pattern.matcher(StringUtils.defaultString(connectionUri)).find();
+ }
+
+ /**
+ * Adds {@link #DEFAULT_CONNECTION_COLLATION} to a connection URI configured either through
+ * {@code db..url.params} or directly in {@code db..uri}, keeping the URI untouched if the operator
+ * already defined the charset or the collation in it.
+ */
+ protected static String addDefaultConnectionCollation(String connectionUri) {
+ if (!shouldPinConnectionCollation(connectionUri)) {
+ return connectionUri;
+ }
+
+ String separator = "?";
+ if (StringUtils.contains(connectionUri, "?")) {
+ separator = StringUtils.endsWithAny(connectionUri, "?", "&") ? StringUtils.EMPTY : "&";
+ }
+
+ return String.format("%s%s%s=%s", connectionUri, separator, CONNECTION_COLLATION_PARAM, DEFAULT_CONNECTION_COLLATION);
+ }
+
/**
* Creates a data source
*/
diff --git a/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java
index 2e0af6fa1866..813df9f8f2cb 100644
--- a/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java
+++ b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java
@@ -47,7 +47,8 @@ public void getConnectionUriAndDriverTestWithoutUri() {
Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
- Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams", result.first());
+ Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&scrollTolerantForwardOnly=true"
+ + "&connectionCollation=utf8mb4_general_ci", result.first());
Assert.assertEquals("driver", result.second());
}
@@ -57,7 +58,7 @@ public void getConnectionUriAndDriverTestWithUri() {
Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
- Assert.assertEquals("jdbc:driver:myFavoriteUri", result.first());
+ Assert.assertEquals("jdbc:driver:myFavoriteUri?connectionCollation=utf8mb4_general_ci", result.first());
Assert.assertEquals("jdbc:driver", result.second());
}
@@ -65,7 +66,7 @@ public void getConnectionUriAndDriverTestWithUri() {
public void getPropertiesAndBuildConnectionUriTestDbHaDisabled() {
String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud");
- Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true", result);
+ Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true&scrollTolerantForwardOnly=true", result);
}
@Test
@@ -82,14 +83,14 @@ public void getPropertiesAndBuildConnectionUriTestDbHaEnabled() {
String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud");
Assert.assertEquals("driver://host,second_host:5555/name?autoReconnect=false&someParams&useSSL=true&failOverReadOnly=true&reconnectAtTxEnd=false&autoReconnectFor"
- + "Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat", result);
+ + "Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat&scrollTolerantForwardOnly=true", result);
}
@Test
public void buildConnectionUriTestDbHaDisabled() {
String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, null, null);
- Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false", result);
+ Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&scrollTolerantForwardOnly=true", result);
}
@Test
@@ -98,20 +99,257 @@ public void buildConnectionUriTestDbHaEnabled() {
String result = TransactionLegacy.buildConnectionUri("strat", "driver", false, "host", "second_host", 5555, "cloud", false, null, "dbHaParams");
- Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat", result);
+ Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat&scrollTolerantForwardOnly=true", result);
}
@Test
public void buildConnectionUriTestUrlParamsNotNull() {
String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, "urlParams", null);
- Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams", result);
+ Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams&scrollTolerantForwardOnly=true", result);
}
@Test
public void buildConnectionUriTestUseSslTrue() {
String result = TransactionLegacy.buildConnectionUri(null, "driver", true, "host", null, 5555, "cloud", false, null, null);
- Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true", result);
+ Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true&scrollTolerantForwardOnly=true", result);
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithoutUriAndUrlParamsDefiningConnectionCollationDoesNotPinTheDefaultOne() {
+ properties.setProperty("db.cloud.uri", "");
+ properties.setProperty("db.cloud.driver", "driver");
+ properties.setProperty("db.cloud.url.params", "connectionCollation=utf8mb4_unicode_ci");
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("driver://host:5555/name?autoReconnect=false&connectionCollation=utf8mb4_unicode_ci"
+ + "&scrollTolerantForwardOnly=true", result.first());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithoutUriAndUrlParamsDefiningCharacterEncodingDoesNotPinTheDefaultCollation() {
+ properties.setProperty("db.cloud.uri", "");
+ properties.setProperty("db.cloud.driver", "driver");
+ properties.setProperty("db.cloud.url.params", "characterEncoding=UTF-8");
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("driver://host:5555/name?autoReconnect=false&characterEncoding=UTF-8&scrollTolerantForwardOnly=true",
+ result.first());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithUriWithoutParametersPinsTheDefaultCollation() {
+ properties.setProperty("db.cloud.uri", "jdbc:mysql://host:5555/name");
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_general_ci", result.first());
+ Assert.assertEquals("jdbc:mysql", result.second());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithUriWithParametersPinsTheDefaultCollation() {
+ properties.setProperty("db.cloud.uri", "jdbc:mysql://host:5555/name?autoReconnect=false&someParams");
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/name?autoReconnect=false&someParams&connectionCollation=utf8mb4_general_ci",
+ result.first());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithUriDefiningConnectionCollationKeepsItUntouched() {
+ String uri = "jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_unicode_ci";
+ properties.setProperty("db.cloud.uri", uri);
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals(uri, result.first());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithUriDefiningCharacterEncodingKeepsItUntouched() {
+ String uri = "jdbc:mysql://host:5555/name?characterEncoding=UTF-8";
+ properties.setProperty("db.cloud.uri", uri);
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals(uri, result.first());
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestNullConnectionParamsReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation(null));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestEmptyConnectionParamsReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation(""));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestConnectionParamsWithoutCollationAndEncodingReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("cachePrepStmts=true&serverTimezone=UTC"));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestConnectionParamsWithConnectionCollationReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("cachePrepStmts=true&connectionCollation=utf8mb4_unicode_ci"));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestConnectionParamsWithCharacterEncodingReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("cachePrepStmts=true&characterEncoding=UTF-8"));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestIsCaseInsensitive() {
+ Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("CONNECTIONCOLLATION=utf8mb4_unicode_ci"));
+ Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("characterencoding=UTF-8"));
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriWithoutParametersAddsTheQueryStringSeparator() {
+ String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_general_ci", result);
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriWithParametersAddsTheParameterSeparator() {
+ String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name?someParams");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/name?someParams&connectionCollation=utf8mb4_general_ci", result);
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriEndingWithQueryStringSeparatorDoesNotDuplicateIt() {
+ String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name?");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_general_ci", result);
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriEndingWithParameterSeparatorDoesNotDuplicateIt() {
+ String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name?someParams&");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/name?someParams&connectionCollation=utf8mb4_general_ci", result);
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriDefiningConnectionCollationKeepsItUntouched() {
+ String uri = "jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_unicode_ci";
+
+ Assert.assertEquals(uri, TransactionLegacy.addDefaultConnectionCollation(uri));
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriDefiningCharacterEncodingKeepsItUntouched() {
+ String uri = "jdbc:mysql://host:5555/name?characterEncoding=UTF-8";
+
+ Assert.assertEquals(uri, TransactionLegacy.addDefaultConnectionCollation(uri));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestUriWithHostContainingTheParameterNamesReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://characterEncoding:5555/name"));
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://connectionCollation.example.com:5555/name?someParams"));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestUriWithDatabaseNameContainingTheParameterNamesReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/connectionCollation"));
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/characterEncoding?someParams"));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestParameterValueContainingTheParameterNamesReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?user=connectionCollation"));
+ Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?user=characterEncoding&someParams"));
+ }
+
+ @Test
+ public void shouldPinConnectionCollationTestUriDefiningTheParametersAsTheFirstOneReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_unicode_ci"));
+ Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?characterEncoding=UTF-8&someParams"));
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriWithHostContainingTheParameterNamesAddsTheDefaultCollation() {
+ String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://characterEncoding.example.com:5555/name");
+
+ Assert.assertEquals("jdbc:mysql://characterEncoding.example.com:5555/name?connectionCollation=utf8mb4_general_ci", result);
+ }
+
+ @Test
+ public void addDefaultConnectionCollationTestUriWithDatabaseNameContainingTheParameterNamesAddsTheDefaultCollation() {
+ String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/connectionCollation?someParams");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/connectionCollation?someParams&connectionCollation=utf8mb4_general_ci", result);
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithUriWhoseDatabaseNameContainsTheParameterNamePinsTheDefaultCollation() {
+ properties.setProperty("db.cloud.uri", "jdbc:mysql://host:5555/characterEncoding");
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("jdbc:mysql://host:5555/characterEncoding?connectionCollation=utf8mb4_general_ci", result.first());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithoutUriAndDatabaseNameContainingTheParameterNamePinsTheDefaultCollation() {
+ properties.setProperty("db.cloud.uri", "");
+ properties.setProperty("db.cloud.driver", "driver");
+ properties.setProperty("db.cloud.name", "connectionCollation");
+
+ Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("driver://host:5555/connectionCollation?autoReconnect=false&someParams&scrollTolerantForwardOnly=true"
+ + "&connectionCollation=utf8mb4_general_ci", result.first());
+ }
+
+ @Test
+ public void containsUriParamTestParamDefinedAsTheFirstOneReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?serverTimezone=UTC", "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestParamDefinedAfterOtherParamsReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?someParams&serverTimezone=UTC", "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestOnlyTheUriParamsReturnsTrue() {
+ Assert.assertTrue(TransactionLegacy.containsUriParam("serverTimezone=UTC&someParams", "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestParamIsPartOfTheHostOrOfTheDatabaseNameReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://serverTimezone:5555/name", "serverTimezone"));
+ Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/serverTimezone?someParams", "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestParamIsTheValueOfAnotherParamReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?user=serverTimezone", "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestParamIsNotDefinedReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?someParams", "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestNullConnectionUriReturnsFalse() {
+ Assert.assertFalse(TransactionLegacy.containsUriParam(null, "serverTimezone"));
+ }
+
+ @Test
+ public void containsUriParamTestIsCaseInsensitive() {
+ Assert.assertTrue(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?SERVERTIMEZONE=UTC", "serverTimezone"));
}
}