diff --git a/framework/db/pom.xml b/framework/db/pom.xml index ced0f64c663d..1e7ac57de1e3 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -75,7 +75,17 @@ com/cloud/utils/testcase/*TestCase* - com/cloud/utils/db/*Test* + + com/cloud/utils/db/DbTest* + com/cloud/utils/db/ElementCollectionTest* + com/cloud/utils/db/FilterTest* + com/cloud/utils/db/GenericDaoBaseTest* + com/cloud/utils/db/GlobalLockTest* + com/cloud/utils/db/GroupByTest* + com/cloud/utils/db/Merovingian2Test* + com/cloud/utils/db/TestTransaction* + com/cloud/utils/db/TransactionContextBuilderTest* diff --git a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java index 42a8b27fa854..3d54da78f491 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java +++ b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java @@ -31,6 +31,7 @@ import java.util.Properties; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; import javax.sql.DataSource; @@ -87,6 +88,17 @@ public class TransactionLegacy implements Closeable { public static final short CONNECTED_DB = -1; public static final String CONNECTION_PARAMS = "scrollTolerantForwardOnly=true"; + /** + * Format of the pattern that matches a parameter of a connection URI: the name must be at the start of the URI + * parameters or right after a parameter separator, and must be followed by "=". Searching for the bare name would + * consider the parameter as configured when it is just part of a host, of a database name or of the value of + * another parameter. + */ + private static final String URI_PARAM_PATTERN_FORMAT = "(?:^|[?&])%s="; + private static final String CONNECTION_COLLATION_PARAM = "connectionCollation"; + private static final String CHARACTER_ENCODING_PARAM = "characterEncoding"; + private static final String DEFAULT_CONNECTION_COLLATION = "utf8mb4_general_ci"; + private static AtomicLong s_id = new AtomicLong(); private static final TransactionMBeanImpl s_mbean = new TransactionMBeanImpl(); static { @@ -1196,6 +1208,9 @@ protected static Pair getConnectionUriAndDriver(Properties dbPro connectionUri = propertyUri; } + + connectionUri = addDefaultConnectionCollation(connectionUri); + LOGGER.info("Using the following URI to connect to {} database [{}].", schema, connectionUri); return new Pair<>(connectionUri, driver); } @@ -1260,6 +1275,48 @@ protected static String buildConnectionUri(String loadBalanceStrategy, String dr return connectionUri.toString(); } + /** + * Informs whether {@link #DEFAULT_CONNECTION_COLLATION} should be added to a connection URI. It is only added for + * connections that do not already define the charset or the collation, either through {@code db..url.params} + * or directly in {@code db..uri}. + * + * @param connectionUri the connection URI configured by the operator. + */ + protected static boolean shouldPinConnectionCollation(String connectionUri) { + return !containsUriParam(connectionUri, CONNECTION_COLLATION_PARAM) + && !containsUriParam(connectionUri, CHARACTER_ENCODING_PARAM); + } + + /** + * Informs whether the given parameter is defined in the connection URI. + * + * @param connectionUri the connection URI configured by the operator; it also accepts only the parameters of a URI, + * as in {@code db..url.params}; + * @param param the name of the parameter to look for. + */ + protected static boolean containsUriParam(String connectionUri, String param) { + Pattern pattern = Pattern.compile(String.format(URI_PARAM_PATTERN_FORMAT, Pattern.quote(param)), Pattern.CASE_INSENSITIVE); + return pattern.matcher(StringUtils.defaultString(connectionUri)).find(); + } + + /** + * Adds {@link #DEFAULT_CONNECTION_COLLATION} to a connection URI configured either through + * {@code db..url.params} or directly in {@code db..uri}, keeping the URI untouched if the operator + * already defined the charset or the collation in it. + */ + protected static String addDefaultConnectionCollation(String connectionUri) { + if (!shouldPinConnectionCollation(connectionUri)) { + return connectionUri; + } + + String separator = "?"; + if (StringUtils.contains(connectionUri, "?")) { + separator = StringUtils.endsWithAny(connectionUri, "?", "&") ? StringUtils.EMPTY : "&"; + } + + return String.format("%s%s%s=%s", connectionUri, separator, CONNECTION_COLLATION_PARAM, DEFAULT_CONNECTION_COLLATION); + } + /** * Creates a data source */ diff --git a/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java index 2e0af6fa1866..813df9f8f2cb 100644 --- a/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java +++ b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java @@ -47,7 +47,8 @@ public void getConnectionUriAndDriverTestWithoutUri() { Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); - Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams", result.first()); + Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&scrollTolerantForwardOnly=true" + + "&connectionCollation=utf8mb4_general_ci", result.first()); Assert.assertEquals("driver", result.second()); } @@ -57,7 +58,7 @@ public void getConnectionUriAndDriverTestWithUri() { Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); - Assert.assertEquals("jdbc:driver:myFavoriteUri", result.first()); + Assert.assertEquals("jdbc:driver:myFavoriteUri?connectionCollation=utf8mb4_general_ci", result.first()); Assert.assertEquals("jdbc:driver", result.second()); } @@ -65,7 +66,7 @@ public void getConnectionUriAndDriverTestWithUri() { public void getPropertiesAndBuildConnectionUriTestDbHaDisabled() { String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud"); - Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true", result); + Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true&scrollTolerantForwardOnly=true", result); } @Test @@ -82,14 +83,14 @@ public void getPropertiesAndBuildConnectionUriTestDbHaEnabled() { String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud"); Assert.assertEquals("driver://host,second_host:5555/name?autoReconnect=false&someParams&useSSL=true&failOverReadOnly=true&reconnectAtTxEnd=false&autoReconnectFor" - + "Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat", result); + + "Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat&scrollTolerantForwardOnly=true", result); } @Test public void buildConnectionUriTestDbHaDisabled() { String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, null, null); - Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false", result); + Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&scrollTolerantForwardOnly=true", result); } @Test @@ -98,20 +99,257 @@ public void buildConnectionUriTestDbHaEnabled() { String result = TransactionLegacy.buildConnectionUri("strat", "driver", false, "host", "second_host", 5555, "cloud", false, null, "dbHaParams"); - Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat", result); + Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat&scrollTolerantForwardOnly=true", result); } @Test public void buildConnectionUriTestUrlParamsNotNull() { String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, "urlParams", null); - Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams", result); + Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams&scrollTolerantForwardOnly=true", result); } @Test public void buildConnectionUriTestUseSslTrue() { String result = TransactionLegacy.buildConnectionUri(null, "driver", true, "host", null, 5555, "cloud", false, null, null); - Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true", result); + Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true&scrollTolerantForwardOnly=true", result); + } + + @Test + public void getConnectionUriAndDriverTestWithoutUriAndUrlParamsDefiningConnectionCollationDoesNotPinTheDefaultOne() { + properties.setProperty("db.cloud.uri", ""); + properties.setProperty("db.cloud.driver", "driver"); + properties.setProperty("db.cloud.url.params", "connectionCollation=utf8mb4_unicode_ci"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("driver://host:5555/name?autoReconnect=false&connectionCollation=utf8mb4_unicode_ci" + + "&scrollTolerantForwardOnly=true", result.first()); + } + + @Test + public void getConnectionUriAndDriverTestWithoutUriAndUrlParamsDefiningCharacterEncodingDoesNotPinTheDefaultCollation() { + properties.setProperty("db.cloud.uri", ""); + properties.setProperty("db.cloud.driver", "driver"); + properties.setProperty("db.cloud.url.params", "characterEncoding=UTF-8"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("driver://host:5555/name?autoReconnect=false&characterEncoding=UTF-8&scrollTolerantForwardOnly=true", + result.first()); + } + + @Test + public void getConnectionUriAndDriverTestWithUriWithoutParametersPinsTheDefaultCollation() { + properties.setProperty("db.cloud.uri", "jdbc:mysql://host:5555/name"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_general_ci", result.first()); + Assert.assertEquals("jdbc:mysql", result.second()); + } + + @Test + public void getConnectionUriAndDriverTestWithUriWithParametersPinsTheDefaultCollation() { + properties.setProperty("db.cloud.uri", "jdbc:mysql://host:5555/name?autoReconnect=false&someParams"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("jdbc:mysql://host:5555/name?autoReconnect=false&someParams&connectionCollation=utf8mb4_general_ci", + result.first()); + } + + @Test + public void getConnectionUriAndDriverTestWithUriDefiningConnectionCollationKeepsItUntouched() { + String uri = "jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_unicode_ci"; + properties.setProperty("db.cloud.uri", uri); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals(uri, result.first()); + } + + @Test + public void getConnectionUriAndDriverTestWithUriDefiningCharacterEncodingKeepsItUntouched() { + String uri = "jdbc:mysql://host:5555/name?characterEncoding=UTF-8"; + properties.setProperty("db.cloud.uri", uri); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals(uri, result.first()); + } + + @Test + public void shouldPinConnectionCollationTestNullConnectionParamsReturnsTrue() { + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation(null)); + } + + @Test + public void shouldPinConnectionCollationTestEmptyConnectionParamsReturnsTrue() { + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("")); + } + + @Test + public void shouldPinConnectionCollationTestConnectionParamsWithoutCollationAndEncodingReturnsTrue() { + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("cachePrepStmts=true&serverTimezone=UTC")); + } + + @Test + public void shouldPinConnectionCollationTestConnectionParamsWithConnectionCollationReturnsFalse() { + Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("cachePrepStmts=true&connectionCollation=utf8mb4_unicode_ci")); + } + + @Test + public void shouldPinConnectionCollationTestConnectionParamsWithCharacterEncodingReturnsFalse() { + Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("cachePrepStmts=true&characterEncoding=UTF-8")); + } + + @Test + public void shouldPinConnectionCollationTestIsCaseInsensitive() { + Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("CONNECTIONCOLLATION=utf8mb4_unicode_ci")); + Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("characterencoding=UTF-8")); + } + + @Test + public void addDefaultConnectionCollationTestUriWithoutParametersAddsTheQueryStringSeparator() { + String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name"); + + Assert.assertEquals("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_general_ci", result); + } + + @Test + public void addDefaultConnectionCollationTestUriWithParametersAddsTheParameterSeparator() { + String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name?someParams"); + + Assert.assertEquals("jdbc:mysql://host:5555/name?someParams&connectionCollation=utf8mb4_general_ci", result); + } + + @Test + public void addDefaultConnectionCollationTestUriEndingWithQueryStringSeparatorDoesNotDuplicateIt() { + String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name?"); + + Assert.assertEquals("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_general_ci", result); + } + + @Test + public void addDefaultConnectionCollationTestUriEndingWithParameterSeparatorDoesNotDuplicateIt() { + String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/name?someParams&"); + + Assert.assertEquals("jdbc:mysql://host:5555/name?someParams&connectionCollation=utf8mb4_general_ci", result); + } + + @Test + public void addDefaultConnectionCollationTestUriDefiningConnectionCollationKeepsItUntouched() { + String uri = "jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_unicode_ci"; + + Assert.assertEquals(uri, TransactionLegacy.addDefaultConnectionCollation(uri)); + } + + @Test + public void addDefaultConnectionCollationTestUriDefiningCharacterEncodingKeepsItUntouched() { + String uri = "jdbc:mysql://host:5555/name?characterEncoding=UTF-8"; + + Assert.assertEquals(uri, TransactionLegacy.addDefaultConnectionCollation(uri)); + } + + @Test + public void shouldPinConnectionCollationTestUriWithHostContainingTheParameterNamesReturnsTrue() { + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://characterEncoding:5555/name")); + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://connectionCollation.example.com:5555/name?someParams")); + } + + @Test + public void shouldPinConnectionCollationTestUriWithDatabaseNameContainingTheParameterNamesReturnsTrue() { + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/connectionCollation")); + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/characterEncoding?someParams")); + } + + @Test + public void shouldPinConnectionCollationTestParameterValueContainingTheParameterNamesReturnsTrue() { + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?user=connectionCollation")); + Assert.assertTrue(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?user=characterEncoding&someParams")); + } + + @Test + public void shouldPinConnectionCollationTestUriDefiningTheParametersAsTheFirstOneReturnsFalse() { + Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?connectionCollation=utf8mb4_unicode_ci")); + Assert.assertFalse(TransactionLegacy.shouldPinConnectionCollation("jdbc:mysql://host:5555/name?characterEncoding=UTF-8&someParams")); + } + + @Test + public void addDefaultConnectionCollationTestUriWithHostContainingTheParameterNamesAddsTheDefaultCollation() { + String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://characterEncoding.example.com:5555/name"); + + Assert.assertEquals("jdbc:mysql://characterEncoding.example.com:5555/name?connectionCollation=utf8mb4_general_ci", result); + } + + @Test + public void addDefaultConnectionCollationTestUriWithDatabaseNameContainingTheParameterNamesAddsTheDefaultCollation() { + String result = TransactionLegacy.addDefaultConnectionCollation("jdbc:mysql://host:5555/connectionCollation?someParams"); + + Assert.assertEquals("jdbc:mysql://host:5555/connectionCollation?someParams&connectionCollation=utf8mb4_general_ci", result); + } + + @Test + public void getConnectionUriAndDriverTestWithUriWhoseDatabaseNameContainsTheParameterNamePinsTheDefaultCollation() { + properties.setProperty("db.cloud.uri", "jdbc:mysql://host:5555/characterEncoding"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("jdbc:mysql://host:5555/characterEncoding?connectionCollation=utf8mb4_general_ci", result.first()); + } + + @Test + public void getConnectionUriAndDriverTestWithoutUriAndDatabaseNameContainingTheParameterNamePinsTheDefaultCollation() { + properties.setProperty("db.cloud.uri", ""); + properties.setProperty("db.cloud.driver", "driver"); + properties.setProperty("db.cloud.name", "connectionCollation"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("driver://host:5555/connectionCollation?autoReconnect=false&someParams&scrollTolerantForwardOnly=true" + + "&connectionCollation=utf8mb4_general_ci", result.first()); + } + + @Test + public void containsUriParamTestParamDefinedAsTheFirstOneReturnsTrue() { + Assert.assertTrue(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?serverTimezone=UTC", "serverTimezone")); + } + + @Test + public void containsUriParamTestParamDefinedAfterOtherParamsReturnsTrue() { + Assert.assertTrue(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?someParams&serverTimezone=UTC", "serverTimezone")); + } + + @Test + public void containsUriParamTestOnlyTheUriParamsReturnsTrue() { + Assert.assertTrue(TransactionLegacy.containsUriParam("serverTimezone=UTC&someParams", "serverTimezone")); + } + + @Test + public void containsUriParamTestParamIsPartOfTheHostOrOfTheDatabaseNameReturnsFalse() { + Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://serverTimezone:5555/name", "serverTimezone")); + Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/serverTimezone?someParams", "serverTimezone")); + } + + @Test + public void containsUriParamTestParamIsTheValueOfAnotherParamReturnsFalse() { + Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?user=serverTimezone", "serverTimezone")); + } + + @Test + public void containsUriParamTestParamIsNotDefinedReturnsFalse() { + Assert.assertFalse(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?someParams", "serverTimezone")); + } + + @Test + public void containsUriParamTestNullConnectionUriReturnsFalse() { + Assert.assertFalse(TransactionLegacy.containsUriParam(null, "serverTimezone")); + } + + @Test + public void containsUriParamTestIsCaseInsensitive() { + Assert.assertTrue(TransactionLegacy.containsUriParam("jdbc:mysql://host:5555/name?SERVERTIMEZONE=UTC", "serverTimezone")); } }