Skip to content

Commit 48953e3

Browse files
committed
Allow list for JDBC properties
1 parent 287c9be commit 48953e3

File tree

33 files changed

+2209
-69
lines changed

33 files changed

+2209
-69
lines changed
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.utils;
21+
22+
import com.google.common.base.Preconditions;
23+
24+
import java.util.Set;
25+
26+
public final class ConnectionUriUtils
27+
{
28+
// Note: MySQL JDBC connector 8 supports 7 other protocols than just `jdbc:mysql:`
29+
// (https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-jdbc-url-format.html).
30+
// We should consider either expanding recognized mysql protocols or restricting allowed protocols to
31+
// just a basic one.
32+
public static final String MYSQL_PREFIX = "jdbc:mysql:";
33+
public static final String POSTGRES_PREFIX = "jdbc:postgresql:";
34+
35+
/**
36+
* This method checks {@param actualProperties} against {@param allowedProperties} if they are not system properties.
37+
* A property is regarded as a system property if its name starts with a prefix in {@param systemPropertyPrefixes}.
38+
* See org.apache.druid.server.initialization.JDBCAccessSecurityConfig for more details.
39+
*
40+
* If a non-system property that is not allowed is found, this method throws an {@link IllegalArgumentException}.
41+
*/
42+
public static void throwIfPropertiesAreNotAllowed(
43+
Set<String> actualProperties,
44+
Set<String> systemPropertyPrefixes,
45+
Set<String> allowedProperties
46+
)
47+
{
48+
for (String property : actualProperties) {
49+
if (systemPropertyPrefixes.stream().noneMatch(property::startsWith)) {
50+
Preconditions.checkArgument(
51+
allowedProperties.contains(property),
52+
"The property [%s] is not in the allowed list %s",
53+
property,
54+
allowedProperties
55+
);
56+
}
57+
}
58+
}
59+
60+
private ConnectionUriUtils()
61+
{
62+
}
63+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.utils;
21+
22+
public final class Throwables
23+
{
24+
public static boolean isThrowable(Throwable t, Class<? extends Throwable> searchFor)
25+
{
26+
if (t.getClass().isAssignableFrom(searchFor)) {
27+
return true;
28+
} else {
29+
if (t.getCause() != null) {
30+
return isThrowable(t.getCause(), searchFor);
31+
} else {
32+
return false;
33+
}
34+
}
35+
}
36+
37+
private Throwables()
38+
{
39+
}
40+
}
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.utils;
21+
22+
import com.google.common.collect.ImmutableSet;
23+
import org.junit.Rule;
24+
import org.junit.Test;
25+
import org.junit.experimental.runners.Enclosed;
26+
import org.junit.rules.ExpectedException;
27+
import org.junit.runner.RunWith;
28+
29+
@RunWith(Enclosed.class)
30+
public class ConnectionUriUtilsTest
31+
{
32+
public static class ThrowIfURLHasNotAllowedPropertiesTest
33+
{
34+
@Rule
35+
public ExpectedException expectedException = ExpectedException.none();
36+
37+
@Test
38+
public void testEmptyActualProperties()
39+
{
40+
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
41+
ImmutableSet.of(),
42+
ImmutableSet.of("valid_key1", "valid_key2"),
43+
ImmutableSet.of("system_key1", "system_key2")
44+
);
45+
}
46+
47+
@Test
48+
public void testThrowForNonAllowedProperties()
49+
{
50+
expectedException.expect(IllegalArgumentException.class);
51+
expectedException.expectMessage("The property [invalid_key] is not in the allowed list [valid_key1, valid_key2]");
52+
53+
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
54+
ImmutableSet.of("valid_key1", "invalid_key"),
55+
ImmutableSet.of("system_key1", "system_key2"),
56+
ImmutableSet.of("valid_key1", "valid_key2")
57+
);
58+
}
59+
60+
@Test
61+
public void testAllowedProperties()
62+
{
63+
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
64+
ImmutableSet.of("valid_key2"),
65+
ImmutableSet.of("system_key1", "system_key2"),
66+
ImmutableSet.of("valid_key1", "valid_key2")
67+
);
68+
}
69+
70+
@Test
71+
public void testAllowSystemProperties()
72+
{
73+
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
74+
ImmutableSet.of("system_key1", "valid_key2"),
75+
ImmutableSet.of("system_key1", "system_key2"),
76+
ImmutableSet.of("valid_key1", "valid_key2")
77+
);
78+
}
79+
80+
@Test
81+
public void testMatchSystemProperties()
82+
{
83+
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
84+
ImmutableSet.of("system_key1.1", "system_key1.5", "system_key11.11", "valid_key2"),
85+
ImmutableSet.of("system_key1", "system_key2"),
86+
ImmutableSet.of("valid_key1", "valid_key2")
87+
);
88+
}
89+
}
90+
}
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.utils;
21+
22+
import org.junit.Assert;
23+
import org.junit.Test;
24+
25+
public class ThrowablesTest
26+
{
27+
@Test
28+
public void testIsThrowableItself()
29+
{
30+
Assert.assertTrue(Throwables.isThrowable(new NoClassDefFoundError(), NoClassDefFoundError.class));
31+
}
32+
33+
@Test
34+
public void testIsThrowableNestedThrowable()
35+
{
36+
Assert.assertTrue(
37+
Throwables.isThrowable(new RuntimeException(new NoClassDefFoundError()), NoClassDefFoundError.class)
38+
);
39+
}
40+
41+
@Test
42+
public void testIsThrowableNonTarget()
43+
{
44+
Assert.assertFalse(
45+
Throwables.isThrowable(new RuntimeException(new ClassNotFoundException()), NoClassDefFoundError.class)
46+
);
47+
}
48+
}

docs/configuration/index.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,25 @@ This deep storage is used to interface with Cassandra. Note that the `druid-cas
515515
|`druid.storage.keyspace`|Cassandra key space.|none|
516516

517517

518+
### Ingestion Security Configuration
519+
520+
#### JDBC Connections to External Databases
521+
522+
You can use the following properties to specify permissible JDBC options for:
523+
- [SQL input source](../ingestion/native-batch.md#sql-input-source)
524+
- [SQL firehose](../ingestion/native-batch.md#sqlfirehose),
525+
- [globally cached JDBC lookups](../development/extensions-core/lookups-cached-global.md#jdbc-lookup)
526+
- [JDBC Data Fetcher for per-lookup caching](../development/extensions-core/druid-lookups.md#data-fetcher-layer).
527+
528+
These properties do not apply to metadata storage connections.
529+
530+
|Property|Possible Values|Description|Default|
531+
|--------|---------------|-----------|-------|
532+
|`druid.access.jdbc.enforceAllowedProperties`|Boolean|When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is deprecated and will be removed in a future release.|false|
533+
|`druid.access.jdbc.allowedProperties`|List of JDBC properties|Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.<br/><br/>This option is tested against MySQL connector 5.1.48 and PostgreSQL connector 42.2.14. Other connector versions might not work.|["useSSL", "requireSSL", "ssl", "sslmode"]|
534+
|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean|When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL.|true|
535+
536+
518537
### Task Logging
519538

520539
If you are running the indexing service in remote mode, the task logs must be stored in S3, Azure Blob Store, Google Cloud Storage or HDFS.

docs/development/extensions-core/druid-lookups.md

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ Same for Loading cache, developer can implement a new type of loading cache by i
7272

7373
|Field|Type|Description|Required|default|
7474
|-----|----|-----------|--------|-------|
75-
|dataFetcher|JSON object|Specifies the lookup data fetcher type to use in order to fetch data|yes|null|
75+
|dataFetcher|JSON object|Specifies the lookup data fetcher type for fetching data|yes|null|
7676
|cacheFactory|JSON Object|Cache factory implementation|no |onHeapPolling|
7777
|pollPeriod|Period|polling period |no |null (poll once)|
7878

@@ -129,7 +129,7 @@ Guava cache configuration spec.
129129
"type":"loadingLookup",
130130
"dataFetcher":{ "type":"jdbcDataFetcher", "connectorConfig":"jdbc://mysql://localhost:3306/my_data_base", "table":"lookup_table_name", "keyColumn":"key_column_name", "valueColumn": "value_column_name"},
131131
"loadingCacheSpec":{"type":"guava"},
132-
"reverseLoadingCacheSpec":{"type":"guava", "maximumSize":500000, "expireAfterAccess":100000, "expireAfterAccess":10000}
132+
"reverseLoadingCacheSpec":{"type":"guava", "maximumSize":500000, "expireAfterAccess":100000, "expireAfterWrite":10000}
133133
}
134134
```
135135

@@ -150,6 +150,16 @@ Off heap cache is backed by [MapDB](http://www.mapdb.org/) implementation. MapDB
150150
"type":"loadingLookup",
151151
"dataFetcher":{ "type":"jdbcDataFetcher", "connectorConfig":"jdbc://mysql://localhost:3306/my_data_base", "table":"lookup_table_name", "keyColumn":"key_column_name", "valueColumn": "value_column_name"},
152152
"loadingCacheSpec":{"type":"mapDb", "maxEntriesSize":100000},
153-
"reverseLoadingCacheSpec":{"type":"mapDb", "maxStoreSize":5, "expireAfterAccess":100000, "expireAfterAccess":10000}
153+
"reverseLoadingCacheSpec":{"type":"mapDb", "maxStoreSize":5, "expireAfterAccess":100000, "expireAfterWrite":10000}
154154
}
155155
```
156+
157+
### JDBC Data Fetcher
158+
159+
|Field|Type|Description|Required|default|
160+
|-----|----|-----------|--------|-------|
161+
|`connectorConfig`|JSON object|Specifies the database connection details. You can set `connectURI`, `user` and `password`. You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../../configuration/index.md#jdbc-connections-to-external-databases) for more details.|yes||
162+
|`table`|string|The table name to read from.|yes||
163+
|`keyColumn`|string|The column name that contains the lookup key.|yes||
164+
|`valueColumn`|string|The column name that contains the lookup value.|yes||
165+
|`streamingFetchSize`|int|Fetch size used in JDBC connections.|no|1000|

docs/development/extensions-core/lookups-cached-global.md

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ Globally cached lookups can be specified as part of the [cluster wide config for
6464
"extractionNamespace": {
6565
"type": "jdbc",
6666
"connectorConfig": {
67-
"createTables": true,
6867
"connectURI": "jdbc:mysql:\/\/localhost:3306\/druid",
6968
"user": "druid",
7069
"password": "diurd"
@@ -107,7 +106,6 @@ In a simple case where only one [tier](../../querying/lookups.html#dynamic-confi
107106
"extractionNamespace": {
108107
"type": "jdbc",
109108
"connectorConfig": {
110-
"createTables": true,
111109
"connectURI": "jdbc:mysql:\/\/localhost:3306\/druid",
112110
"user": "druid",
113111
"password": "diurd"
@@ -136,7 +134,6 @@ Where the Coordinator endpoint `/druid/coordinator/v1/lookups/realtime_customer2
136134
"extractionNamespace": {
137135
"type": "jdbc",
138136
"connectorConfig": {
139-
"createTables": true,
140137
"connectURI": "jdbc:mysql://localhost:3306/druid",
141138
"user": "druid",
142139
"password": "diurd"
@@ -342,7 +339,7 @@ The JDBC lookups will poll a database to populate its local cache. If the `tsCol
342339
|Parameter|Description|Required|Default|
343340
|---------|-----------|--------|-------|
344341
|`namespace`|The namespace to define|Yes||
345-
|`connectorConfig`|The connector config to use|Yes||
342+
|`connectorConfig`|The connector config to use. You can set `connectURI`, `user` and `password`. You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../../configuration/index.md#jdbc-connections-to-external-databases) for more details.|Yes||
346343
|`table`|The table which contains the key value pairs|Yes||
347344
|`keyColumn`|The column in `table` which contains the keys|Yes||
348345
|`valueColumn`|The column in `table` which contains the values|Yes||
@@ -355,7 +352,6 @@ The JDBC lookups will poll a database to populate its local cache. If the `tsCol
355352
"type":"jdbc",
356353
"namespace":"some_lookup",
357354
"connectorConfig":{
358-
"createTables":true,
359355
"connectURI":"jdbc:mysql://localhost:3306/druid",
360356
"user":"druid",
361357
"password":"diurd"

docs/development/extensions-core/mysql.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,6 @@ Copy or symlink this file to `extensions/mysql-metadata-storage` under the distr
105105
|`druid.metadata.mysql.ssl.enabledSSLCipherSuites`|Overrides the existing cipher suites with these cipher suites.|none|no|
106106
|`druid.metadata.mysql.ssl.enabledTLSProtocols`|Overrides the TLS protocols with these protocols.|none|no|
107107

108-
109108
### MySQL Firehose
110109

111110
The MySQL extension provides an implementation of an [SqlFirehose](../../ingestion/native-batch.md#firehoses-deprecated) which can be used to ingest data into Druid from a MySQL database.

docs/ingestion/native-batch.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1346,7 +1346,7 @@ Please refer to the Recommended practices section below before using this input
13461346
|property|description|required?|
13471347
|--------|-----------|---------|
13481348
|type|This should be "sql".|Yes|
1349-
|database|Specifies the database connection details. The database type corresponds to the extension that supplies the `connectorConfig` support and this extension must be loaded into Druid. For database types `mysql` and `postgresql`, the `connectorConfig` support is provided by [mysql-metadata-storage](../development/extensions-core/mysql.md) and [postgresql-metadata-storage](../development/extensions-core/postgresql.md) extensions respectively.|Yes|
1349+
|database|Specifies the database connection details. The database type corresponds to the extension that supplies the `connectorConfig` support. The specified extension must be loaded into Druid:<br/><br/><ul><li>[mysql-metadata-storage](../development/extensions-core/mysql.md) for `mysql`</li><li> [postgresql-metadata-storage](../development/extensions-core/postgresql.md) extension for `postgresql`.</li></ul><br/><br/>You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../configuration/index.md#jdbc-connections-to-external-databases) for more details.|Yes|
13501350
|foldCase|Toggle case folding of database column names. This may be enabled in cases where the database returns case insensitive column names in query results.|No|
13511351
|sqls|List of SQL queries where each SQL query would retrieve the data to be indexed.|Yes|
13521352

@@ -1692,7 +1692,7 @@ Requires one of the following extensions:
16921692
|property|description|default|required?|
16931693
|--------|-----------|-------|---------|
16941694
|type|This should be "sql".||Yes|
1695-
|database|Specifies the database connection details.||Yes|
1695+
|database|Specifies the database connection details. The database type corresponds to the extension that supplies the `connectorConfig` support. The specified extension must be loaded into Druid:<br/><br/><ul><li>[mysql-metadata-storage](../development/extensions-core/mysql.md) for `mysql`</li><li> [postgresql-metadata-storage](../development/extensions-core/postgresql.md) extension for `postgresql`.</li></ul><br/><br/>You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../configuration/index.md#jdbc-connections-to-external-databases) for more details.||Yes|
16961696
|maxCacheCapacityBytes|Maximum size of the cache space in bytes. 0 means disabling cache. Cached files are not removed until the ingestion task completes.|1073741824|No|
16971697
|maxFetchCapacityBytes|Maximum size of the fetch space in bytes. 0 means disabling prefetch. Prefetched files are removed immediately once they are read.|1073741824|No|
16981698
|prefetchTriggerBytes|Threshold to trigger prefetching SQL result objects.|maxFetchCapacityBytes / 2|No|

extensions-core/lookups-cached-global/pom.xml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,12 +110,15 @@
110110
<artifactId>jsr311-api</artifactId>
111111
<scope>provided</scope>
112112
</dependency>
113-
114-
<!-- Included to improve the out-of-the-box experience for supported JDBC connectors -->
113+
<dependency>
114+
<groupId>mysql</groupId>
115+
<artifactId>mysql-connector-java</artifactId>
116+
<version>${mysql.version}</version>
117+
<scope>provided</scope>
118+
</dependency>
115119
<dependency>
116120
<groupId>org.postgresql</groupId>
117121
<artifactId>postgresql</artifactId>
118-
<scope>runtime</scope>
119122
</dependency>
120123

121124
<!-- Tests -->

0 commit comments

Comments
 (0)