diff --git a/presto-docs/src/main/sphinx/connector/deltalake.rst b/presto-docs/src/main/sphinx/connector/deltalake.rst index 7630cb80074d5..96c7899e56972 100644 --- a/presto-docs/src/main/sphinx/connector/deltalake.rst +++ b/presto-docs/src/main/sphinx/connector/deltalake.rst @@ -41,6 +41,9 @@ Property Name Description metastore to find the location of Delta Lake tables. From the Delta Log at given location, schema and data file list of the table is found. + +``hive.metastore.catalog.name`` Specifies the catalog name to be passed to the metastore. + ``delta.parquet-dereference-pushdown-enabled`` Enable pushing nested column dereferences into ``true`` table scan so that only the required fields selected in a ``struct`` data type column are selected. diff --git a/presto-docs/src/main/sphinx/connector/hive.rst b/presto-docs/src/main/sphinx/connector/hive.rst index 440aadf69c312..113c5520dd12c 100644 --- a/presto-docs/src/main/sphinx/connector/hive.rst +++ b/presto-docs/src/main/sphinx/connector/hive.rst @@ -206,6 +206,8 @@ Property Name Description error iterating through empty files. ``hive.file-status-cache.max-retained-size`` Maximum size in bytes of the directory listing cache ``0KB`` + + ``hive.metastore.catalog.name`` Specifies the catalog name to be passed to the metastore. ================================================== ============================================================ ============ Metastore Configuration Properties diff --git a/presto-docs/src/main/sphinx/connector/hudi.rst b/presto-docs/src/main/sphinx/connector/hudi.rst index 2581948de6bf4..6d6ef20b17187 100644 --- a/presto-docs/src/main/sphinx/connector/hudi.rst +++ b/presto-docs/src/main/sphinx/connector/hudi.rst @@ -38,6 +38,8 @@ Property Name Description ======================================= ============================================= =========== ``hudi.metadata-table-enabled`` Fetch the list of file names and sizes from false Hudi's metadata table rather than storage. +``hive.metastore.catalog.name`` Specifies the catalog name to be passed to + the metastore. ======================================= ============================================= =========== File-Based Metastore diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index 74154c079f763..dbc1c7a730b6d 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -69,6 +69,8 @@ Property Name Description ``iceberg.catalog.type`` is ``hive`` and ``hive.metastore`` is ``thrift``. +``hive.metastore.catalog.name`` Specifies the catalog name to be passed to the metastore. + ``iceberg.hive-statistics-merge-strategy`` Comma separated list of statistics to use from the Hive Metastore to override Iceberg table statistics. The available values are ``NUMBER_OF_DISTINCT_VALUES`` diff --git a/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveCommonClientConfig.java b/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveCommonClientConfig.java index f86e252f9880d..9a2ec46dd00f9 100644 --- a/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveCommonClientConfig.java +++ b/presto-hive-common/src/main/java/com/facebook/presto/hive/HiveCommonClientConfig.java @@ -46,6 +46,7 @@ public class HiveCommonClientConfig private boolean readNullMaskedParquetEncryptedValueEnabled; private boolean useParquetColumnNames; private boolean zstdJniDecompressionEnabled; + private String catalogName; private DataSize affinitySchedulingFileSectionSize = new DataSize(256, MEGABYTE); public NodeSelectionStrategy getNodeSelectionStrategy() @@ -286,6 +287,19 @@ public HiveCommonClientConfig setZstdJniDecompressionEnabled(boolean zstdJniDeco return this; } + public String getCatalogName() + { + return catalogName; + } + + @Config("hive.metastore.catalog.name") + @ConfigDescription("Specified property to store the metastore catalog name.") + public HiveCommonClientConfig setCatalogName(String catalogName) + { + this.catalogName = catalogName; + return this; + } + @NotNull public DataSize getAffinitySchedulingFileSectionSize() { diff --git a/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java b/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java index b1428356b3644..67e4c4201968c 100644 --- a/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java +++ b/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java @@ -30,6 +30,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import javax.annotation.Nullable; + import java.util.HashSet; import java.util.List; import java.util.Map; @@ -49,6 +51,10 @@ public final class MetadataUtils { + private static final String CATALOG_DB_SEPARATOR = "#"; + private static final String CATALOG_DB_THRIFT_NAME_MARKER = "@"; + private static final String DB_EMPTY_MARKER = "!"; + private static final String DEFAULT_DATABASE = "default"; private MetadataUtils() {} public static Optional getDiscretePredicates(List partitionColumns, List partitions) @@ -160,4 +166,26 @@ private static Domain buildColumnDomain(ColumnHandle column, List return Domain.onlyNull(type); } + + /** + * Constructs the schema name, including catalog name if applicable. + * + * @param schemaName the original schema name + * @return the formatted schema name (Example - @catalog_name#schema_name) + */ + public static String constructSchemaName(Optional catalogName, @Nullable String schemaName) + { + if (!catalogName.isPresent() || DEFAULT_DATABASE.equals(schemaName) || + (schemaName != null && schemaName.contains(CATALOG_DB_SEPARATOR))) { + return schemaName; + } + + StringBuilder catalogDatabaseName = new StringBuilder() + .append(CATALOG_DB_THRIFT_NAME_MARKER) + .append(catalogName.get()) // Safe since we checked isPresent() + .append(CATALOG_DB_SEPARATOR) + .append(schemaName == null ? "" : schemaName.isEmpty() ? DB_EMPTY_MARKER : schemaName); + + return catalogDatabaseName.toString(); + } } diff --git a/presto-hive-common/src/test/java/com/facebook/presto/hive/TestHiveCommonClientConfig.java b/presto-hive-common/src/test/java/com/facebook/presto/hive/TestHiveCommonClientConfig.java index 5db4217f6f17e..b543c456cd308 100644 --- a/presto-hive-common/src/test/java/com/facebook/presto/hive/TestHiveCommonClientConfig.java +++ b/presto-hive-common/src/test/java/com/facebook/presto/hive/TestHiveCommonClientConfig.java @@ -49,6 +49,7 @@ public void testDefaults() .setParquetBatchReaderVerificationEnabled(false) .setParquetBatchReadOptimizationEnabled(false) .setReadNullMaskedParquetEncryptedValue(false) + .setCatalogName(null) .setAffinitySchedulingFileSectionSize(new DataSize(256, MEGABYTE))); } @@ -74,6 +75,7 @@ public void testExplicitPropertyMappings() .put("hive.enable-parquet-batch-reader-verification", "true") .put("hive.parquet-batch-read-optimization-enabled", "true") .put("hive.read-null-masked-parquet-encrypted-value-enabled", "true") + .put("hive.metastore.catalog.name", "catalogName") .put("hive.affinity-scheduling-file-section-size", "512MB") .build(); @@ -96,6 +98,7 @@ public void testExplicitPropertyMappings() .setParquetBatchReaderVerificationEnabled(true) .setParquetBatchReadOptimizationEnabled(true) .setReadNullMaskedParquetEncryptedValue(true) + .setCatalogName("catalogName") .setAffinitySchedulingFileSectionSize(new DataSize(512, MEGABYTE)); ConfigAssertions.assertFullMapping(properties, expected); diff --git a/presto-hive-common/src/test/java/com/facebook/presto/hive/TestMetadataUtils.java b/presto-hive-common/src/test/java/com/facebook/presto/hive/TestMetadataUtils.java new file mode 100644 index 0000000000000..d8bea3a4396f5 --- /dev/null +++ b/presto-hive-common/src/test/java/com/facebook/presto/hive/TestMetadataUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.hive.MetadataUtils.constructSchemaName; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +public class TestMetadataUtils +{ + @Test + public void testWithCatalogAndValidSchema() + { + String result = constructSchemaName(Optional.of("testCatalog"), "testSchema"); + assertTrue(result.equals("@testCatalog#testSchema")); + } + + @Test + public void testWithCatalogAndDefaultSchema() + { + String result = constructSchemaName(Optional.of("testCatalog"), "default"); + assertTrue(result.equals("default")); + } + + @Test + public void testWithCatalogAndSchemaContainingSeparator() + { + String result = constructSchemaName(Optional.of("testCatalog"), "schema#with#dot"); + assertTrue(result.equals("schema#with#dot")); + } + + @Test + public void testWithoutCatalog() + { + String result = constructSchemaName(Optional.empty(), "testSchema"); + assertTrue(result.equals("testSchema")); + } + + @Test + public void testWithNullSchema() + { + String result = constructSchemaName(Optional.empty(), null); + assertNull(result); + } + + @Test + public void testWithoutCatalogNameAndEmptySchemaName() + { + String result = constructSchemaName(Optional.empty(), ""); + assertTrue(result.isEmpty()); + } + + @Test + public void testWithCatalogNameAndEmptySchemaName() + { + String result = constructSchemaName(Optional.of("testCatalog"), ""); + assertTrue(result.equals("@testCatalog#!")); + } +} diff --git a/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java b/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java index e81470dda7e9a..e28d632f64354 100644 --- a/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java +++ b/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java @@ -28,6 +28,7 @@ import com.facebook.presto.hive.HiveCoercionPolicy; import com.facebook.presto.hive.HiveColumnConverterProvider; import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.HiveEncryptionInformationProvider; import com.facebook.presto.hive.HiveFileRenamer; import com.facebook.presto.hive.HiveHdfsConfiguration; @@ -137,7 +138,7 @@ public S3SelectTestHelper(String host, metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy)); } - HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, host, port); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, host, port, new HiveCommonClientConfig()); executor = newCachedThreadPool(daemonThreadsNamed("hive-%s")); HivePartitionManager hivePartitionManager = new HivePartitionManager(FUNCTION_AND_TYPE_MANAGER, config); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java index bbc6ac56bc0bb..2d751e714c694 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java @@ -39,6 +39,7 @@ public class Database private final PrincipalType ownerType; private final Optional comment; private final Map parameters; + private final Optional catalogName; @JsonCreator public Database( @@ -47,7 +48,8 @@ public Database( @JsonProperty("ownerName") String ownerName, @JsonProperty("ownerType") PrincipalType ownerType, @JsonProperty("comment") Optional comment, - @JsonProperty("parameters") Map parameters) + @JsonProperty("parameters") Map parameters, + @JsonProperty("catalogName") Optional catalogName) { this.databaseName = requireNonNull(databaseName, "databaseName is null"); this.location = requireNonNull(location, "location is null"); @@ -55,6 +57,7 @@ public Database( this.ownerType = requireNonNull(ownerType, "ownerType is null"); this.comment = requireNonNull(comment, "comment is null"); this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); + this.catalogName = requireNonNull(catalogName, "catalogName is null"); } @JsonProperty @@ -103,6 +106,12 @@ public static Builder builder(Database database) return new Builder(database); } + @JsonProperty + public Optional getCatalogName() + { + return catalogName; + } + public static class Builder { private String databaseName; @@ -111,6 +120,7 @@ public static class Builder private PrincipalType ownerType; private Optional comment = Optional.empty(); private Map parameters = new LinkedHashMap<>(); + private Optional catalogName = Optional.empty(); public Builder() {} @@ -122,6 +132,7 @@ public Builder(Database database) this.ownerType = database.ownerType; this.comment = database.comment; this.parameters = database.parameters; + this.catalogName = database.catalogName; } public Builder setDatabaseName(String databaseName) @@ -166,6 +177,12 @@ public Builder setParameters(Map parameters) return this; } + public Builder setCatalogName(Optional catalogName) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + return this; + } + public Database build() { return new Database( @@ -174,7 +191,8 @@ public Database build() ownerName, ownerType, comment, - parameters); + parameters, + catalogName); } } @@ -188,6 +206,7 @@ public String toString() .add("ownerType", ownerType) .add("comment", comment) .add("parameters", parameters) + .add("catalogName", catalogName) .toString(); } @@ -207,12 +226,13 @@ public boolean equals(Object o) Objects.equals(ownerName, database.ownerName) && ownerType == database.ownerType && Objects.equals(comment, database.comment) && - Objects.equals(parameters, database.parameters); + Objects.equals(parameters, database.parameters) && + Objects.equals(catalogName, database.catalogName); } @Override public int hashCode() { - return Objects.hash(databaseName, location, ownerName, ownerType, comment, parameters); + return Objects.hash(databaseName, location, ownerName, ownerType, comment, parameters, catalogName); } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java index fdf8e87a3153e..40163d4bc302e 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java @@ -35,6 +35,7 @@ @Immutable public class Partition { + private final Optional catalogName; private final String databaseName; private final String tableName; private final List values; @@ -63,6 +64,37 @@ public Partition( @JsonProperty("lastDataCommitTime") long lastDataCommitTime, @JsonProperty("rowIdPartitionComponent") Optional rowIdPartitionComponent) { + this( + Optional.empty(), + databaseName, + tableName, + values, + storage, + columns, + parameters, + partitionVersion, + eligibleToIgnore, + sealedPartition, + createTime, + lastDataCommitTime, + rowIdPartitionComponent); + } + public Partition( + Optional catalogName, + String databaseName, + String tableName, + List values, + Storage storage, + List columns, + Map parameters, + Optional partitionVersion, + boolean eligibleToIgnore, + boolean sealedPartition, + int createTime, + long lastDataCommitTime, + Optional rowIdPartitionComponent) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.databaseName = requireNonNull(databaseName, "databaseName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); this.values = ImmutableList.copyOf(requireNonNull(values, "values is null")); @@ -77,6 +109,11 @@ public Partition( this.rowIdPartitionComponent = requireNonNull(rowIdPartitionComponent); } + @JsonProperty + public Optional getCatalogName() + { + return catalogName; + } @JsonProperty public String getDatabaseName() { @@ -184,7 +221,8 @@ public boolean equals(Object o) } Partition partition = (Partition) o; - return Objects.equals(databaseName, partition.databaseName) && + return Objects.equals(catalogName, partition.catalogName) && + Objects.equals(databaseName, partition.databaseName) && Objects.equals(tableName, partition.tableName) && Objects.equals(values, partition.values) && Objects.equals(storage, partition.storage) && @@ -200,7 +238,7 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(databaseName, tableName, values, storage, columns, parameters, partitionVersion, eligibleToIgnore, sealedPartition, createTime, lastDataCommitTime); + return Objects.hash(catalogName, databaseName, tableName, values, storage, columns, parameters, partitionVersion, eligibleToIgnore, sealedPartition, createTime, lastDataCommitTime); } public static Builder builder() @@ -216,6 +254,7 @@ public static Builder builder(Partition partition) public static class Builder { private final Storage.Builder storageBuilder; + private Optional catalogName; private String databaseName; private String tableName; private List values; @@ -236,6 +275,7 @@ private Builder() private Builder(Partition partition) { this.storageBuilder = Storage.builder(partition.getStorage()); + this.catalogName = partition.getCatalogName(); this.databaseName = partition.getDatabaseName(); this.tableName = partition.getTableName(); this.values = partition.getValues(); @@ -248,6 +288,11 @@ private Builder(Partition partition) this.rowIdPartitionComponent = partition.getRowIdPartitionComponent(); } + public Builder setCatalogName(Optional catalogName) + { + this.catalogName = catalogName; + return this; + } public Builder setDatabaseName(String databaseName) { this.databaseName = databaseName; @@ -332,7 +377,7 @@ public Builder setRowIdPartitionComponent(Optional rowIdPartitionCompone public Partition build() { - return new Partition(databaseName, tableName, values, storageBuilder.build(), columns, parameters, partitionVersion, isEligibleToIgnore, isSealedPartition, createTime, lastDataCommitTime, rowIdPartitionComponent); + return new Partition(catalogName, databaseName, tableName, values, storageBuilder.build(), columns, parameters, partitionVersion, isEligibleToIgnore, isSealedPartition, createTime, lastDataCommitTime, rowIdPartitionComponent); } } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java index 654e2fd4b475a..9f8ee10dba1dc 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java @@ -37,6 +37,7 @@ @Immutable public class Table { + private final Optional catalogName; private final String databaseName; private final String tableName; private final String owner; @@ -50,6 +51,7 @@ public class Table @JsonCreator public Table( + @JsonProperty("catalogName") Optional catalogName, @JsonProperty("databaseName") String databaseName, @JsonProperty("tableName") String tableName, @JsonProperty("owner") String owner, @@ -61,6 +63,7 @@ public Table( @JsonProperty("viewOriginalText") Optional viewOriginalText, @JsonProperty("viewExpandedText") Optional viewExpandedText) { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.databaseName = requireNonNull(databaseName, "databaseName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); this.owner = requireNonNull(owner, "owner is null"); @@ -73,6 +76,11 @@ public Table( this.viewExpandedText = requireNonNull(viewExpandedText, "viewExpandedText is null"); } + @JsonProperty + public Optional getCatalogName() + { + return catalogName; + } @JsonProperty public String getDatabaseName() { @@ -160,6 +168,7 @@ public static Builder builder(Table table) public String toString() { return toStringHelper(this) + .add("catalogName", catalogName) .add("databaseName", databaseName) .add("tableName", tableName) .add("owner", owner) @@ -184,7 +193,8 @@ public boolean equals(Object o) } Table table = (Table) o; - return Objects.equals(databaseName, table.databaseName) && + return Objects.equals(catalogName, table.catalogName) && + Objects.equals(databaseName, table.databaseName) && Objects.equals(tableName, table.tableName) && Objects.equals(owner, table.owner) && Objects.equals(tableType, table.tableType) && @@ -200,6 +210,7 @@ public boolean equals(Object o) public int hashCode() { return Objects.hash( + catalogName, databaseName, tableName, owner, @@ -215,6 +226,7 @@ public int hashCode() public static class Builder { private final Storage.Builder storageBuilder; + private Optional catalogName = Optional.empty(); private String databaseName; private String tableName; private String owner; @@ -232,6 +244,7 @@ private Builder() private Builder(Table table) { + catalogName = table.catalogName; databaseName = table.databaseName; tableName = table.tableName; owner = table.owner; @@ -244,6 +257,11 @@ private Builder(Table table) viewExpandedText = table.viewExpandedText; } + public Builder setCatalogName(Optional catalogName) + { + this.catalogName = catalogName; + return this; + } public Builder setDatabaseName(String databaseName) { this.databaseName = databaseName; @@ -324,6 +342,7 @@ public Builder withStorage(Consumer consumer) public Table build() { return new Table( + catalogName, databaseName, tableName, owner, diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java index 212e0579433f6..c37a22af47b7f 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java @@ -13,20 +13,22 @@ */ package com.facebook.presto.hive.metastore.file; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.ForCachingHiveMetastore; +import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; import com.google.inject.Binder; -import com.google.inject.Module; import com.google.inject.Scopes; import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; public class FileMetastoreModule - implements Module + extends AbstractConfigurationAwareModule { private final String connectorId; @@ -36,8 +38,9 @@ public FileMetastoreModule(String connectorId) } @Override - public void configure(Binder binder) + public void setup(Binder binder) { + checkArgument(buildConfigObject(HiveCommonClientConfig.class).getCatalogName() == null, "'hive.metastore.catalog.name' should not be set for file metastore"); configBinder(binder).bindConfig(FileHiveMetastoreConfig.class); binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(FileHiveMetastore.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java index f9520ea40d33e..705796b940e3b 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java @@ -318,6 +318,7 @@ public TableMetadata withColumnStatistics(Map colu public Table toTable(String databaseName, String tableName, String location) { return new Table( + Optional.empty(), databaseName, tableName, owner, diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java index 98d7c3aa1e3dc..0bc9bee6b81fc 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java @@ -14,11 +14,12 @@ package com.facebook.presto.hive.metastore.glue; import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.ForCachingHiveMetastore; +import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; import com.google.inject.Binder; -import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; @@ -27,6 +28,7 @@ import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -34,7 +36,7 @@ import static org.weakref.jmx.guice.ExportBinder.newExporter; public class GlueMetastoreModule - implements Module + extends AbstractConfigurationAwareModule { private final String connectorId; @@ -44,8 +46,9 @@ public GlueMetastoreModule(String connectorId) } @Override - public void configure(Binder binder) + public void setup(Binder binder) { + checkArgument(buildConfigObject(HiveCommonClientConfig.class).getCatalogName() == null, "'hive.metastore.catalog.name' should not be set for glue metastore"); configBinder(binder).bindConfig(GlueHiveMetastoreConfig.class); binder.bind(GlueHiveMetastore.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(GlueHiveMetastore.class).in(Scopes.SINGLETON); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java index 542cb8b74f609..83de7d0150fc7 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java @@ -166,6 +166,7 @@ public Partition apply(com.amazonaws.services.glue.model.Partition gluePartition } Partition.Builder partitionBuilder = Partition.builder() + .setCatalogName(Optional.empty()) .setDatabaseName(databaseName) .setTableName(tableName) .setValues(gluePartition.getValues()) // No memoization benefit diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java index d5172c78ceefc..cf23a1e85aadf 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java @@ -62,6 +62,11 @@ public interface HiveMetastore MetastoreOperationResult alterTable(MetastoreContext metastoreContext, String databaseName, String tableName, Table table); + default List getDatabases(MetastoreContext metastoreContext, String pattern) + { + return getAllDatabases(metastoreContext); + } + List getAllDatabases(MetastoreContext metastoreContext); Optional> getAllTables(MetastoreContext metastoreContext, String databaseName); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java index 7f240bdbd4404..4880eedcdd6ee 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java @@ -50,6 +50,9 @@ public interface HiveMetastoreClient String getDelegationToken(String owner, String renewer) throws TException; + List getDatabases(String pattern) + throws TException; + List getAllDatabases() throws TException; diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java index ff58232057acc..19787c95fbad1 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive.metastore.thrift; import com.facebook.airlift.security.pem.PemReader; +import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; import com.facebook.presto.spi.PrestoException; @@ -55,22 +56,25 @@ public class HiveMetastoreClientFactory private final Optional socksProxy; private final int timeoutMillis; private final HiveMetastoreAuthentication metastoreAuthentication; + private final String catalogName; public static final String PROTOCOL = "TLS"; public HiveMetastoreClientFactory( Optional sslContext, Optional socksProxy, Duration timeout, - HiveMetastoreAuthentication metastoreAuthentication) + HiveMetastoreAuthentication metastoreAuthentication, + String catalogName) { this.sslContext = requireNonNull(sslContext, "sslContext is null"); this.socksProxy = requireNonNull(socksProxy, "socksProxy is null"); this.timeoutMillis = toIntExact(timeout.toMillis()); this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); + this.catalogName = catalogName; } @Inject - public HiveMetastoreClientFactory(MetastoreClientConfig metastoreClientConfig, ThriftHiveMetastoreConfig thriftHiveMetastoreConfig, HiveMetastoreAuthentication metastoreAuthentication) + public HiveMetastoreClientFactory(MetastoreClientConfig metastoreClientConfig, ThriftHiveMetastoreConfig thriftHiveMetastoreConfig, HiveMetastoreAuthentication metastoreAuthentication, HiveCommonClientConfig hiveCommonClientConfig) { this(buildSslContext(thriftHiveMetastoreConfig.isTlsEnabled(), Optional.ofNullable(thriftHiveMetastoreConfig.getKeystorePath()), @@ -78,13 +82,13 @@ public HiveMetastoreClientFactory(MetastoreClientConfig metastoreClientConfig, T Optional.ofNullable(thriftHiveMetastoreConfig.getTruststorePath()), Optional.ofNullable(thriftHiveMetastoreConfig.getTrustStorePassword())), Optional.ofNullable(metastoreClientConfig.getMetastoreSocksProxy()), - metastoreClientConfig.getMetastoreTimeout(), metastoreAuthentication); + metastoreClientConfig.getMetastoreTimeout(), metastoreAuthentication, hiveCommonClientConfig.getCatalogName()); } public HiveMetastoreClient create(HostAndPort address, Optional token) throws TTransportException { - return new ThriftHiveMetastoreClient(Transport.create(address, sslContext, socksProxy, timeoutMillis, metastoreAuthentication, token)); + return new ThriftHiveMetastoreClient(Transport.create(address, sslContext, socksProxy, timeoutMillis, metastoreAuthentication, token), catalogName); } /** diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java index 1171f9a85458d..24056038648c9 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java @@ -321,6 +321,23 @@ public List> getNotNullConstraints(MetastoreContext me } } + @Override + public List getDatabases(MetastoreContext context, String pattern) + { + try { + return retry() + .stopOnIllegalExceptions() + .run("getDatabases", stats.getGetDatabases().wrap(() -> + getMetastoreClientThenCall(context, client -> client.getDatabases(pattern)))); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + throw propagate(e); + } + } + @Override public List getAllDatabases(MetastoreContext context) { @@ -1646,15 +1663,16 @@ public MetastoreOperationResult addConstraint(MetastoreContext metastoreContext, if (tableConstraint instanceof PrimaryKeyConstraint) { for (String column : constraintColumns) { - primaryKeyConstraint.add( - new SQLPrimaryKey(table.getDbName(), - table.getTableName(), - column, - keySequence++, - tableConstraint.getName().orElse(null), - tableConstraint.isEnabled(), - tableConstraint.isEnforced(), - tableConstraint.isRely())); + SQLPrimaryKey sqlPrimaryKey = new SQLPrimaryKey(table.getDbName(), + table.getTableName(), + column, + keySequence++, + tableConstraint.getName().orElse(null), + tableConstraint.isEnabled(), + tableConstraint.isEnforced(), + tableConstraint.isRely()); + sqlPrimaryKey.setCatName(table.getCatName()); + primaryKeyConstraint.add(sqlPrimaryKey); } callableName = "addPrimaryKeyConstraint"; apiStats = stats.getAddPrimaryKeyConstraint(); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java index 4b6ef497b17e1..f34955d09432e 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java @@ -64,6 +64,8 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.hive.MetadataUtils.constructSchemaName; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; import static org.apache.thrift.TApplicationException.UNKNOWN_METHOD; @@ -73,17 +75,20 @@ public class ThriftHiveMetastoreClient { private final TTransport transport; private final ThriftHiveMetastore.Client client; + private final Optional catalogName; - public ThriftHiveMetastoreClient(TTransport transport) + public ThriftHiveMetastoreClient(TTransport transport, String catalogName) { this.transport = requireNonNull(transport, "transport is null"); this.client = new ThriftHiveMetastore.Client(new TBinaryProtocol(transport)); + this.catalogName = Optional.ofNullable(catalogName); } - public ThriftHiveMetastoreClient(TProtocol protocol) + public ThriftHiveMetastoreClient(TProtocol protocol, String catalogName) { this.transport = protocol.getTransport(); this.client = new ThriftHiveMetastore.Client(protocol); + this.catalogName = Optional.ofNullable(catalogName); } @Override @@ -98,11 +103,20 @@ public String getDelegationToken(String owner, String renewer) { return client.get_delegation_token(owner, renewer); } + @Override + public List getDatabases(String pattern) + throws TException + { + return client.get_databases(constructSchemaName(catalogName, pattern)); + } @Override public List getAllDatabases() throws TException { + if (catalogName.isPresent()) { + return getDatabases(constructSchemaName(catalogName, null)); + } return client.get_all_databases(); } @@ -110,27 +124,30 @@ public List getAllDatabases() public Database getDatabase(String dbName) throws TException { - return client.get_database(dbName); + return client.get_database(constructSchemaName(catalogName, dbName)); } @Override public List getAllTables(String databaseName) throws TException { - return client.get_all_tables(databaseName); + return client.get_all_tables(constructSchemaName(catalogName, databaseName)); } @Override public List getTableNamesByFilter(String databaseName, String filter) throws TException { - return client.get_table_names_by_filter(databaseName, filter, (short) -1); + return client.get_table_names_by_filter(constructSchemaName(catalogName, databaseName), filter, (short) -1); } @Override public void createDatabase(Database database) throws TException { + if (catalogName.isPresent()) { + database.setCatalogName(catalogName.get()); + } client.create_database(database); } @@ -138,20 +155,23 @@ public void createDatabase(Database database) public void dropDatabase(String databaseName, boolean deleteData, boolean cascade) throws TException { - client.drop_database(databaseName, deleteData, cascade); + client.drop_database(constructSchemaName(catalogName, databaseName), deleteData, cascade); } @Override public void alterDatabase(String databaseName, Database database) throws TException { - client.alter_database(databaseName, database); + client.alter_database(constructSchemaName(catalogName, databaseName), database); } @Override public void createTable(Table table) throws TException { + if (catalogName.isPresent()) { + table.setCatName(catalogName.get()); + } client.create_table(table); } @@ -159,6 +179,9 @@ public void createTable(Table table) public void createTableWithConstraints(Table table, List primaryKeys, List uniqueConstraints, List notNullConstraints) throws TException { + if (catalogName.isPresent()) { + table.setCatName(catalogName.get()); + } client.create_table_with_constraints(table, primaryKeys, emptyList(), uniqueConstraints, notNullConstraints, emptyList(), emptyList()); } @@ -166,28 +189,28 @@ public void createTableWithConstraints(Table table, List primaryK public void dropTable(String databaseName, String name, boolean deleteData) throws TException { - client.drop_table(databaseName, name, deleteData); + client.drop_table(constructSchemaName(catalogName, databaseName), name, deleteData); } @Override public void alterTable(String databaseName, String tableName, Table newTable) throws TException { - client.alter_table(databaseName, tableName, newTable); + client.alter_table(constructSchemaName(catalogName, databaseName), tableName, newTable); } @Override public Table getTable(String databaseName, String tableName) throws TException { - return client.get_table(databaseName, tableName); + return client.get_table(constructSchemaName(catalogName, databaseName), tableName); } @Override public List getFields(String databaseName, String tableName) throws TException { - return client.get_fields(databaseName, tableName); + return client.get_fields(constructSchemaName(catalogName, databaseName), tableName); } @Override @@ -195,6 +218,9 @@ public List getTableColumnStatistics(String databaseName, S throws TException { TableStatsRequest tableStatsRequest = new TableStatsRequest(databaseName, tableName, columnNames); + if (catalogName.isPresent()) { + tableStatsRequest.setCatName(catalogName.get()); + } return client.get_table_statistics_req(tableStatsRequest).getTableStats(); } @@ -203,6 +229,9 @@ public void setTableColumnStatistics(String databaseName, String tableName, List throws TException { ColumnStatisticsDesc statisticsDescription = new ColumnStatisticsDesc(true, databaseName, tableName); + if (catalogName.isPresent()) { + statisticsDescription.setCatName(catalogName.get()); + } ColumnStatistics request = new ColumnStatistics(statisticsDescription, statistics); client.update_table_column_statistics(request); } @@ -211,7 +240,7 @@ public void setTableColumnStatistics(String databaseName, String tableName, List public void deleteTableColumnStatistics(String databaseName, String tableName, String columnName) throws TException { - client.delete_table_column_statistics(databaseName, tableName, columnName); + client.delete_table_column_statistics(constructSchemaName(catalogName, databaseName), tableName, columnName); } @Override @@ -219,6 +248,9 @@ public Map> getPartitionColumnStatistics(Strin throws TException { PartitionsStatsRequest partitionsStatsRequest = new PartitionsStatsRequest(databaseName, tableName, columnNames, partitionNames); + if (catalogName.isPresent()) { + partitionsStatsRequest.setCatName(catalogName.get()); + } return client.get_partitions_statistics_req(partitionsStatsRequest).getPartStats(); } @@ -228,6 +260,9 @@ public void setPartitionColumnStatistics(String databaseName, String tableName, { ColumnStatisticsDesc statisticsDescription = new ColumnStatisticsDesc(false, databaseName, tableName); statisticsDescription.setPartName(partitionName); + if (catalogName.isPresent()) { + statisticsDescription.setCatName(catalogName.get()); + } ColumnStatistics request = new ColumnStatistics(statisticsDescription, statistics); client.update_partition_column_statistics(request); } @@ -236,27 +271,34 @@ public void setPartitionColumnStatistics(String databaseName, String tableName, public void deletePartitionColumnStatistics(String databaseName, String tableName, String partitionName, String columnName) throws TException { - client.delete_partition_column_statistics(databaseName, tableName, partitionName, columnName); + client.delete_partition_column_statistics(constructSchemaName(catalogName, databaseName), tableName, partitionName, columnName); } @Override public List getPartitionNames(String databaseName, String tableName) throws TException { - return client.get_partition_names(databaseName, tableName, (short) -1); + return client.get_partition_names(constructSchemaName(catalogName, databaseName), tableName, (short) -1); } @Override public List getPartitionNamesFiltered(String databaseName, String tableName, List partitionValues) throws TException { - return client.get_partition_names_ps(databaseName, tableName, partitionValues, (short) -1); + return client.get_partition_names_ps(constructSchemaName(catalogName, databaseName), tableName, partitionValues, (short) -1); } @Override public int addPartitions(List newPartitions) throws TException { + // Check if catalog name is present, update each partition in the newPartitions list by setting its catalog name. + if (catalogName.isPresent()) { + String catalog = catalogName.get(); + for (Partition partition : newPartitions) { + partition.setCatName(catalog); + } + } return client.add_partitions(newPartitions); } @@ -264,28 +306,28 @@ public int addPartitions(List newPartitions) public boolean dropPartition(String databaseName, String tableName, List partitionValues, boolean deleteData) throws TException { - return client.drop_partition(databaseName, tableName, partitionValues, deleteData); + return client.drop_partition(constructSchemaName(catalogName, databaseName), tableName, partitionValues, deleteData); } @Override public void alterPartition(String databaseName, String tableName, Partition partition) throws TException { - client.alter_partition(databaseName, tableName, partition); + client.alter_partition(constructSchemaName(catalogName, databaseName), tableName, partition); } @Override public Partition getPartition(String databaseName, String tableName, List partitionValues) throws TException { - return client.get_partition(databaseName, tableName, partitionValues); + return client.get_partition(constructSchemaName(catalogName, databaseName), tableName, partitionValues); } @Override public List getPartitionsByNames(String databaseName, String tableName, List partitionNames) throws TException { - return client.get_partitions_by_names(databaseName, tableName, partitionNames); + return client.get_partitions_by_names(constructSchemaName(catalogName, databaseName), tableName, partitionNames); } @Override @@ -454,7 +496,9 @@ public Optional getPrimaryKey(String dbName, String tableNa throws TException { PrimaryKeysRequest pkRequest = new PrimaryKeysRequest(dbName, tableName); - PrimaryKeysResponse pkResponse; + if (catalogName.isPresent()) { + pkRequest.setCatName(catalogName.get()); + } try { return Optional.of(client.get_primary_keys(pkRequest)); @@ -473,8 +517,10 @@ public Optional getPrimaryKey(String dbName, String tableNa public Optional getUniqueConstraints(String catName, String dbName, String tableName) throws TException { + if (catalogName.isPresent()) { + catName = catalogName.get(); + } UniqueConstraintsRequest uniqueConstraintsRequest = new UniqueConstraintsRequest(catName, dbName, tableName); - UniqueConstraintsResponse uniqueConstraintsResponse; try { return Optional.of(client.get_unique_constraints(uniqueConstraintsRequest)); @@ -491,8 +537,10 @@ public Optional getUniqueConstraints(String catName, public Optional getNotNullConstraints(String catName, String dbName, String tableName) throws TException { + if (catalogName.isPresent()) { + catName = catalogName.get(); + } NotNullConstraintsRequest notNullConstraintsRequest = new NotNullConstraintsRequest(catName, dbName, tableName); - NotNullConstraintsResponse notNullConstraintsResponse; try { return Optional.of(client.get_not_null_constraints(notNullConstraintsRequest)); @@ -510,6 +558,9 @@ public void dropConstraint(String dbName, String tableName, String constraintNam throws TException { DropConstraintRequest dropConstraintRequest = new DropConstraintRequest(dbName, tableName, constraintName); + if (catalogName.isPresent()) { + dropConstraintRequest.setCatName(catalogName.get()); + } client.drop_constraint(dropConstraintRequest); } @@ -517,7 +568,16 @@ public void dropConstraint(String dbName, String tableName, String constraintNam public void addUniqueConstraint(List constraint) throws TException { - AddUniqueConstraintRequest addUniqueConstraintRequest = new AddUniqueConstraintRequest(constraint); + List updatedConstraints = constraint; + if (catalogName.isPresent()) { + updatedConstraints = constraint.stream().map(uniqueConstraint -> { + uniqueConstraint = uniqueConstraint.deepCopy(); + uniqueConstraint.setCatName(catalogName.get()); + return uniqueConstraint; + }) + .collect(toImmutableList()); + } + AddUniqueConstraintRequest addUniqueConstraintRequest = new AddUniqueConstraintRequest(updatedConstraints); client.add_unique_constraint(addUniqueConstraintRequest); } @@ -525,7 +585,16 @@ public void addUniqueConstraint(List constraint) public void addPrimaryKeyConstraint(List constraint) throws TException { - AddPrimaryKeyRequest addPrimaryKeyRequest = new AddPrimaryKeyRequest(constraint); + List updatedConstraints = constraint; + if (catalogName.isPresent()) { + updatedConstraints = constraint.stream().map(primaryKeyConstraint -> { + primaryKeyConstraint = primaryKeyConstraint.deepCopy(); + primaryKeyConstraint.setCatName(catalogName.get()); + return primaryKeyConstraint; + }) + .collect(toImmutableList()); + } + AddPrimaryKeyRequest addPrimaryKeyRequest = new AddPrimaryKeyRequest(updatedConstraints); client.add_primary_key(addPrimaryKeyRequest); } @@ -533,7 +602,16 @@ public void addPrimaryKeyConstraint(List constraint) public void addNotNullConstraint(List constraint) throws TException { - AddNotNullConstraintRequest addNotNullConstraintRequest = new AddNotNullConstraintRequest(constraint); + List updatedConstraints = constraint; + if (catalogName.isPresent()) { + updatedConstraints = constraint.stream().map(notNullConstraint -> { + notNullConstraint = notNullConstraint.deepCopy(); + notNullConstraint.setCatName(catalogName.get()); + return notNullConstraint; + }) + .collect(toImmutableList()); + } + AddNotNullConstraintRequest addNotNullConstraintRequest = new AddNotNullConstraintRequest(updatedConstraints); client.add_not_null_constraint(addNotNullConstraintRequest); } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java index 9b152bf2d6596..543c1a2ba27da 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java @@ -19,6 +19,7 @@ public class ThriftHiveMetastoreStats { private final HiveMetastoreApiStats getAllDatabases = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getDatabases = new HiveMetastoreApiStats(); private final HiveMetastoreApiStats getDatabase = new HiveMetastoreApiStats(); private final HiveMetastoreApiStats getAllTables = new HiveMetastoreApiStats(); private final HiveMetastoreApiStats getAllViews = new HiveMetastoreApiStats(); @@ -61,6 +62,13 @@ public class ThriftHiveMetastoreStats private final HiveMetastoreApiStats addPrimaryKeyConstraint = new HiveMetastoreApiStats(); private final HiveMetastoreApiStats addNotNullConstraint = new HiveMetastoreApiStats(); + @Managed + @Nested + public HiveMetastoreApiStats getGetDatabases() + { + return getDatabases; + } + @Managed @Nested public HiveMetastoreApiStats getGetAllDatabases() diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java index 40a1afcc6398c..74858baba829c 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java @@ -150,6 +150,7 @@ public static org.apache.hadoop.hive.metastore.api.Database toMetastoreApiDataba result.setOwnerType(toMetastoreApiPrincipalType(database.getOwnerType())); database.getComment().ifPresent(result::setDescription); result.setParameters(database.getParameters()); + database.getCatalogName().ifPresent(result::setCatalogName); return result; } @@ -157,6 +158,7 @@ public static org.apache.hadoop.hive.metastore.api.Table toMetastoreApiTable(Tab { org.apache.hadoop.hive.metastore.api.Table result = new org.apache.hadoop.hive.metastore.api.Table(); + table.getCatalogName().ifPresent(result::setCatName); result.setDbName(table.getDatabaseName()); result.setTableName(table.getTableName()); result.setOwner(table.getOwner()); @@ -371,6 +373,7 @@ public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiParti public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition, ColumnConverter columnConverter) { org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition(); + partition.getCatalogName().ifPresent(result::setCatName); result.setDbName(partition.getDatabaseName()); result.setTableName(partition.getTableName()); result.setValues(partition.getValues()); @@ -506,6 +509,7 @@ public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metasto } Partition.Builder partitionBuilder = Partition.builder() + .setCatalogName(Optional.ofNullable(partition.getCatName())) .setDatabaseName(partition.getDbName()) .setTableName(partition.getTableName()) .setValues(partition.getValues()) diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java index e267316145fa3..d98968c73af62 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java @@ -67,7 +67,8 @@ public class TestRecordingHiveMetastore "owner", USER, Optional.of("comment"), - ImmutableMap.of("param", "value")); + ImmutableMap.of("param", "value"), + Optional.of("catalogName")); private static final Column TABLE_COLUMN = new Column( "column", HiveType.HIVE_INT, @@ -96,6 +97,7 @@ public class TestRecordingHiveMetastore ImmutableMap.of("param", "value2"), ImmutableMap.of()); private static final Table TABLE = new Table( + Optional.of("catalogName"), "database", "table", "owner", diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java index d76118d55e7b9..4ee92ab0e2b47 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java @@ -135,6 +135,16 @@ public String getDelegationToken(String owner, String renewer) return TEST_TOKEN; } + @Override + public List getDatabases(String pattern) + { + accessCount.incrementAndGet(); + if (throwException) { + throw new IllegalStateException(); + } + return ImmutableList.of(TEST_DATABASE); + } + @Override public List getAllTables(String dbName) { diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java index 5ff6e10b0b7c0..196a72018ca72 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java @@ -32,7 +32,7 @@ public class MockHiveMetastoreClientFactory public MockHiveMetastoreClientFactory(Optional socksProxy, Duration timeout, List clients) { - super(Optional.empty(), socksProxy, timeout, new NoHiveMetastoreAuthentication()); + super(Optional.empty(), socksProxy, timeout, new NoHiveMetastoreAuthentication(), null); this.clients = new ArrayList<>(requireNonNull(clients, "clients is null")); } diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java index 74e8e3f2ec0ad..0e755b81b41ff 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive.metastore.thrift; +import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHiveMetastoreAuthentication; import com.google.common.net.HostAndPort; @@ -29,19 +30,21 @@ public class TestingHiveCluster private final MetastoreClientConfig metastoreClientConfig; private final ThriftHiveMetastoreConfig thriftHiveMetastoreConfig; private final HostAndPort address; + private final HiveCommonClientConfig hiveCommonClientConfig; - public TestingHiveCluster(MetastoreClientConfig metastoreClientConfig, ThriftHiveMetastoreConfig thriftHiveMetastoreConfig, String host, int port) + public TestingHiveCluster(MetastoreClientConfig metastoreClientConfig, ThriftHiveMetastoreConfig thriftHiveMetastoreConfig, String host, int port, HiveCommonClientConfig hiveCommonClientConfig) { this.metastoreClientConfig = requireNonNull(metastoreClientConfig, "metastore config is null"); this.thriftHiveMetastoreConfig = requireNonNull(thriftHiveMetastoreConfig, "thrift metastore config is null"); this.address = HostAndPort.fromParts(requireNonNull(host, "host is null"), port); + this.hiveCommonClientConfig = hiveCommonClientConfig; } @Override public HiveMetastoreClient createMetastoreClient(Optional token) throws TException { - return new HiveMetastoreClientFactory(metastoreClientConfig, thriftHiveMetastoreConfig, new NoHiveMetastoreAuthentication()).create(address, token); + return new HiveMetastoreClientFactory(metastoreClientConfig, thriftHiveMetastoreConfig, new NoHiveMetastoreAuthentication(), hiveCommonClientConfig).create(address, token); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 2c96999e3242d..7ab586065f8c0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -492,6 +492,13 @@ public SemiTransactionalHiveMetastore getMetastore() return metastore; } + @Override + public boolean schemaExists(ConnectorSession session, String schemaName) + { + Optional database = metastore.getDatabase(getMetastoreContext(session), schemaName); + return database.isPresent(); + } + @Override public HiveStatisticsProvider getHiveStatisticsProvider() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java index c6ef5d983a65d..786bedc460fa0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java @@ -47,6 +47,7 @@ public Partition buildPartitionObject( param -> extraParametersBuilder.put("user_supplied", param)); return Partition.builder() + .setCatalogName(table.getCatalogName()) .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/SyncPartitionMetadataProcedure.java b/presto-hive/src/main/java/com/facebook/presto/hive/SyncPartitionMetadataProcedure.java index 67f6b59d0f924..3bdbb0f0f7b8a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/SyncPartitionMetadataProcedure.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/SyncPartitionMetadataProcedure.java @@ -260,6 +260,7 @@ private static void dropPartitions( private static Partition buildPartitionObject(ConnectorSession session, Table table, String partitionName) { return Partition.builder() + .setCatalogName(table.getCatalogName()) .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index db870cc59d846..9035d5b5de72d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -984,8 +984,7 @@ protected final void setup(String host, int port, String databaseName, String ti if (proxy != null) { metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy)); } - - HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, host, port); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, host, port, new HiveCommonClientConfig()); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of(), hiveClientConfig); hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); ExtendedHiveMetastore metastore = new InMemoryCachingHiveMetastore( @@ -3916,6 +3915,7 @@ protected Partition createDummyPartition(Table table, String partitionName, Opti { byte[] rowIdPartitionComponent = {98, 45}; return Partition.builder() + .setCatalogName(table.getCatalogName()) .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java index 9f3c1eec79977..e27b042b04a66 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java @@ -188,7 +188,7 @@ protected void setup(String host, int port, String databaseName, BiFunction tableEncryptionProperties, boolean isPartitioned) { return new Table( + Optional.of("catalogName"), "dbName", "tableName", "owner", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java index ec67d51d49b27..3af29f6740729 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java @@ -220,6 +220,7 @@ private Partition createDummyPartition(Table table, String partitionName, Option .putAll(dynamicPartitionParameters) .build(); return Partition.builder() + .setCatalogName(table.getCatalogName()) .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCommitHandleOutput.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCommitHandleOutput.java index f8d4c592a124a..ead56db6ac6c1 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCommitHandleOutput.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCommitHandleOutput.java @@ -267,6 +267,7 @@ private HiveMetadata getHiveMetadata(TestingExtendedHiveMetastore metastore, Hiv private Partition createPartition(String partitionName, String partitionLocation) { Partition.Builder partitionBuilder = Partition.builder() + .setCatalogName(Optional.of("hive")) .setDatabaseName(TEST_SCHEMA) .setTableName(TEST_TABLE) .setColumns(ImmutableList.of()) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveEncryptionInformationProvider.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveEncryptionInformationProvider.java index 6547e2f658c0b..083ea6d5b6794 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveEncryptionInformationProvider.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveEncryptionInformationProvider.java @@ -37,6 +37,7 @@ public class TestHiveEncryptionInformationProvider { private static final Table TEST_TABLE = new Table( + Optional.of("catalogName"), "test_db", "test_table", "test_owner", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java index 3761e8098049c..c1d9c11660993 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java @@ -930,6 +930,7 @@ public void testMetadataAggregationFoldingWithFilters(boolean pushdownSubfieldsE private static Partition createDummyPartition(Table table, String partitionName) { return Partition.builder() + .setCatalogName(Optional.of("hive")) .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewUtils.java index 37a9a0455f94b..85a6c471bb373 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewUtils.java @@ -623,6 +623,7 @@ private static Table getTable(List partitionColumns) private static Table getTable(String tableName, List partitionColumns) { return new Table( + Optional.of("catalogName"), SCHEMA_NAME, tableName, USER_NAME, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadata.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadata.java index bf486a436dcbf..046d925b39f6c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadata.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadata.java @@ -84,6 +84,7 @@ public void testColumnMetadataGetter() Optional.empty()); Column partitionColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty()); Table mockTable = new Table( + Optional.of("catalogName"), "schema", "table", "user", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java index db0986ab5531f..c63f5e9d36edb 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java @@ -59,6 +59,7 @@ public class TestHivePartitionManager private static final Column PARTITION_COLUMN = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty()); private static final Column BUCKET_COLUMN = new Column("c1", HIVE_INT, Optional.empty(), Optional.empty()); private static final Table TABLE = new Table( + Optional.of("catalogName"), SCHEMA_NAME, TABLE_NAME, USER_NAME, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveQueriesWithCatalogName.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveQueriesWithCatalogName.java new file mode 100644 index 0000000000000..fd5170abd6e4a --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveQueriesWithCatalogName.java @@ -0,0 +1,244 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hive.containers.HiveMinIODataLake; +import com.facebook.presto.hive.s3.S3HiveQueryRunner; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; +import static com.facebook.presto.tests.sql.TestTable.randomTableSuffix; +import static java.lang.String.format; +import static java.util.stream.Collectors.joining; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestHiveQueriesWithCatalogName + extends AbstractTestQueryFramework +{ + private static final String HIVE_TEST_SCHEMA_1 = "hive_test_schema_1"; + private static final String HIVE_CATALOG = "hive"; + private String bucketName; + private HiveMinIODataLake dockerizedS3DataLake; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + this.bucketName = "test-schema-with-hive-catalog-name-" + randomTableSuffix(); + this.dockerizedS3DataLake = new HiveMinIODataLake(bucketName, ImmutableMap.of()); + this.dockerizedS3DataLake.start(); + return S3HiveQueryRunner.create( + this.dockerizedS3DataLake.getHiveHadoop().getHiveMetastoreEndpoint(), + this.dockerizedS3DataLake.getMinio().getMinioApiEndpoint(), + HiveMinIODataLake.ACCESS_KEY, + HiveMinIODataLake.SECRET_KEY, + ImmutableMap.builder() + // This is required when using MinIO which requires path style access + .put("hive.s3.path-style-access", "true") + .put("hive.insert-existing-partitions-behavior", "OVERWRITE") + .put("hive.non-managed-table-writes-enabled", "true") + // This new conf is added to pass the catalog information to metastore + .put("hive.metastore.catalog.name", HIVE_CATALOG) + .build(), new HashMap<>()); + } + + @BeforeClass + public void setUp() + { + computeActual(format( + "CREATE SCHEMA hive.%1$s WITH (location='s3a://%2$s/%1$s')", + HIVE_TEST_SCHEMA_1, + bucketName)); + } + + @AfterClass(alwaysRun = true) + public void close() + throws Exception + { + closeAllRuntimeException(dockerizedS3DataLake); + } + + @Test + public void testInsertOverwritePartitionedTable() + { + String testTable = getTestTableName(); + computeActual(getCreateTableStatement( + testTable, + "partitioned_by=ARRAY['regionkey']")); + copyTpchNationToTable(testTable); + assertOverwritePartition(testTable); + } + + @Test + public void testInsertOverwritePartitionedAndBucketedTable() + { + String testTable = getTestTableName(); + computeActual(getCreateTableStatement( + testTable, + "partitioned_by=ARRAY['regionkey']", + "bucketed_by = ARRAY['nationkey']", + "bucket_count = 3")); + copyTpchNationToTable(testTable); + assertOverwritePartition(testTable); + } + + @Test + public void testInsertOverwritePartitionedAndBucketedExternalTable() + { + String testTable = getTestTableName(); + // Store table data in data lake bucket + computeActual(getCreateTableStatement( + testTable, + "partitioned_by=ARRAY['regionkey']", + "bucketed_by = ARRAY['nationkey']", + "bucket_count = 3")); + copyTpchNationToTable(testTable); + + String tableName = testTable.substring(testTable.lastIndexOf('.') + 1); + // Map this table as external table + String externalTableName = testTable + "_ext"; + computeActual(getCreateTableStatement( + externalTableName, + "partitioned_by=ARRAY['regionkey']", + "bucketed_by = ARRAY['nationkey']", + "bucket_count = 3", + format("external_location = 's3a://%s/%s/%s/'", this.bucketName, HIVE_TEST_SCHEMA_1, tableName))); + copyTpchNationToTable(testTable); + assertOverwritePartition(externalTableName); + } + + protected void assertOverwritePartition(String testTable) + { + computeActual(format( + "INSERT INTO %s VALUES " + + "('POLAND', 'Test Data', 25, 5), " + + "('CZECH', 'Test Data', 26, 5)", + testTable)); + + String oldPartitionPath = getPartitionPath(testTable); + assertQuery(format("SELECT count(*) FROM %s WHERE regionkey = 5", testTable), "SELECT 2"); + + computeActual(format("INSERT INTO %s values('POLAND', 'Overwrite', 25, 5)", testTable)); + + String newPartitionPath = getPartitionPath(testTable); + assertQuery(format("SELECT count(*) FROM %s WHERE regionkey = 5", testTable), "SELECT 1"); + + assertEquals(oldPartitionPath, newPartitionPath); + computeActual(format("DROP TABLE %s", testTable)); + } + + private String getPartitionPath(String testTable) + { + MaterializedResult result = computeActual(format("SELECT \"$PATH\" from %s where regionkey = 5", testTable)); + assertTrue(result.getMaterializedRows().size() > 0); + String path = result.getMaterializedRows().get(0).getField(0).toString(); + return path.substring(0, path.lastIndexOf("/")); + } + + protected String getTestTableName() + { + return format("hive.%s.%s", HIVE_TEST_SCHEMA_1, "nation_" + randomTableSuffix()); + } + + protected String getCreateTableStatement(String tableName, String... propertiesEntries) + { + return getCreateTableStatement(tableName, Arrays.asList(propertiesEntries)); + } + + protected String getCreateTableStatement(String tableName, List propertiesEntries) + { + return format( + "CREATE TABLE %s (" + + " name varchar(25), " + + " comment varchar(152), " + + " nationkey bigint, " + + " regionkey bigint) " + + (propertiesEntries.isEmpty() ? "" : propertiesEntries + .stream() + .collect(joining(",", "WITH (", ")"))), + tableName); + } + + protected void copyTpchNationToTable(String testTable) + { + computeActual(format("INSERT INTO " + testTable + " SELECT name, comment, nationkey, regionkey FROM tpch.tiny.nation")); + } + + @Test + public void testListSchemasAndListTablesAcrossCatalogs() + { + String hiveTestSchema01 = "hive_test_schema_01"; + String hiveTestSchema02 = "hive_test_schema_02"; + // Create two schemas in different locations + computeActual(format( + "CREATE SCHEMA hive.%1$s WITH (location='s3a://%2$s/%1$s')", + hiveTestSchema01, + bucketName)); + + computeActual(format( + "CREATE SCHEMA hive.%1$s WITH (location='s3a://%2$s/%1$s')", + hiveTestSchema02, + bucketName)); + + // Create tables in both schemas + String testTableSchema1 = format("hive.%s.%s", hiveTestSchema01, "nation_" + randomTableSuffix()); + String testTableSchema2 = format("hive.%s.%s", hiveTestSchema02, "region_" + randomTableSuffix()); + + computeActual(getCreateTableStatement(testTableSchema1)); + computeActual(getCreateTableStatement(testTableSchema2)); + + // Verify that listSchemas contains both schemas + MaterializedResult schemas = computeActual("SHOW SCHEMAS FROM hive"); + List schemaNames = schemas.getMaterializedRows().stream() + .map(row -> row.getField(0).toString()) + .collect(Collectors.toList()); + + assertTrue(schemaNames.contains(hiveTestSchema01), "Schema 1 is missing"); + assertTrue(schemaNames.contains(hiveTestSchema02), "Schema 2 is missing"); + + // Verify that each table is listed under its own schema + MaterializedResult tablesSchema1 = computeActual(format("SHOW TABLES FROM hive.%s", hiveTestSchema01)); + MaterializedResult tablesSchema2 = computeActual(format("SHOW TABLES FROM hive.%s", hiveTestSchema02)); + + List tableNamesSchema1 = tablesSchema1.getMaterializedRows().stream() + .map(row -> row.getField(0).toString()) + .collect(Collectors.toList()); + + List tableNamesSchema2 = tablesSchema2.getMaterializedRows().stream() + .map(row -> row.getField(0).toString()) + .collect(Collectors.toList()); + + assertTrue(tableNamesSchema1.contains(testTableSchema1.substring(testTableSchema1.lastIndexOf('.') + 1)), + "Table in Schema 1 is missing"); + + assertTrue(tableNamesSchema2.contains(testTableSchema2.substring(testTableSchema2.lastIndexOf('.') + 1)), + "Table in Schema 2 is missing"); + + // Cleanup tables + computeActual(format("DROP TABLE %s", testTableSchema1)); + computeActual(format("DROP TABLE %s", testTableSchema2)); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitManager.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitManager.java index 9be421b8d3632..9d1ca4454779f 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitManager.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitManager.java @@ -147,6 +147,7 @@ public class TestHiveSplitManager private static final Table TEST_TABLE = createTestTable(VIEW_STORAGE_FORMAT, ImmutableMap.of()); private ListeningExecutorService executor; + private static final String TEST_CATALOG_NAME = "catalogName"; @BeforeClass public void setUp() @@ -162,7 +163,8 @@ public void shutdown() private static Table createTestTable(StorageFormat storageFormat, Map parameters) { - return new Table("test_db", + return new Table(Optional.of(TEST_CATALOG_NAME), + "test_db", "test_table", "test_owner", MANAGED_TABLE, @@ -472,8 +474,7 @@ private void assertRedundantColumnDomains(Range predicateRange, PartitionStatist // Prepare partition with stats PartitionWithStatistics partitionWithStatistics = new PartitionWithStatistics( - new Partition( - "test_db", + new Partition("test_db", "test_table", ImmutableList.of(PARTITION_VALUE), new Storage( diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHudiDirectoryLister.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHudiDirectoryLister.java index 3426f5bfb23a6..b3cd8801f5c85 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHudiDirectoryLister.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHudiDirectoryLister.java @@ -91,6 +91,7 @@ private Configuration getHadoopConfWithCopyOnFirstWriteEnabled() private Table getMockTable() { return new Table( + Optional.of("catalogName"), "schema", "hudi_non_part_cow", "user", @@ -116,6 +117,7 @@ private Table getMockTable() private Table getMockMORTableWithPartition() { return new Table( + Optional.empty(), "schema", "hudi_mor_part_update", "user", @@ -288,6 +290,7 @@ public void testDirectoryListerForHudiTableWithCopyOnFirstWriteEnabled() public void testDirectoryListerForNonHudiTable() { Table mockTable = new Table( + Optional.of("catalogName"), "schema", "non_hudi_table", "user", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingExtendedHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingExtendedHiveMetastore.java index ad0c3e6848608..c962bd843e474 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestingExtendedHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingExtendedHiveMetastore.java @@ -52,7 +52,7 @@ public List getAllDatabases(MetastoreContext metastoreContext) @Override public Optional getDatabase(MetastoreContext metastoreContext, String databaseName) { - return Optional.of(new Database(databaseName, Optional.of("/"), "test_owner", PrincipalType.USER, Optional.empty(), ImmutableMap.of())); + return Optional.of(new Database(databaseName, Optional.of("/"), "test_owner", PrincipalType.USER, Optional.empty(), ImmutableMap.of(), Optional.of("testcatalog"))); } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java index 81d93de579b8e..e60cb67dfacec 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java @@ -81,7 +81,7 @@ public static TestingSemiTransactionalHiveMetastore create() ThriftHiveMetastoreConfig thriftHiveMetastoreConfig = new ThriftHiveMetastoreConfig(); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreClientConfig), ImmutableSet.of(), config); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); - HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, HOST, PORT); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, HOST, PORT, new HiveCommonClientConfig()); ColumnConverterProvider columnConverterProvider = HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER; ExtendedHiveMetastore delegate = new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig, hdfsEnvironment), new HivePartitionMutator()); ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s")); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/hudi/HudiTestingDataGenerator.java b/presto-hive/src/test/java/com/facebook/presto/hive/hudi/HudiTestingDataGenerator.java index 512125b8e2ea6..02f702c14cafd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/hudi/HudiTestingDataGenerator.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/hudi/HudiTestingDataGenerator.java @@ -173,6 +173,7 @@ private void addPartition(HoodieTableType type, String tableName, List p List partitions = new ArrayList<>(); for (String partitionName : partitionNames) { Partition partition = Partition.builder() + .setCatalogName(Optional.of("hive")) .setDatabaseName(schemaName) .setTableName(tableName) .setValues(extractPartitionValues(partitionName)) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java index c9ce1106caa82..be540ec76279a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java @@ -126,6 +126,7 @@ public static com.facebook.presto.hive.metastore.Table getPrestoTestTable(String public static com.facebook.presto.hive.metastore.Partition getPrestoTestPartition(String dbName, String tblName, List values) { return com.facebook.presto.hive.metastore.Partition.builder() + .setCatalogName(Optional.of("hive")) .setDatabaseName(dbName) .setTableName(tblName) .setValues(values) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/s3/S3HiveQueryRunner.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3/S3HiveQueryRunner.java index 568aece12130c..9f275f58bf9e4 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/s3/S3HiveQueryRunner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3/S3HiveQueryRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive.s3; +import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.HiveQueryRunner; import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.metastore.HivePartitionMutator; @@ -73,7 +74,7 @@ public static DistributedQueryRunner create( new TestingHiveCluster(metastoreClientConfig, thriftHiveMetastoreConfig, hiveEndpoint.getHost(), - hiveEndpoint.getPort()), metastoreClientConfig, + hiveEndpoint.getPort(), new HiveCommonClientConfig()), metastoreClientConfig, HDFS_ENVIRONMENT), new HivePartitionMutator())), ImmutableMap.of()); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java index 7d2570f1a7817..721ed29fc21e4 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java @@ -77,6 +77,7 @@ public void setUp() partition = new Partition("db", "table", emptyList(), storage, singletonList(column), emptyMap(), Optional.empty(), false, false, 1234, 4567L, Optional.empty()); table = new Table( + Optional.of("catalogName"), "db", "table", "owner", @@ -132,6 +133,7 @@ public void testShouldNotEnableSelectPushdownWhenIsNotSupportedSerde() .setLocation("location") .build(); Table newTable = new Table( + Optional.of("catalogName"), "db", "table", "owner", @@ -157,6 +159,7 @@ public void testShouldNotEnableSelectPushdownWhenInputFormatIsNotSupported() .setLocation("location") .build(); Table newTable = new Table( + Optional.of("catalogName"), "db", "table", "owner", @@ -175,6 +178,7 @@ public void testShouldNotEnableSelectPushdownWhenColumnTypesAreNotSupported() { Column newColumn = new Column("column", HIVE_BINARY, Optional.empty(), Optional.empty()); Table newTable = new Table( + Optional.of("catalogName"), "db", "table", "owner", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestParquetQuickStatsBuilder.java b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestParquetQuickStatsBuilder.java index 2d88a15e49bc7..c8cb678cfbcd1 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestParquetQuickStatsBuilder.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestParquetQuickStatsBuilder.java @@ -191,6 +191,7 @@ private ImmutableList buildHiveFileInfos(String basePath, String p private void setUp() { Table table = new Table( + Optional.of("catalogName"), TEST_SCHEMA, TEST_TABLE, "owner", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestQuickStatsProvider.java b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestQuickStatsProvider.java index 31166b5142ce4..b22d980cb042f 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestQuickStatsProvider.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestQuickStatsProvider.java @@ -148,6 +148,7 @@ public void setUp() ImmutableMap.of(), ImmutableMap.of()); Partition mockPartition = new Partition( + Optional.of("catalogName"), TEST_SCHEMA, TEST_TABLE, ImmutableList.of(), @@ -161,6 +162,7 @@ public void setUp() 0, Optional.empty()); Table mockTable = new Table( + Optional.of("catalogName"), TEST_SCHEMA, TEST_TABLE, "owner", diff --git a/presto-hudi/src/test/java/com/facebook/presto/hudi/TestHudiPartitionManager.java b/presto-hudi/src/test/java/com/facebook/presto/hudi/TestHudiPartitionManager.java index afe713bc21dba..ba445bc1d2054 100644 --- a/presto-hudi/src/test/java/com/facebook/presto/hudi/TestHudiPartitionManager.java +++ b/presto-hudi/src/test/java/com/facebook/presto/hudi/TestHudiPartitionManager.java @@ -56,6 +56,7 @@ public class TestHudiPartitionManager private static final Column PARTITION_COLUMN = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty()); private static final Column BUCKET_COLUMN = new Column("c1", HIVE_INT, Optional.empty(), Optional.empty()); private static final Table TABLE = new Table( + Optional.of("catalogName"), SCHEMA_NAME, TABLE_NAME, USER_NAME, diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java index 1dcbc670b8957..9b237a0ee6440 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java @@ -194,6 +194,13 @@ public ManifestFileCache getManifestFileCache() return manifestFileCache; } + @Override + public boolean schemaExists(ConnectorSession session, String schemaName) + { + Optional database = metastore.getDatabase(getMetastoreContext(session), schemaName); + return database.isPresent(); + } + @Override protected org.apache.iceberg.Table getRawIcebergTable(ConnectorSession session, SchemaTableName schemaTableName) { diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 8e72dc28e4f3c..acd0244e054f5 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -667,6 +667,7 @@ public static void setupJsonFunctionNamespaceManager(QueryRunner queryRunner, St private static Table createHiveSymlinkTable(String databaseName, String tableName, List columns, String location) { return new Table( + Optional.of("catalogName"), databaseName, tableName, "hive",