# This is a combination of 17 commits.

# This is the 1st commit message:

[native] Add a test group for async data cache e2e tests.

# This is the commit message prestodb#2:

[native] Advance velox.

# This is the commit message prestodb#3:

Fix error when describing a nonexistent table

# This is the commit message prestodb#4:

[native] test modification.

# This is the commit message prestodb#5:

Changes to enable ssl/tls in hms
Co-authored-by: Arin Mathew <arin.mathew1@ibm.com>

Changes to move ssl related properties to seperate class

# This is the commit message prestodb#6:

[native] Add tests for UUID type

# This is the commit message prestodb#7:

Reintroduced json_extract to generate canonicalized output (prestodb#24879)

## Description
The original pull request
[prestodb#24614](prestodb#24614) incorrectly
compares canonicalizedJsonExtract and legacyJsonCast in the equals
function of an object. This issue can be seen in the code
[here](https://github.com/prestodb/presto/pull/24614/files#diff-e921c5d186f9d5daa836bc7330f52caf8c1b84d19cf42288d5a8a7c9a6d2a5d5R156).

As a result, whenever a SQL function requires caching, the cache is
never hit, leading to the creation of new SQL function objects
repeatedly. This behavior eventually causes an OOM error in the JVM
metaspace. and eventually this error led to UER SEV.

After the problematic comparison was updated and tested through shadow
cluster by @rschlussel , we are confident that the issue has been
resolved in this PR. Therefore, we plan to bring back the json
canonicalized extract


## Motivation and Context
Reintroduced json_extract to generate canonicalized output

## Impact
<!---Describe any public API or user-facing feature change or any
performance impact-->
low impact 

## Test Plan
<!---Please fill in how you tested your change-->
N/A

## Contributor checklist

- [x] Please make sure your submission complies with our [contributing
guide](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md),
in particular [code
style](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#code-style)
and [commit
standards](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#commit-standards).
- [x] PR description addresses the issue accurately and concisely. If
the change is non-trivial, a GitHub Issue is referenced.
- [x] Documented new properties (with its default value), SQL syntax,
functions, or other functionality.
- [x] If release notes are required, they follow the [release notes
guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines).
- [x] Adequate tests were added if applicable.
- [x] CI passed.

## Release Notes
Please follow [release notes
guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines)
and fill in the release notes below.

```
== NO RELEASE NOTE ==
```
# This is the commit message prestodb#8:

Reuse JoinType in IndexJoinNode

Reuse JoinType instead of creating IndexJoinNode's own. JoinType is
already part of Prestissimo protocol. Adding IndexJoinNode with another
JoinType would cause naming conflict.

# This is the commit message prestodb#9:

Add jaro-winkler implementation, documentation and tests

# This is the commit message prestodb#10:

Add documentation for Iceberg support in PrestoCPP

# This is the commit message prestodb#11:

Add memory pool debug regex

# This is the commit message prestodb#12:

doc on hive csv limitations

# This is the commit message prestodb#13:

Add support for S3 WebIdentity authentication

# This is the commit message prestodb#14:

use com.facebook.airlift:security in presto-hive-metastore

# This is the commit message prestodb#15:

[native] Advance Velox

# This is the commit message prestodb#16:

[native] Add protocol for index lookup join plan

# This is the commit message prestodb#17:

[native] Add sidecar and sidecar plugin documentation

Author: amitkdutta

.github/workflows/hive-tests.yml

@@ -114,6 +114,9 @@ jobs:
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
           ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :presto-hive
-      - name: Run Hive Dockerized Tests
+      - name: Run Hive Insert Overwrite Tests
         if: needs.changes.outputs.codechange == 'true'
         run: ./mvnw test ${MAVEN_TEST} -pl :presto-hive -P test-hive-insert-overwrite
+      - name: Run Hive SSL Enabled Tests
+        if: needs.changes.outputs.codechange == 'true'
+        run: ./mvnw test ${MAVEN_TEST} -pl :presto-hive -P test-ssl-enabled-hms

presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowMetadata.java

@@ -175,7 +175,9 @@ public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSess
         for (SchemaTableName tableName : tables) {
             try {
                 ConnectorTableHandle tableHandle = getTableHandle(session, tableName);
-                columns.put(tableName, getTableMetadata(session, tableHandle).getColumns());
+                if (tableHandle != null) {
+                    columns.put(tableName, getTableMetadata(session, tableHandle).getColumns());
+                }
             }
             catch (ClassCastException | NotFoundException e) {
                 throw new ArrowException(ARROW_FLIGHT_METADATA_ERROR, "Table columns could not be listed for table: " + tableName, e);

presto-base-arrow-flight/src/test/java/com/facebook/plugin/arrow/TestArrowFlightQueries.java

@@ -44,6 +44,7 @@
 import static com.facebook.presto.common.type.VarcharType.VARCHAR;
 import static com.facebook.presto.common.type.VarcharType.createVarcharType;
 import static com.facebook.presto.testing.MaterializedResult.resultBuilder;
+import static com.facebook.presto.testing.assertions.Assert.assertEquals;
 import static java.lang.String.format;
 import static org.testng.Assert.assertTrue;
 
@@ -137,6 +138,16 @@ public void testSelectTime()
         assertTrue(actualRow.equals(expectedRow));
     }
 
+    @Test
+    public void testDescribeUnknownTable()
+    {
+        MaterializedResult actualRows = computeActual("DESCRIBE information_schema.enabled_roles");
+        MaterializedResult expectedRows = resultBuilder(getSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR)
+                .row("role_name", "varchar", "", "")
+                .build();
+        assertEquals(actualRows, expectedRows);
+    }
+
     private LocalDate getDate(String dateString)
     {
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");

presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java

@@ -37,6 +37,7 @@ public class SqlFunctionProperties
     private final boolean legacyJsonCast;
     private final Map<String, String> extraCredentials;
     private final boolean warnOnCommonNanPatterns;
+    private final boolean canonicalizedJsonExtract;
 
     private SqlFunctionProperties(
             boolean parseDecimalLiteralAsDouble,
@@ -50,7 +51,8 @@ private SqlFunctionProperties(
             boolean fieldNamesInJsonCastEnabled,
             boolean legacyJsonCast,
             Map<String, String> extraCredentials,
-            boolean warnOnCommonNanPatterns)
+            boolean warnOnCommonNanPatterns,
+            boolean canonicalizedJsonExtract)
     {
         this.parseDecimalLiteralAsDouble = parseDecimalLiteralAsDouble;
         this.legacyRowFieldOrdinalAccessEnabled = legacyRowFieldOrdinalAccessEnabled;
@@ -64,6 +66,7 @@ private SqlFunctionProperties(
         this.legacyJsonCast = legacyJsonCast;
         this.extraCredentials = requireNonNull(extraCredentials, "extraCredentials is null");
         this.warnOnCommonNanPatterns = warnOnCommonNanPatterns;
+        this.canonicalizedJsonExtract = canonicalizedJsonExtract;
     }
 
     public boolean isParseDecimalLiteralAsDouble()
@@ -127,6 +130,9 @@ public boolean shouldWarnOnCommonNanPatterns()
         return warnOnCommonNanPatterns;
     }
 
+    public boolean isCanonicalizedJsonExtract()
+    { return canonicalizedJsonExtract; }
+
     @Override
     public boolean equals(Object o)
     {
@@ -146,15 +152,16 @@ public boolean equals(Object o)
                 Objects.equals(sessionLocale, that.sessionLocale) &&
                 Objects.equals(sessionUser, that.sessionUser) &&
                 Objects.equals(extraCredentials, that.extraCredentials) &&
-                Objects.equals(legacyJsonCast, that.legacyJsonCast);
+                Objects.equals(legacyJsonCast, that.legacyJsonCast) &&
+                Objects.equals(canonicalizedJsonExtract, that.canonicalizedJsonExtract);
     }
 
     @Override
     public int hashCode()
     {
         return Objects.hash(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey,
                 legacyTimestamp, legacyMapSubscript, sessionStartTime, sessionLocale, sessionUser,
-                extraCredentials, legacyJsonCast);
+                extraCredentials, legacyJsonCast, canonicalizedJsonExtract);
     }
 
     public static Builder builder()
@@ -176,6 +183,7 @@ public static class Builder
         private boolean legacyJsonCast;
         private Map<String, String> extraCredentials = emptyMap();
         private boolean warnOnCommonNanPatterns;
+        private boolean canonicalizedJsonExtract;
 
         private Builder() {}
 
@@ -251,6 +259,12 @@ public Builder setWarnOnCommonNanPatterns(boolean warnOnCommonNanPatterns)
             return this;
         }
 
+        public Builder setCanonicalizedJsonExtract(boolean canonicalizedJsonExtract)
+        {
+            this.canonicalizedJsonExtract = canonicalizedJsonExtract;
+            return this;
+        }
+
         public SqlFunctionProperties build()
         {
             return new SqlFunctionProperties(
@@ -265,7 +279,8 @@ public SqlFunctionProperties build()
                     fieldNamesInJsonCastEnabled,
                     legacyJsonCast,
                     extraCredentials,
-                    warnOnCommonNanPatterns);
+                    warnOnCommonNanPatterns,
+                    canonicalizedJsonExtract);
         }
     }
 }

presto-delta/src/main/java/com/facebook/presto/delta/DeltaModule.java

@@ -44,6 +44,7 @@
 import com.facebook.presto.hive.metastore.InvalidateMetastoreCacheProcedure;
 import com.facebook.presto.hive.metastore.MetastoreCacheStats;
 import com.facebook.presto.hive.metastore.MetastoreConfig;
+import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastoreConfig;
 import com.facebook.presto.spi.procedure.Procedure;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
@@ -104,6 +105,7 @@ protected void setup(Binder binder)
         configBinder(binder).bindConfig(MetastoreConfig.class);
         configBinder(binder).bindConfig(HiveClientConfig.class);
         configBinder(binder).bindConfig(MetastoreClientConfig.class);
+        configBinder(binder).bindConfig(ThriftHiveMetastoreConfig.class);
         binder.bind(MetastoreCacheStats.class).to(HiveMetastoreCacheStats.class).in(Scopes.SINGLETON);
         newExporter(binder).export(MetastoreCacheStats.class).as(generatedNameOf(MetastoreCacheStats.class, connectorId));
         binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON);

presto-docs/src/main/sphinx/connector/hive.rst

@@ -213,9 +213,9 @@ Metastore Configuration Properties
 
 The required Hive metastore can be configured with a number of properties.
 
-======================================================= ============================================================= ============
+======================================================== ============================================================= ============
 Property Name                                                         Description                                       Default
-======================================================= ============================================================= ============
+======================================================== ============================================================= ============
 ``hive.metastore-timeout``                               Timeout for Hive metastore requests.                           ``10s``
 
 ``hive.metastore-cache-ttl``                             Duration how long cached metastore data should be considered   ``0s``
@@ -232,7 +232,17 @@ Property Name                                                         Descriptio
 ``hive.invalidate-metastore-cache-procedure-enabled``    When enabled, users will be able to invalidate metastore        false
                                                          cache on demand.
 
-======================================================= ============================================================= ============
+``hive.metastore.thrift.client.tls.enabled``             Whether TLS security is enabled.                                false
+
+``hive.metastore.thrift.client.tls.keystore-path``       Path to the PEM or JKS key store.                               NONE
+
+``hive.metastore.thrift.client.tls.keystore-password``   Password for the key store.                                     NONE
+
+``hive.metastore.thrift.client.tls.truststore-path``     Path to the PEM or JKS trust store.                             NONE
+
+``hive.metastore.thrift.client.tls.truststore-password`` Password for the trust store.                                   NONE
+
+======================================================== ============================================================= ============
 
 AWS Glue Catalog Configuration Properties
 -----------------------------------------
@@ -361,6 +371,13 @@ Property Name                                Description
 ``hive.s3.skip-glacier-objects``             Ignore Glacier objects rather than failing the query. This
                                              will skip data that may be expected to be part of the table
                                              or partition. Defaults to ``false``.
+
+``hive.s3.web.identity.auth.enabled``        Enables Web Identity authentication for S3 access. Requires
+                                             ``hive.s3.iam-role`` to be specified. Additionally, ensure that
+                                             the environment variables ``AWS_WEB_IDENTITY_TOKEN_FILE`` and
+                                             ``AWS_REGION`` are set for proper authentication. Since this
+                                             implementation uses AWS SDK 1.x, setting these environment
+                                             variables is necessary.
 ============================================ =================================================================
 
 S3 Credentials
@@ -1117,4 +1134,56 @@ Drop a schema::
 Hive Connector Limitations
 --------------------------
 
+SQL DELETE
+^^^^^^^^^^
+
 :doc:`/sql/delete` is only supported if the ``WHERE`` clause matches entire partitions.
+
+CSV Format Type Limitations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When creating tables with CSV format, all columns must be defined as ``VARCHAR`` due to 
+the underlying OpenCSVSerde limitations. `OpenCSVSerde <https://github.com/apache/hive/blob/master/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java>`_ deserializes all CSV columns 
+as strings only. Using any other data type will result in an error similar to the following::
+
+  CREATE TABLE hive.csv.csv_fail ( 
+    id BIGINT, 
+    value INT, 
+    date_col DATE
+  ) with ( format = 'CSV' ) ;
+
+.. code-block:: none
+
+    Query failed: Hive CSV storage format only supports VARCHAR (unbounded). 
+    Unsupported columns: id integer, value integer, date_col date
+
+To work with other data types when using CSV format:
+
+1. Create the table with all the columns as ``VARCHAR``
+2. Create a view or another table that casts the columns to their desired data types
+
+Example::
+
+    -- First create table with VARCHAR columns
+    CREATE TABLE hive.csv.csv_data (
+        id VARCHAR,
+        value VARCHAR,
+        date_col VARCHAR
+    )
+    WITH (format = 'CSV');
+
+    -- Then create a view with the proper data types
+    CREATE VIEW hive.csv.csv_data_view AS
+    SELECT 
+        CAST(id AS BIGINT) AS id,
+        CAST(value AS INT) AS value,
+        CAST(date_col AS DATE) AS date_col
+    FROM hive.csv.csv_data;
+
+    -- OR another table with the proper data types
+    CREATE TABLE hive.csv.csv_data_cast AS
+    SELECT 
+        CAST(id AS BIGINT) AS id,
+        CAST(value AS INT) AS value,
+        CAST(date_col AS DATE) AS date_col
+    FROM hive.csv.csv_data;