diff --git a/docs/building/windows-instructions.md b/docs/building/windows-instructions.md index 84874a129..aad141b68 100644 --- a/docs/building/windows-instructions.md +++ b/docs/building/windows-instructions.md @@ -30,10 +30,10 @@ If you already have all the pre-requisites, skip to the [build](windows-instruct 3. Install **[Java 1.8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)** - Select the appropriate version for your operating system e.g., jdk-8u201-windows-x64.exe for Win x64 machine. - Install using the installer and verify you are able to run `java` from your command-line - 4. Install **[Apache Maven 3.6.0+](https://maven.apache.org/download.cgi)** - - Download [Apache Maven 3.6.0](http://mirror.metrocast.net/apache/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.zip) - - Extract to a local directory e.g., `c:\bin\apache-maven-3.6.0\` - - Add Apache Maven to your [PATH environment variable](https://www.java.com/en/download/help/path.xml) e.g., `c:\bin\apache-maven-3.6.0\bin` + 4. Install **[Apache Maven 3.6.3+](https://maven.apache.org/download.cgi)** + - Download [Apache Maven 3.6.3](http://mirror.metrocast.net/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip) + - Extract to a local directory e.g., `c:\bin\apache-maven-3.6.3\` + - Add Apache Maven to your [PATH environment variable](https://www.java.com/en/download/help/path.xml) e.g., `c:\bin\apache-maven-3.6.3\bin` - Verify you are able to run `mvn` from your command-line 5. Install **[Apache Spark 2.3+](https://spark.apache.org/downloads.html)** - Download [Apache Spark 2.3+](https://spark.apache.org/downloads.html) and extract it into a local folder (e.g., `c:\bin\spark-2.3.2-bin-hadoop2.7\`) using [7-zip](https://www.7-zip.org/). diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Constants.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Constants.cs new file mode 100644 index 000000000..969dd85f1 --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Constants.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Spark.Extensions.Hyperspace.E2ETest +{ + /// + /// Constants related to the Hyperspace test suite. + /// + internal class Constants + { + public const string HyperspaceTestContainerName = "Hyperspace Tests"; + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs new file mode 100644 index 000000000..8578c77f0 --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.Spark.E2ETest; +using Xunit; + +namespace Microsoft.Spark.Extensions.Hyperspace.E2ETest +{ + public class HyperspaceFixture + { + public HyperspaceFixture() + { + Environment.SetEnvironmentVariable( + SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs, + "--packages com.microsoft.hyperspace:hyperspace-core_2.11:0.1.0"); + + SparkFixture = new SparkFixture(); + } + + public SparkFixture SparkFixture { get; private set; } + } + + [CollectionDefinition(Constants.HyperspaceTestContainerName)] + public class HyperspaceTestCollection : ICollectionFixture + { + // This class has no code, and is never created. Its purpose is simply + // to be the place to apply [CollectionDefinition] and all the + // ICollectionFixture<> interfaces. + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs new file mode 100644 index 000000000..12e8bca60 --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs @@ -0,0 +1,141 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.Spark.E2ETest.Utils; +using Microsoft.Spark.Extensions.Hyperspace.Index; +using Microsoft.Spark.Sql; +using Microsoft.Spark.UnitTest.TestUtils; +using Xunit; + +namespace Microsoft.Spark.Extensions.Hyperspace.E2ETest +{ + /// + /// Test suite for Hyperspace index management APIs. + /// + [Collection(Constants.HyperspaceTestContainerName)] + public class HyperspaceTests : IDisposable + { + private readonly SparkSession _spark; + private readonly TemporaryDirectory _hyperspaceSystemDirectory; + private readonly Hyperspace _hyperspace; + + // Fields needed for sample DataFrame. + private readonly DataFrame _sampleDataFrame; + private readonly string _sampleIndexName; + private readonly IndexConfig _sampleIndexConfig; + + public HyperspaceTests(HyperspaceFixture fixture) + { + _spark = fixture.SparkFixture.Spark; + _hyperspaceSystemDirectory = new TemporaryDirectory(); + _spark.Conf().Set("spark.hyperspace.system.path", _hyperspaceSystemDirectory.Path); + _hyperspace = new Hyperspace(_spark); + + _sampleDataFrame = _spark.Read() + .Option("header", true) + .Option("delimiter", ";") + .Csv("Resources\\people.csv"); + _sampleIndexName = "sample_dataframe"; + _sampleIndexConfig = new IndexConfig(_sampleIndexName, new[] { "job" }, new[] { "name" }); + _hyperspace.CreateIndex(_sampleDataFrame, _sampleIndexConfig); + } + + /// + /// Clean up the Hyperspace system directory in between tests. + /// + public void Dispose() + { + _hyperspaceSystemDirectory.Dispose(); + } + + /// + /// Test the method signatures for all Hyperspace APIs. + /// + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void TestSignatures() + { + // Indexes API. + Assert.IsType(_hyperspace.Indexes()); + + // Delete and Restore APIs. + _hyperspace.DeleteIndex(_sampleIndexName); + _hyperspace.RestoreIndex(_sampleIndexName); + + // Refresh API. + _hyperspace.RefreshIndex(_sampleIndexName); + + // Cancel API. + Assert.Throws(() => _hyperspace.Cancel(_sampleIndexName)); + + // Explain API. + _hyperspace.Explain(_sampleDataFrame, true); + _hyperspace.Explain(_sampleDataFrame, true, s => Console.WriteLine(s)); + + // Delete and Vacuum APIs. + _hyperspace.DeleteIndex(_sampleIndexName); + _hyperspace.VacuumIndex(_sampleIndexName); + + // Enable and disable Hyperspace. + Assert.IsType(_spark.EnableHyperspace()); + Assert.IsType(_spark.DisableHyperspace()); + Assert.IsType(_spark.IsHyperspaceEnabled()); + } + + /// + /// Test E2E functionality of index CRUD APIs. + /// + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void TestIndexCreateAndDelete() + { + // Should be one active index. + DataFrame indexes = _hyperspace.Indexes(); + Assert.Equal(1, indexes.Count()); + Assert.Equal(_sampleIndexName, indexes.SelectExpr("name").First()[0]); + Assert.Equal(States.Active, indexes.SelectExpr("state").First()[0]); + + // Delete the index then verify it has been deleted. + _hyperspace.DeleteIndex(_sampleIndexName); + indexes = _hyperspace.Indexes(); + Assert.Equal(1, indexes.Count()); + Assert.Equal(States.Deleted, indexes.SelectExpr("state").First()[0]); + + // Restore the index to active state and verify it is back. + _hyperspace.RestoreIndex(_sampleIndexName); + indexes = _hyperspace.Indexes(); + Assert.Equal(1, indexes.Count()); + Assert.Equal(States.Active, indexes.SelectExpr("state").First()[0]); + + // Delete and vacuum the index, then verify it is gone. + _hyperspace.DeleteIndex(_sampleIndexName); + _hyperspace.VacuumIndex(_sampleIndexName); + Assert.Equal(0, _hyperspace.Indexes().Count()); + } + + /// + /// Test that the explain API generates the expected string. + /// + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void TestExplainAPI() + { + // Run a query that hits the index. + DataFrame queryDataFrame = _sampleDataFrame + .Where("job == 'Developer'") + .Select("name"); + + string explainString = string.Empty; + _hyperspace.Explain(queryDataFrame, true, s => explainString = s); + Assert.False(string.IsNullOrEmpty(explainString)); + } + + /// + /// Index states used in testing. + /// + private static class States + { + public const string Active = "ACTIVE"; + public const string Deleted = "DELETED"; + } + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Index/IndexConfigTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Index/IndexConfigTests.cs new file mode 100644 index 000000000..b96f85432 --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Index/IndexConfigTests.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Spark.E2ETest.Utils; +using Microsoft.Spark.Extensions.Hyperspace.Index; +using Xunit; + +namespace Microsoft.Spark.Extensions.Hyperspace.E2ETest.Index +{ + /// + /// Test suite for Hyperspace IndexConfig tests. + /// + [Collection(Constants.HyperspaceTestContainerName)] + public class IndexConfigTests + { + public IndexConfigTests(HyperspaceFixture fixture) + { + } + + /// + /// Test the method signatures for IndexConfig and IndexConfigBuilder APIs. + /// + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void TestSignatures() + { + string indexName = "testIndexName"; + var indexConfig = new IndexConfig(indexName, new[] { "Id" }, new string[] { }); + Assert.IsType(indexConfig.IndexName); + Assert.IsType>(indexConfig.IndexedColumns); + Assert.IsType>(indexConfig.IncludedColumns); + Assert.IsType(IndexConfig.Builder()); + Assert.IsType(indexConfig.Equals(indexConfig)); + Assert.IsType(indexConfig.GetHashCode()); + Assert.IsType(indexConfig.ToString()); + + Builder builder = IndexConfig.Builder(); + Assert.IsType(builder); + Assert.IsType(builder.IndexName("indexName")); + Assert.IsType(builder.IndexBy("indexed1", "indexed2")); + Assert.IsType(builder.Include("included1")); + Assert.IsType(builder.Create()); + } + + /// + /// Test creating an IndexConfig using its class constructor. + /// + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void TestIndexConfigConstructor() + { + string indexName = "indexName"; + string[] indexedColumns = { "idx1" }; + string[] includedColumns = { "inc1", "inc2", "inc3" }; + var config = new IndexConfig(indexName, indexedColumns, includedColumns); + + // Validate that the config was built correctly. + Assert.Equal(indexName, config.IndexName); + Assert.Equal(indexedColumns, config.IndexedColumns); + Assert.Equal(includedColumns, config.IncludedColumns); + } + + /// + /// Test creating an IndexConfig using the builder pattern. + /// + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void TestIndexConfigBuilder() + { + string indexName = "indexName"; + string[] indexedColumns = { "idx1" }; + string[] includedColumns = { "inc1", "inc2", "inc3" }; + + Builder builder = IndexConfig.Builder(); + builder.IndexName(indexName); + builder.Include(includedColumns[0], includedColumns[1], includedColumns[2]); + builder.IndexBy(indexedColumns[0]); + + // Validate that the config was built correctly. + IndexConfig config = builder.Create(); + Assert.Equal(indexName, config.IndexName); + Assert.Equal(indexedColumns, config.IndexedColumns); + Assert.Equal(includedColumns, config.IncludedColumns); + } + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Microsoft.Spark.Extensions.Hyperspace.E2ETest.csproj b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Microsoft.Spark.Extensions.Hyperspace.E2ETest.csproj new file mode 100644 index 000000000..231022e4b --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/Microsoft.Spark.Extensions.Hyperspace.E2ETest.csproj @@ -0,0 +1,13 @@ + + + + netcoreapp3.1 + false + + + + + + + + diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Hyperspace.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Hyperspace.cs new file mode 100644 index 000000000..13509779d --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Hyperspace.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.Spark.Extensions.Hyperspace.Index; +using Microsoft.Spark.Interop.Ipc; +using Microsoft.Spark.Sql; + +namespace Microsoft.Spark.Extensions.Hyperspace +{ + /// + /// .Net for Spark binding for Hyperspace index management APIs. + /// + public class Hyperspace : IJvmObjectReferenceProvider + { + private static readonly string s_hyperspaceClassName = + "com.microsoft.hyperspace.Hyperspace"; + private readonly SparkSession _spark; + private readonly IJvmBridge _jvmBridge; + private readonly JvmObjectReference _jvmObject; + + public Hyperspace(SparkSession spark) + { + _spark = spark; + _jvmBridge = ((IJvmObjectReferenceProvider)spark).Reference.Jvm; + _jvmObject = _jvmBridge.CallConstructor(s_hyperspaceClassName, spark); + } + + JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; + + /// + /// Collect all the index metadata. + /// + /// All index metadata as a . + public DataFrame Indexes() => + new DataFrame((JvmObjectReference)_jvmObject.Invoke("indexes")); + + /// + /// Create index. + /// + /// The DataFrame object to build index on. + /// The configuration of index to be created. + public void CreateIndex(DataFrame df, IndexConfig indexConfig) => + _jvmObject.Invoke("createIndex", df, indexConfig); + + /// + /// Soft deletes the index with given index name. + /// + /// The name of index to delete. + public void DeleteIndex(string indexName) => _jvmObject.Invoke("deleteIndex", indexName); + + /// + /// Restores index with given index name. + /// + /// Name of the index to restore. + public void RestoreIndex(string indexName) => _jvmObject.Invoke("restoreIndex", indexName); + + /// + /// Does hard delete of indexes marked as DELETED. + /// + /// Name of the index to restore. + public void VacuumIndex(string indexName) => _jvmObject.Invoke("vacuumIndex", indexName); + + /// + /// Update indexes for the latest version of the data. + /// + /// Name of the index to refresh. + public void RefreshIndex(string indexName) => _jvmObject.Invoke("refreshIndex", indexName); + + /// + /// Cancel api to bring back index from an inconsistent state to the last known stable + /// state. + /// + /// E.g. if index fails during creation, in CREATING state. + /// The index will not allow any index modifying operations unless a cancel is called. + /// + /// Note: Cancel from VACUUMING state will move it forward to DOESNOTEXIST + /// state. + /// + /// Note: If no previous stable state exists, cancel will move it to DOESNOTEXIST + /// state. + /// + /// Name of the index to cancel. + public void Cancel(string indexName) => _jvmObject.Invoke("cancel", indexName); + + /// + /// Explains how indexes will be applied to the given dataframe. + /// + /// dataFrame + /// Flag to enable verbose mode. + public void Explain(DataFrame df, bool verbose) => + Explain(df, verbose, s => Console.WriteLine(s)); + + /// + /// Explains how indexes will be applied to the given dataframe. + /// + /// dataFrame + /// Flag to enable verbose mode. + /// Function to redirect output of explain. + public void Explain(DataFrame df, bool verbose, Action redirectFunc) + { + var explainString = (string)_jvmBridge.CallStaticJavaMethod( + "com.microsoft.hyperspace.index.plananalysis.PlanAnalyzer", + "explainString", + df, + _spark, + Indexes(), + verbose); + redirectFunc(explainString); + } + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/HyperspaceSparkSessionExtensions.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/HyperspaceSparkSessionExtensions.cs new file mode 100644 index 000000000..3c43f369c --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/HyperspaceSparkSessionExtensions.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Spark.Interop; +using Microsoft.Spark.Interop.Ipc; +using Microsoft.Spark.Sql; + +namespace Microsoft.Spark.Extensions.Hyperspace +{ + /// + /// Hyperspace-specific extension methods on . + /// + public static class HyperspaceSparkSessionExtensions + { + private static readonly string s_pythonUtilsClassName = + "com.microsoft.hyperspace.util.PythonUtils"; + + /// + /// Plug in Hyperspace-specific rules. + /// + /// A spark session that does not contain Hyperspace-specific rules. + /// + /// A spark session that contains Hyperspace-specific rules. + public static SparkSession EnableHyperspace(this SparkSession session) => + new SparkSession( + (JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_pythonUtilsClassName, + "enableHyperspace", + session)); + + /// + /// Plug out Hyperspace-specific rules. + /// + /// A spark session that contains Hyperspace-specific rules. + /// A spark session that does not contain Hyperspace-specific rules. + public static SparkSession DisableHyperspace(this SparkSession session) => + new SparkSession( + (JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_pythonUtilsClassName, + "disableHyperspace", + session)); + + /// + /// Checks if Hyperspace is enabled or not. + /// + /// + /// True if Hyperspace is enabled or false otherwise. + public static bool IsHyperspaceEnabled(this SparkSession session) => + (bool)SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_pythonUtilsClassName, + "isHyperspaceEnabled", + session); + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Index/Builder.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Index/Builder.cs new file mode 100644 index 000000000..4623de3e7 --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Index/Builder.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Spark.Interop.Ipc; + +namespace Microsoft.Spark.Extensions.Hyperspace.Index +{ + /// + /// Builder for . + /// + public sealed class Builder : IJvmObjectReferenceProvider + { + private readonly JvmObjectReference _jvmObject; + + internal Builder(JvmObjectReference jvmObject) + { + _jvmObject = jvmObject; + } + + JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; + + /// + /// Updates index name for . + /// + /// Index name for the . + /// An object with updated indexname. + public Builder IndexName(string indexName) + { + _jvmObject.Invoke("indexName", indexName); + return this; + } + + /// + /// Updates column names for . + /// + /// Note: API signature supports passing one or more argument. + /// + /// Indexed column for the + /// . + /// Indexed columns for the + /// . + /// An object with updated indexed columns. + public Builder IndexBy(string indexedColumn, params string[] indexedColumns) + { + _jvmObject.Invoke("indexBy", indexedColumn, indexedColumns); + return this; + } + + /// + /// Updates included columns for . + /// + /// Note: API signature supports passing one or more argument. + /// + /// Included column for . + /// + /// Included columns for . + /// + /// An object with updated included columns. + public Builder Include(string includedColumn, params string[] includedColumns) + { + _jvmObject.Invoke("include", includedColumn, includedColumns); + return this; + } + + /// + /// Creates IndexConfig from supplied index name, indexed columns and included columns + /// to . + /// + /// An object. + public IndexConfig Create() => + new IndexConfig((JvmObjectReference)_jvmObject.Invoke("create")); + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Index/IndexConfig.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Index/IndexConfig.cs new file mode 100644 index 000000000..030dda2ca --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Index/IndexConfig.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.Spark.Interop; +using Microsoft.Spark.Interop.Internal.Scala; +using Microsoft.Spark.Interop.Ipc; + +namespace Microsoft.Spark.Extensions.Hyperspace.Index +{ + /// + /// specifies the configuration of an index. + /// + public sealed class IndexConfig : IJvmObjectReferenceProvider + { + private static readonly string s_className = "com.microsoft.hyperspace.index.IndexConfig"; + private readonly JvmObjectReference _jvmObject; + + /// + /// specifies the configuration of an index. + /// + /// Index name. + /// Columns from which an index is created. + public IndexConfig(string indexName, IEnumerable indexedColumns) + : this(indexName, indexedColumns, new string[] { }) + { + } + + /// + /// specifies the configuration of an index. + /// + /// Index name. + /// Columns from which an index is created. + /// Columns to be included in the index. + public IndexConfig( + string indexName, + IEnumerable indexedColumns, + IEnumerable includedColumns) + { + IndexName = indexName; + IndexedColumns = new List(indexedColumns); + IncludedColumns = new List(includedColumns); + + _jvmObject = (JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_className, + "apply", + IndexName, + IndexedColumns, + IncludedColumns); + } + + /// + /// specifies the configuration of an index. + /// + /// JVM object reference. + internal IndexConfig(JvmObjectReference jvmObject) + { + _jvmObject = jvmObject; + IndexName = (string)_jvmObject.Invoke("indexName"); + IndexedColumns = new List( + new Seq((JvmObjectReference)_jvmObject.Invoke("indexedColumns"))); + IncludedColumns = new List( + new Seq((JvmObjectReference)_jvmObject.Invoke("includedColumns"))); + } + + JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; + + public string IndexName { get; private set; } + + public List IndexedColumns { get; private set; } + + public List IncludedColumns { get; private set; } + + /// + /// Creates new for constructing an + /// . + /// + /// An object. + public static Builder Builder() => + new Builder( + (JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_className, + "builder")); + + public override bool Equals(object that) => (bool)_jvmObject.Invoke("equals", that); + + public override int GetHashCode() => (int)_jvmObject.Invoke("hashCode"); + + public override string ToString() => (string)_jvmObject.Invoke("toString"); + } +} diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Microsoft.Spark.Extensions.Hyperspace.csproj b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Microsoft.Spark.Extensions.Hyperspace.csproj new file mode 100644 index 000000000..d85c62f71 --- /dev/null +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace/Microsoft.Spark.Extensions.Hyperspace.csproj @@ -0,0 +1,13 @@ + + + + netstandard2.0;netstandard2.1 + true + true + + + + + + + diff --git a/src/csharp/Microsoft.Spark.E2ETest/Microsoft.Spark.E2ETest.csproj b/src/csharp/Microsoft.Spark.E2ETest/Microsoft.Spark.E2ETest.csproj index e03519853..7a6240ecc 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/Microsoft.Spark.E2ETest.csproj +++ b/src/csharp/Microsoft.Spark.E2ETest/Microsoft.Spark.E2ETest.csproj @@ -12,6 +12,7 @@ + diff --git a/src/csharp/Microsoft.Spark.sln b/src/csharp/Microsoft.Spark.sln index 73047bff3..75c071377 100644 --- a/src/csharp/Microsoft.Spark.sln +++ b/src/csharp/Microsoft.Spark.sln @@ -39,6 +39,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Spark.Extensions. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest", "Extensions\Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest\Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest.csproj", "{7BDE09ED-04B3-41B2-A466-3D6F7225291E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Spark.Extensions.Hyperspace", "Extensions\Microsoft.Spark.Extensions.Hyperspace\Microsoft.Spark.Extensions.Hyperspace.csproj", "{70DDA4E9-1195-4A29-9AA1-96A8223A6D4F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Spark.Extensions.Hyperspace.E2ETest", "Extensions\Microsoft.Spark.Extensions.Hyperspace.E2ETest\Microsoft.Spark.Extensions.Hyperspace.E2ETest.csproj", "{C6019E44-C777-4DE2-B70E-EA025B7D044D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -101,6 +105,14 @@ Global {7BDE09ED-04B3-41B2-A466-3D6F7225291E}.Debug|Any CPU.Build.0 = Debug|Any CPU {7BDE09ED-04B3-41B2-A466-3D6F7225291E}.Release|Any CPU.ActiveCfg = Release|Any CPU {7BDE09ED-04B3-41B2-A466-3D6F7225291E}.Release|Any CPU.Build.0 = Release|Any CPU + {70DDA4E9-1195-4A29-9AA1-96A8223A6D4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {70DDA4E9-1195-4A29-9AA1-96A8223A6D4F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {70DDA4E9-1195-4A29-9AA1-96A8223A6D4F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {70DDA4E9-1195-4A29-9AA1-96A8223A6D4F}.Release|Any CPU.Build.0 = Release|Any CPU + {C6019E44-C777-4DE2-B70E-EA025B7D044D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C6019E44-C777-4DE2-B70E-EA025B7D044D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C6019E44-C777-4DE2-B70E-EA025B7D044D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C6019E44-C777-4DE2-B70E-EA025B7D044D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -113,6 +125,8 @@ Global {47652C7D-B076-4FD9-98AC-959E38BE18E3} = {71A19F75-8279-40AB-BEA0-7D4B153FC416} {9C32014D-8C0C-40F1-9ABA-C3BF19687E5C} = {71A19F75-8279-40AB-BEA0-7D4B153FC416} {7BDE09ED-04B3-41B2-A466-3D6F7225291E} = {71A19F75-8279-40AB-BEA0-7D4B153FC416} + {70DDA4E9-1195-4A29-9AA1-96A8223A6D4F} = {71A19F75-8279-40AB-BEA0-7D4B153FC416} + {C6019E44-C777-4DE2-B70E-EA025B7D044D} = {71A19F75-8279-40AB-BEA0-7D4B153FC416} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FD15FFDB-EA1B-436F-841D-3386DDF94538} diff --git a/src/csharp/Microsoft.Spark/Interop/Internal/Scala/Seq.cs b/src/csharp/Microsoft.Spark/Interop/Internal/Scala/Seq.cs new file mode 100644 index 000000000..9d9ed3bc1 --- /dev/null +++ b/src/csharp/Microsoft.Spark/Interop/Internal/Scala/Seq.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections; +using System.Collections.Generic; +using Microsoft.Spark.Interop.Ipc; + +namespace Microsoft.Spark.Interop.Internal.Scala +{ + /// + /// Limited read-only implementation of Scala Seq[T] so that Seq objects can be read + /// into POCO collection types such as List. + /// + /// + internal sealed class Seq : IJvmObjectReferenceProvider, IEnumerable + { + private readonly JvmObjectReference _jvmObject; + + internal Seq(JvmObjectReference jvmObject) + { + _jvmObject = jvmObject; + } + + JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public int Size => (int)_jvmObject.Invoke("size"); + + public IEnumerator GetEnumerator() + { + for (int i = 0; i < Size; ++i) + { + yield return Apply(i); + } + } + + public T Apply(int index) => (T)_jvmObject.Invoke("apply", index); + } +} diff --git a/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj b/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj index 050a43493..2cddc5627 100644 --- a/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj +++ b/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj @@ -19,6 +19,8 @@ + +