dotnet-interactive assembly extension

src/csharp/Extensions/Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest/Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <RootNamespace>Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest</RootNamespace>
+    <IsPackable>false</IsPackable>
+
+    <RestoreAdditionalProjectSources>https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json</RestoreAdditionalProjectSources>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Moq" Version="4.10.0" />
+    <PackageReference Include="Microsoft.DotNet.Interactive" Version="1.0.0-beta.20262.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.Spark.Extensions.DotNet.Interactive\Microsoft.Spark.Extensions.DotNet.Interactive.csproj" />
+    <ProjectReference Include="..\..\Microsoft.Spark\Microsoft.Spark.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\Microsoft.Spark.UnitTest\TestUtils\TemporaryDirectory.cs" />
+  </ItemGroup>
+
+</Project>

src/csharp/Extensions/Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest/PackageResolverTests.cs

@@ -0,0 +1,95 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Microsoft.DotNet.Interactive.Utility;
+using Microsoft.Spark.UnitTest.TestUtils;
+using Microsoft.Spark.Utils;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest
+{
+    public class PackageResolverTests
+    {
+        [Fact]
+        public void TestPackageResolver()
+        {
+            using var tempDir = new TemporaryDirectory();
+
+            string packageName = "package.name";
+            string packageVersion = "0.1.0";
+            string packageRootPath =
+                Path.Combine(tempDir.Path, "path", "to", "packages", packageName, packageVersion);
+            string packageFrameworkPath = Path.Combine(packageRootPath, "lib", "framework");
+
+            Directory.CreateDirectory(packageRootPath);
+            var nugetFile = new FileInfo(
+                Path.Combine(packageRootPath, $"{packageName}.{packageVersion}.nupkg"));
+            using (File.Create(nugetFile.FullName))
+            {
+            }
+
+            var assemblyPaths = new List<FileInfo>
+            {
+                new FileInfo(Path.Combine(packageFrameworkPath, "1.dll")),
+                new FileInfo(Path.Combine(packageFrameworkPath, "2.dll"))
+            };
+            var probingPaths = new List<DirectoryInfo> { new DirectoryInfo(packageRootPath) };
+
+            var mockPackageRestoreContextWrapper = new Mock<PackageRestoreContextWrapper>();
+            mockPackageRestoreContextWrapper
+                .SetupGet(m => m.ResolvedPackageReferences)
+                .Returns(new ResolvedPackageReference[]
+                {
+                    new ResolvedPackageReference(
+                        packageName,
+                        packageVersion,
+                        assemblyPaths,
+                        new DirectoryInfo(packageRootPath),
+                        probingPaths) 
+                });
+
+            var packageResolver = new PackageResolver(mockPackageRestoreContextWrapper.Object);
+            IEnumerable<string> actualFiles = packageResolver.GetFiles(tempDir.Path);
+
+            string metadataFilePath =
+                Path.Combine(tempDir.Path, DependencyProviderUtils.CreateFileName(1));
+            var expectedFiles = new string[]
+            {
+                nugetFile.FullName,
+                metadataFilePath
+            };
+            Assert.True(expectedFiles.SequenceEqual(actualFiles));
+            Assert.True(File.Exists(metadataFilePath));
+
+            DependencyProviderUtils.Metadata actualMetadata =
+                DependencyProviderUtils.Metadata.Deserialize(metadataFilePath);
+            var expectedMetadata = new DependencyProviderUtils.Metadata
+            {
+                AssemblyProbingPaths = new string[]
+                {
+                    Path.Combine(packageName, packageVersion, "lib", "framework", "1.dll"),
+                    Path.Combine(packageName, packageVersion, "lib", "framework", "2.dll")
+                },
+                NativeProbingPaths = new string[]
+                {
+                    Path.Combine(packageName, packageVersion)
+                },
+                NuGets = new DependencyProviderUtils.NuGetMetadata[]
+                {
+                    new DependencyProviderUtils.NuGetMetadata
+                    {
+                        FileName = $"{packageName}.{packageVersion}.nupkg",
+                        PackageName = packageName,
+                        PackageVersion = packageVersion
+                    }
+                }
+            };
+            Assert.True(expectedMetadata.Equals(actualMetadata));
+        }
+    }
+}

src/csharp/Extensions/Microsoft.Spark.Extensions.DotNet.Interactive/AssemblyKernelExtension.cs

@@ -0,0 +1,156 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis;
+using Microsoft.DotNet.Interactive;
+using Microsoft.DotNet.Interactive.Commands;
+using Microsoft.DotNet.Interactive.CSharp;
+using Microsoft.DotNet.Interactive.Utility;
+using Microsoft.Spark.Interop;
+using Microsoft.Spark.Sql;
+using Microsoft.Spark.Utils;
+
+namespace Microsoft.Spark.Extensions.DotNet.Interactive
+{
+    /// <summary>
+    /// A kernel extension when using .NET for Apache Spark with Microsoft.DotNet.Interactive
+    /// Adds nuget and assembly dependencies to the default <see cref="SparkSession"/>
+    /// using <see cref="SparkContext.AddFile(string, bool)"/>.
+    /// </summary>
+    public class AssemblyKernelExtension : IKernelExtension
+    {
+        private const string TempDirEnvVar = "DOTNET_SPARK_EXTENSION_INTERACTIVE_TMPDIR";
+
+        private readonly PackageResolver _packageResolver =
+            new PackageResolver(new PackageRestoreContextWrapper());
+
+        /// <summary>
+        /// Called by the Microsoft.DotNet.Interactive Assembly Extension Loader.
+        /// </summary>
+        /// <param name="kernel">The kernel calling this method.</param>
+        /// <returns><see cref="Task.CompletedTask"/> when extension is loaded.</returns>
+        public Task OnLoadAsync(IKernel kernel)
+        {
+            if (kernel is CompositeKernel kernelBase)
+            {
+                Environment.SetEnvironmentVariable(Constants.RunningREPLEnvVar, "true");
+
+                DirectoryInfo tempDir = CreateTempDirectory();
+                kernelBase.RegisterForDisposal(new DisposableDirectory(tempDir));
+
+                kernelBase.AddMiddleware(async (command, context, next) =>
+                {
+                    if ((context.HandlingKernel is CSharpKernel kernel) &&
+                        (command is SubmitCode) &&
+                        TryGetSparkSession(out SparkSession sparkSession) &&
+                        TryEmitAssembly(kernel, tempDir.FullName, out string assemblyPath))
+                    {
+                        sparkSession.SparkContext.AddFile(assemblyPath);
+
+                        foreach (string filePath in GetPackageFiles(tempDir.FullName))
+                        {
+                            sparkSession.SparkContext.AddFile(filePath);
+                        }
+                    }
+
+                    await next(command, context);
+                });
+            }
+
+            return Task.CompletedTask;
+        }
+
+        private DirectoryInfo CreateTempDirectory()
+        {
+            string envTempDir = Environment.GetEnvironmentVariable(TempDirEnvVar);
+            string tempDirBasePath = string.IsNullOrEmpty(envTempDir) ?
+                Directory.GetCurrentDirectory() :
+                envTempDir;
+
+            if (!IsPathValid(tempDirBasePath))
+            {
+                throw new Exception($"[{GetType().Name}] Spaces in " +
+                    $"'{tempDirBasePath}' is unsupported. Set the {TempDirEnvVar} " +
+                    "environment variable to control the base path. Please see " +
+                    "https://issues.apache.org/jira/browse/SPARK-30126 and " +
+                    "https://github.com/apache/spark/pull/26773 for more details.");
+            }
+
+            return Directory.CreateDirectory(
+                Path.Combine(tempDirBasePath, Path.GetRandomFileName()));
+        }
+
+        private bool TryEmitAssembly(CSharpKernel kernel, string dstPath, out string assemblyPath)
+        {
+            Compilation compilation = kernel.ScriptState.Script.GetCompilation();
+            string assemblyName =
+                AssemblyLoader.NormalizeAssemblyName(compilation.AssemblyName);
+            assemblyPath = Path.Combine(dstPath, $"{assemblyName}.dll");
+            if (!File.Exists(assemblyPath))
+            {
+                FileSystemExtensions.Emit(compilation, assemblyPath);
+                return true;
+            }
+
+            throw new Exception(
+                $"TryEmitAssembly() unexpected duplicate assembly: ${assemblyPath}");
+        }
+
+        private bool TryGetSparkSession(out SparkSession sparkSession)
+        {
+            sparkSession = SparkSession.GetDefaultSession();
+            return sparkSession != null;
+        }
+
+        private IEnumerable<string> GetPackageFiles(string path)
+        {
+            foreach (string filePath in _packageResolver.GetFiles(path))
+            {
+                if (IsPathValid(filePath))
+                {
+                    yield return filePath;
+                }
+                else
+                {
+                    // Copy file to a path without spaces.
+                    string fileDestPath = Path.Combine(
+                        path,
+                        Path.GetFileName(filePath).Replace(" ", string.Empty));
+                    File.Copy(filePath, fileDestPath);
+                    yield return fileDestPath;
+                }
+            }
+        }
+
+        /// <summary>
+        /// In some versions of Spark, spaces is unsupported when using
+        /// <see cref="SparkContext.AddFile(string, bool)"/>.
+        /// 
+        /// For more details please see:
+        /// - https://issues.apache.org/jira/browse/SPARK-30126
+        /// - https://github.com/apache/spark/pull/26773
+        /// </summary>
+        /// <param name="path">The path to validate.</param>
+        /// <returns>true if the path is supported by Spark, false otherwise.</returns>
+        private bool IsPathValid(string path)
+        {
+            if (!path.Contains(" "))
+            {
+                return true;
+            }
+
+            Version version = SparkEnvironment.SparkVersion;
+            return (version.Major, version.Minor, version.Build) switch
+            {
+                (2, _, _) => false,
+                (3, 0, _) => true,
+                _ => throw new NotSupportedException($"Spark {version} not supported.")
+            };
+        }
+    }
+}

src/csharp/Extensions/Microsoft.Spark.Extensions.DotNet.Interactive/Microsoft.Spark.Extensions.DotNet.Interactive.csproj

@@ -0,0 +1,40 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Library</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <RootNamespace>Microsoft.Spark.Extensions.DotNet.Interactive</RootNamespace>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <IsPackable>true</IsPackable>
+    <!-- NU5100 warns that a dll was found outside the 'lib' folder while packaging. DotNet.Interactive expects extension dlls in the 'interactive-extensions/dotnet'. -->
+    <NoWarn>NU5100;$(NoWarn)</NoWarn>
+
+    <Description>DotNet Interactive Extension for .NET for Apache Spark</Description>
+    <PackageReleaseNotes>https://github.com/dotnet/spark/tree/master/docs/release-notes</PackageReleaseNotes>
+    <PackageTags>spark;dotnet;csharp;interactive;dotnet-interactive</PackageTags>
+
+    <RestoreAdditionalProjectSources>https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json</RestoreAdditionalProjectSources>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="Microsoft.Spark.Extensions.DotNet.Interactive.UnitTest" />
+    <InternalsVisibleTo Include="DynamicProxyGenAssembly2" Key="0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Interactive.CSharp" Version="1.0.0-beta.20262.1">
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Microsoft.Spark\Microsoft.Spark.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="$(OutputPath)/Microsoft.Spark.Extensions.DotNet.Interactive.dll"
+      Pack="true"
+      PackagePath="interactive-extensions/dotnet" />
+  </ItemGroup>
+
+</Project>