diff --git a/.yamato/com.unity.ml-agents-pack.yml b/.yamato/com.unity.ml-agents-pack.yml index cf22097dd7..3926eb575b 100644 --- a/.yamato/com.unity.ml-agents-pack.yml +++ b/.yamato/com.unity.ml-agents-pack.yml @@ -5,8 +5,12 @@ pack: image: package-ci/ubuntu:stable flavor: b1.small commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - upm-ci project pack --project-path Project + - | + python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade + unity-downloader-cli -u 2018.4 -c editor --wait --fast + ./.Editor/Unity -projectPath Project -batchMode -executeMethod Unity.MLAgents.SampleExporter.ExportCuratedSamples -logFile - + npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm + upm-ci project pack --project-path Project artifacts: packages: paths: diff --git a/.yamato/ml-agents-sample-export.yml b/.yamato/ml-agents-sample-export.yml deleted file mode 100644 index bbb73926d6..0000000000 --- a/.yamato/ml-agents-sample-export.yml +++ /dev/null @@ -1,22 +0,0 @@ -sample_export: - name: Samples Export 2021.2 - agent: - type: Unity::VM - image: package-ci/ubuntu:stable - flavor: b1.large - variables: - UNITY_VERSION: 2021.2 - commands: - - python3 -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade - - unity-downloader-cli -u 2021.2 -c editor --wait --fast - - python3 -u -m ml-agents.tests.yamato.sample_curation --scene "Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity" "Assets/ML-Agents/Examples/Match3/Scenes/Match3.unity" "Assets/ML-Agents/Examples/WallJump/Scenes/WallJump.unity" "Assets/ML-Agents/TestScenes/TestCompressedGrid/TestGridCompressed.unity" "Assets/ML-Agents/TestScenes/TestCompressedTexture/TestTextureCompressed.unity" - triggers: - cancel_old_ci: true - artifacts: - logs: - paths: - - "artifacts/sample_export.txt" - samples: - paths: - - "artifacts/Samples/**" diff --git a/Project/Assets/ML-Agents/Editor/Tests/SampleExporter.cs b/Project/Assets/ML-Agents/Editor/Tests/SampleExporter.cs index f9ddc19d78..aac35c00d8 100644 --- a/Project/Assets/ML-Agents/Editor/Tests/SampleExporter.cs +++ b/Project/Assets/ML-Agents/Editor/Tests/SampleExporter.cs @@ -1,16 +1,34 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; +using Newtonsoft.Json; using UnityEditor; using UnityEngine; -using UnityEngine.iOS; namespace Unity.MLAgents { public class SampleExporter { - const string k_SceneFlag = "--mlagents-scene-path"; + const string k_MLAgentsSampleFile = "mlagents-sample.json"; + const string k_PackageSampleFile = ".sample.json"; + const string k_MLAgentsDir = "ML-Agents"; + const string k_MLAgentsExamplesDir = "Examples"; + const string k_MLAgentsPackageName = "com.unity.ml-agents"; + const string k_MLAgentsSamplesDirName = "Samples"; + const string k_MLAgentsScriptsDirName = "Scripts"; + + struct MLAgentsSampleJson + { + public string displayName; + public string description; + public List scenes; + } + + struct PackageSampleJson + { + public string displayName; + public string description; + } public static void ExportCuratedSamples() { @@ -18,35 +36,78 @@ public static void ExportCuratedSamples() EditorPrefs.SetBool("BurstCompilation", false); try { - var args = Environment.GetCommandLineArgs(); - var scenes = new List(); - for (var i = 0; i < args.Length - 1; i++) + // Path to Project/Assets + var assetsDir = Application.dataPath; + var repoRoot = Directory.GetParent(Directory.GetParent(assetsDir).FullName).FullName; + + // Top level of where to store the samples + var samplesDir = Path.Combine( + repoRoot, + k_MLAgentsPackageName, + k_MLAgentsSamplesDirName); + + if (!Directory.Exists(samplesDir)) { - if (args[i] == k_SceneFlag) - { - scenes.Add(args[i + 1]); - Debug.Log($"Exporting Scene {scenes.Last()}"); - } + Directory.CreateDirectory(samplesDir); } - foreach (var scene in scenes) + // Path to the examples dir in the project + var examplesDir = Path.Combine(Application.dataPath, k_MLAgentsDir, k_MLAgentsExamplesDir); + foreach (var exampleDirectory in Directory.GetDirectories(examplesDir)) { - var assets = new List { scene }; - var exampleFolderToAdd = Directory.GetParent(Directory.GetParent(scene).FullName).FullName; - Debug.Log($"Parent of Scene: {exampleFolderToAdd}"); - if (Directory.Exists(Path.Combine(exampleFolderToAdd, "Scripts"))) + var mlAgentsSamplePath = Path.Combine(exampleDirectory, k_MLAgentsSampleFile); + if (File.Exists(mlAgentsSamplePath)) { - exampleFolderToAdd = Path.Combine(exampleFolderToAdd, "Scripts"); - } + var sampleJson = JsonConvert.DeserializeObject(File.ReadAllText(mlAgentsSamplePath)); + Debug.Log(JsonConvert.SerializeObject(sampleJson)); + foreach (var scene in sampleJson.scenes) + { + var scenePath = Path.Combine(exampleDirectory, scene); + if (File.Exists(scenePath)) + { + // Create a Sample Directory + var currentSampleDir = Directory.CreateDirectory(Path.Combine(samplesDir, + Path.GetFileNameWithoutExtension(scenePath))); - exampleFolderToAdd = exampleFolderToAdd.Substring(exampleFolderToAdd.IndexOf("Assets")); - foreach (var guid in AssetDatabase.FindAssets("t:Script", new[] { exampleFolderToAdd })) - { - var path = AssetDatabase.GUIDToAssetPath(guid); - assets.Add(path); - Debug.Log($"Adding Asset: {path}"); + + var scriptsPath = Path.Combine(exampleDirectory, k_MLAgentsScriptsDirName); + Debug.Log($"Scene Path: {scenePath}"); + var assets = new List { scenePath.Substring(scenePath.IndexOf("Assets")) }; + if (!Directory.Exists(Path.Combine(scriptsPath))) + { + scriptsPath = exampleDirectory; + } + + scriptsPath = scriptsPath.Substring(scriptsPath.IndexOf("Assets")); + foreach (var guid in AssetDatabase.FindAssets("t:Script", new[] { scriptsPath })) + { + var path = AssetDatabase.GUIDToAssetPath(guid); + assets.Add(path); + Debug.Log($"Adding Asset: {path}"); + } + + var packageFilePath = Path.GetFileNameWithoutExtension(scenePath) + ".unitypackage"; + AssetDatabase.ExportPackage(assets.ToArray(), + Path.Combine(Application.dataPath, packageFilePath), + ExportPackageOptions.IncludeDependencies | ExportPackageOptions.Recurse); + + // Move the .unitypackage into the samples folder. + var packageFileFullPath = Path.Combine(Application.dataPath, packageFilePath); + + var packageInSamplePath = Path.Combine(currentSampleDir.FullName, packageFilePath); + Debug.Log($"Moving {packageFileFullPath} to {packageInSamplePath}"); + File.Move(packageFileFullPath, packageInSamplePath); + + // write the .sample.json file to the sample directory + File.WriteAllText(Path.Combine(currentSampleDir.FullName, k_PackageSampleFile), + JsonConvert.SerializeObject(new PackageSampleJson + { + description = sampleJson.description, + displayName = sampleJson.displayName + })); + } + } } - AssetDatabase.ExportPackage(assets.ToArray(), Path.GetFileNameWithoutExtension(scene) + ".unitypackage", ExportPackageOptions.IncludeDependencies | ExportPackageOptions.Recurse); } } catch (Exception e) diff --git a/Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json b/Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json new file mode 100644 index 0000000000..b1cce936e5 --- /dev/null +++ b/Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json @@ -0,0 +1,7 @@ +{ + "displayName": "3D Ball", + "description": "The 3D Ball sample is a simple environment that is a great for jumping into Ml-Agents to see how things work.", + "scenes": [ + "Scenes/3DBall.unity" + ] +} diff --git a/Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json.meta b/Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json.meta new file mode 100644 index 0000000000..1e0892275e --- /dev/null +++ b/Project/Assets/ML-Agents/Examples/3DBall/mlagents-sample.json.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 6b6f1c189dc84df391d1c3ccb13a54f7 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Project/Packages/manifest.json b/Project/Packages/manifest.json index 72137e00e7..caa9463d39 100644 --- a/Project/Packages/manifest.json +++ b/Project/Packages/manifest.json @@ -37,7 +37,8 @@ "com.unity.modules.video": "1.0.0", "com.unity.modules.vr": "1.0.0", "com.unity.modules.wind": "1.0.0", - "com.unity.modules.xr": "1.0.0" + "com.unity.modules.xr": "1.0.0", + "com.unity.nuget.newtonsoft-json": "2.0.0" }, "testables": [ "com.unity.ml-agents", diff --git a/Project/Project.sln.DotSettings b/Project/Project.sln.DotSettings index aacb62dadd..8870bbaaba 100644 --- a/Project/Project.sln.DotSettings +++ b/Project/Project.sln.DotSettings @@ -1,2 +1,3 @@  + ML True \ No newline at end of file diff --git a/com.unity.ml-agents/.gitignore b/com.unity.ml-agents/.gitignore index 0242da45db..2edd5f2a68 100755 --- a/com.unity.ml-agents/.gitignore +++ b/com.unity.ml-agents/.gitignore @@ -21,9 +21,11 @@ build.bat.meta /Assets/Plugins* /Assets/Demonstrations* /csharp_timers.json +/Samples/ +/Samples.meta # Visual Studio 2015 cache directory /.vs/ *.api -*.api.meta \ No newline at end of file +*.api.meta diff --git a/ml-agents/tests/yamato/sample_curation.py b/ml-agents/tests/yamato/sample_curation.py deleted file mode 100644 index 8ea6884ac3..0000000000 --- a/ml-agents/tests/yamato/sample_curation.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys -import argparse - -from .yamato_utils import get_base_path, create_samples - - -def main(scenes): - base_path = get_base_path() - print(f"Running in base path {base_path}") - - returncode = create_samples( - scenes, - base_path, - log_output_path=None, # Log to stdout so we get timestamps on the logs - ) - - if returncode == 0: - print("Test run SUCCEEDED!") - else: - print("Test run FAILED!") - - sys.exit(returncode) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--scene", nargs="+", default=None, required=True) - args = parser.parse_args() - main(args.scene) diff --git a/ml-agents/tests/yamato/yamato_utils.py b/ml-agents/tests/yamato/yamato_utils.py index 02a6a25386..2302227db1 100644 --- a/ml-agents/tests/yamato/yamato_utils.py +++ b/ml-agents/tests/yamato/yamato_utils.py @@ -1,4 +1,3 @@ -import glob import os import shutil import subprocess @@ -230,52 +229,3 @@ def override_legacy_config_file(python_version, src_path, dest_path, **kwargs): with open(dest_path, "w") as f: yaml.dump(configs, f) - - -def create_samples( - scenes: List[str], - base_path: str, - log_output_path: Optional[str] = f"{get_base_output_path()}/sample_export.txt", -) -> int: - unity_exe = get_unity_executable_path() - test_args = [ - unity_exe, - "-projectPath", - f"{base_path}/Project", - "-batchmode", - "-executeMethod", - "Unity.MLAgents.SampleExporter.ExportCuratedSamples", - ] - - if log_output_path: - os.makedirs(os.path.dirname(log_output_path), exist_ok=True) - subprocess.run(["touch", log_output_path]) - test_args += ["-logfile", log_output_path] - else: - # Log to stdout - test_args += ["-logfile", "-"] - - os.makedirs(os.path.join(get_base_output_path(), "Samples"), exist_ok=True) - - for scene in scenes: - test_args += ["--mlagents-scene-path", scene] - - timeout = 5 * 60 # 5 minutes for now - res: subprocess.CompletedProcess = subprocess.run(test_args, timeout=timeout) - - if res.returncode == 0: - for file in glob.glob(os.path.join(base_path, "Project/*.unitypackage")): - print( - f"moving {file} to {os.path.join(get_base_output_path(), 'Samples', os.path.basename(file))}" - ) - shutil.move( - file, - os.path.join(get_base_output_path(), "Samples", os.path.basename(file)), - ) - - # Print if we fail or want verbosity. - if res.returncode != 0: - if log_output_path: - subprocess.run(["cat", log_output_path]) - - return res.returncode