In-Editor Analytics for inference

com.unity.ml-agents/Runtime/Agent.cs

@@ -421,7 +421,8 @@ public void LazyInitialize()
                 InitializeActuators();
             }
 
-            m_Brain = m_PolicyFactory.GeneratePolicy(m_ActuatorManager.GetCombinedActionSpec(), Heuristic);
+            var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec();
+            m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic);
             ResetData();
             Initialize();
 
@@ -430,6 +431,8 @@ public void LazyInitialize()
                 InitializeSensors();
             }
 
+            m_PolicyFactory.UpdateAnalytics(sensors, combinedActionSpec);
+
             m_Info.storedVectorActions = new float[m_ActuatorManager.TotalNumberOfActions];
 
             // The first time the Academy resets, all Agents in the scene will be
@@ -595,7 +598,9 @@ internal void ReloadPolicy()
                 return;
             }
             m_Brain?.Dispose();
-            m_Brain = m_PolicyFactory.GeneratePolicy(m_ActuatorManager.GetCombinedActionSpec(), Heuristic);
+            var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec();
+            m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic);
+            m_PolicyFactory.UpdateAnalytics(sensors, combinedActionSpec);
         }
 
         /// <summary>

com.unity.ml-agents/Runtime/Analytics/Events.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+namespace Unity.MLAgents.Analytics
+{
+    internal struct InferenceEvent
+    {
+        public string BehaviorName;
+        public string BarracudaModelSource;
+        public string BarracudaModelVersion;
+        public string BarracudaModelProducer;
+        public string BarracudaPackageVersion;
+        public int InferenceDevice;
+        public List<EventObservationSpec> ObservationSpecs;
+        public EventActionSpec ActionSpec;
+        public int MemorySize;
+        // public Int64 ModelHash; //TODO ?
+    }
+
+    /// <summary>
+    /// Simplified version of ActionSpec struct for use in analytics
+    /// </summary>
+    internal struct EventActionSpec
+    {
+        public int NumContinuousActions;
+        public int NumDiscreteActions;
+        public int[] BranchSizes;
+
+        public static EventActionSpec FromActionSpec(ActionSpec actionSpec)
+        {
+            var branchSizes = actionSpec.BranchSizes ?? Array.Empty<int>();
+            return new EventActionSpec
+            {
+                NumContinuousActions = actionSpec.NumContinuousActions,
+                NumDiscreteActions = actionSpec.NumDiscreteActions,
+                BranchSizes = branchSizes,
+            };
+        }
+    }
+
+    /// <summary>
+    /// Simplified summary of Agent observations for use in analytics
+    /// </summary>
+    internal struct EventObservationSpec
+    {
+        public string SensorName;
+        public int[] ObservationShape;
+
+        public static EventObservationSpec FromSensor(ISensor sensor)
+        {
+            return new EventObservationSpec
+            {
+                SensorName = sensor.GetName(),
+                ObservationShape = sensor.GetObservationShape(),
+            };
+        }
+    }
+}

com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs

@@ -0,0 +1,97 @@
+using System.Collections.Generic;
+using Unity.Barracuda;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Inference;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+using UnityEditor;
+using UnityEditor.Analytics;
+using UnityEngine.Analytics;
+
+namespace Unity.MLAgents.Analytics
+{
+    internal class InferenceAnalytics
+    {
+        static bool s_EventRegistered = false;
+        const int k_MaxEventsPerHour = 1000;
+        const int k_MaxNumberOfElements = 1000;
+        const string k_VendorKey = "unity.ml-agents";
+        const string k_EventName = "InferenceModelSet";
+
+        static bool EnableAnalytics()
+        {
+            if (s_EventRegistered)
+            {
+                return true;
+            }
+
+            AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey);
+            if (result == AnalyticsResult.Ok)
+            {
+                s_EventRegistered = true;
+            }
+
+            return s_EventRegistered;
+        }
+
+        public static void InferenceModelSet(
+            NNModel nnModel,
+            string behaviorName,
+            InferenceDevice inferenceDevice,
+            IList<ISensor> sensors,
+            ActionSpec actionSpec
+        )
+        {
+            // The event shouldn't be able to report if this is disabled but if we know we're not going to report
+            // Lets early out and not waste time gathering all the data
+            if (!EditorAnalytics.enabled)
+                return;
+
+            if (!EnableAnalytics())
+                return;
+
+            var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec);
+            //EditorAnalytics.SendEventWithLimit(k_EventName, data);
+        }
+
+        static InferenceEvent GetEventForModel(
+            NNModel nnModel,
+            string behaviorName,
+            InferenceDevice inferenceDevice,
+            IList<ISensor> sensors,
+            ActionSpec actionSpec
+        )
+        {
+            var barracudaModel = ModelLoader.Load(nnModel);
+            var inferenceEvent = new InferenceEvent();
+            inferenceEvent.BehaviorName = behaviorName;
+            inferenceEvent.BarracudaModelSource = barracudaModel.IrSource;
+            inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion;
+            inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName;
+            inferenceEvent.InferenceDevice = (int)inferenceDevice;
+
+            if (barracudaModel.ProducerName == "Script")
+            {
+                // .nn files don't have these fields set correctly. Assign some placeholder values.
+                inferenceEvent.BarracudaModelSource = "NN";
+                inferenceEvent.BarracudaModelProducer = "tf2bc.py";
+            }
+
+#if UNITY_2019_3_OR_NEWER
+            var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly);
+            inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version;
+#else
+            inferenceEvent.BarracudaPackageVersion = "unknown";
+#endif
+
+            inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec);
+            inferenceEvent.ObservationSpecs = new List<EventObservationSpec>(sensors.Count);
+            foreach (var sensor in sensors)
+            {
+                inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor));
+            }
+
+            return inferenceEvent;
+        }
+    }
+}

com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs

@@ -45,7 +45,9 @@ internal class BarracudaPolicy : IPolicy
         public BarracudaPolicy(
             ActionSpec actionSpec,
             NNModel model,
-            InferenceDevice inferenceDevice)
+            InferenceDevice inferenceDevice,
+            string behaviorName
+            )
         {
             var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, actionSpec, inferenceDevice);
             m_ModelRunner = modelRunner;

com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs

@@ -1,8 +1,11 @@
 using Unity.Barracuda;
 using System;
+using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.Serialization;
 using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Analytics;
+using Unity.MLAgents.Sensors;
 using Unity.MLAgents.Sensors.Reflection;
 
 namespace Unity.MLAgents.Policies
@@ -212,7 +215,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen
                                 "Either assign a model, or change to a different Behavior Type."
                             );
                         }
-                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice);
+                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName);
                     }
                 case BehaviorType.Default:
                     if (Academy.Instance.IsCommunicatorOn)
@@ -221,7 +224,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen
                     }
                     if (m_Model != null)
                     {
-                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice);
+                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName);
                     }
                     else
                     {
@@ -241,5 +244,35 @@ internal void UpdateAgentPolicy()
             }
             agent.ReloadPolicy();
         }
+
+        internal void UpdateAnalytics(IList<ISensor> sensors, ActionSpec actionSpec)
+        {
+            switch (m_BehaviorType)
+            {
+                case BehaviorType.HeuristicOnly:
+                    return;
+                case BehaviorType.InferenceOnly:
+                    {
+                        InferenceAnalytics.InferenceModelSet(m_Model, m_BehaviorName, m_InferenceDevice, sensors, actionSpec);
+                        return;
+                    }
+                case BehaviorType.Default:
+                    if (Academy.Instance.IsCommunicatorOn)
+                    {
+                        return;
+                    }
+                    if (m_Model != null)
+                    {
+                        InferenceAnalytics.InferenceModelSet(m_Model, m_BehaviorName, m_InferenceDevice, sensors, actionSpec);
+                        return;
+                    }
+                    else
+                    {
+                        return;
+                    }
+                default:
+                    return;
+            }
+        }
     }
 }