Skip to content

Update Analytics to use new API #6221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/pre-commit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ on:
jobs:
pre-commit:
runs-on: ubuntu-24.04
env:
DOTNET_NOLOGO: 1
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
Expand All @@ -26,6 +28,8 @@ jobs:
- uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.202'
- name: Clean dotnet shared memory
run: sudo rm -rf /tmp/.dotnet/shm
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any objections to do a cleanup first? Anything else is using /tmp/.dotnet/shm?

- name: Install manual dependencies
run: |
python -m pip install pre-commit
Expand Down
58 changes: 54 additions & 4 deletions com.unity.ml-agents/Runtime/Analytics/Events.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,28 @@
using System.Collections.Generic;
using Unity.MLAgents.Actuators;
using Unity.MLAgents.Sensors;
using UnityEngine.Analytics;

namespace Unity.MLAgents.Analytics
{
internal struct InferenceEvent
internal static class AnalyticsConstants
{
public const string k_VendorKey = "unity.ml-agents";

/// <summary>
/// Maximum number of events sent per hour.
/// </summary>
public const int k_MaxEventsPerHour = 1000;

/// <summary>
/// Maximum number of items in an event.
/// </summary>
public const int k_MaxNumberOfElements = 1000;
}

[Serializable]
[AnalyticInfo(eventName: "ml_agents_inferencemodelset", version: 1, vendorKey: AnalyticsConstants.k_VendorKey, maxEventsPerHour: AnalyticsConstants.k_MaxEventsPerHour, maxNumberOfElements: AnalyticsConstants.k_MaxNumberOfElements)]
internal class InferenceEvent : IAnalytic.IData, IAnalytic
{
/// <summary>
/// Hash of the BehaviorName.
Expand All @@ -25,6 +43,12 @@ internal struct InferenceEvent
public int MemorySize;
public long TotalWeightSizeBytes;
public string ModelHash;
public bool TryGatherData(out IAnalytic.IData data, out Exception error)
{
data = this;
error = null;
return true;
}
}

/// <summary>
Expand Down Expand Up @@ -126,7 +150,9 @@ public static EventObservationSpec FromSensor(ISensor sensor)
}
}

internal struct RemotePolicyInitializedEvent
[Serializable]
[AnalyticInfo(eventName: "ml_agents_remote_policy_initialized", vendorKey: AnalyticsConstants.k_VendorKey, maxEventsPerHour: AnalyticsConstants.k_MaxEventsPerHour, maxNumberOfElements: AnalyticsConstants.k_MaxNumberOfElements)]
internal class RemotePolicyInitializedEvent : IAnalytic.IData, IAnalytic
{
public string TrainingSessionGuid;
/// <summary>
Expand All @@ -143,9 +169,18 @@ internal struct RemotePolicyInitializedEvent
/// </summary>
public string MLAgentsEnvsVersion;
public string TrainerCommunicationVersion;
public bool TryGatherData(out IAnalytic.IData data, out Exception error)
{
data = this;
error = null;
return true;
}
}

internal struct TrainingEnvironmentInitializedEvent

[Serializable]
[AnalyticInfo(eventName: "ml_agents_training_environment_initialized", vendorKey: AnalyticsConstants.k_VendorKey, maxEventsPerHour: AnalyticsConstants.k_MaxEventsPerHour, maxNumberOfElements: AnalyticsConstants.k_MaxNumberOfElements)]
internal class TrainingEnvironmentInitializedEvent : IAnalytic.IData, IAnalytic
{
public string TrainingSessionGuid;

Expand All @@ -157,6 +192,12 @@ internal struct TrainingEnvironmentInitializedEvent
public int NumEnvironments;
public int NumEnvironmentParameters;
public string RunOptions;
public bool TryGatherData(out IAnalytic.IData data, out Exception error)
{
data = this;
error = null;
return true;
}
}

[Flags]
Expand All @@ -178,7 +219,9 @@ internal enum TrainingFeatures
Curriculum = 1 << 4,
}

internal struct TrainingBehaviorInitializedEvent
[Serializable]
[AnalyticInfo(eventName: "ml_agents_training_behavior_initialized", vendorKey: AnalyticsConstants.k_VendorKey, maxEventsPerHour: AnalyticsConstants.k_MaxEventsPerHour, maxNumberOfElements: AnalyticsConstants.k_MaxNumberOfElements)]
internal class TrainingBehaviorInitializedEvent : IAnalytic.IData, IAnalytic
{
public string TrainingSessionGuid;

Expand All @@ -190,5 +233,12 @@ internal struct TrainingBehaviorInitializedEvent
public int NumNetworkLayers;
public int NumNetworkHiddenUnits;
public string Config;

public bool TryGatherData(out IAnalytic.IData data, out Exception error)
{
data = this;
error = null;
return true;
}
}
}
38 changes: 6 additions & 32 deletions com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,6 @@ namespace Unity.MLAgents.Analytics
{
internal class InferenceAnalytics
{
const string k_VendorKey = "unity.ml-agents";
const string k_EventName = "ml_agents_inferencemodelset";
const int k_EventVersion = 1;

/// <summary>
/// Whether or not we've registered this particular event yet
/// </summary>
static bool s_EventRegistered;

/// <summary>
/// Hourly limit for this event name
/// </summary>
const int k_MaxEventsPerHour = 1000;

/// <summary>
/// Maximum number of items in this event.
/// </summary>
const int k_MaxNumberOfElements = 1000;


#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS
Expand All @@ -54,25 +36,17 @@ internal class InferenceAnalytics
static bool EnableAnalytics()
{
#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS
if (s_EventRegistered)
{
return true;
}

AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey, k_EventVersion);
if (result == AnalyticsResult.Ok)
{
s_EventRegistered = true;
}
if (s_EventRegistered && s_SentModels == null)
if (s_SentModels == null)
{
s_SentModels = new HashSet<ModelAsset>();
}

return true;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

EnableAnalytics() always returns true if the conditions apply, i.e. UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS. Is that OK? We don't RegisterEvent anymore in the new API.


#else // no editor, no analytics
s_EventRegistered = false;
return false;
#endif
return s_EventRegistered;
}

public static bool IsAnalyticsEnabled()
Expand Down Expand Up @@ -127,7 +101,7 @@ IList<IActuator> actuators
// Debug.Log(JsonUtility.ToJson(data, true));
if (AnalyticsUtils.s_SendEditorAnalytics)
{
EditorAnalytics.SendEventWithLimit(k_EventName, data, k_EventVersion);
EditorAnalytics.SendAnalytic(data);
}
#endif
}
Expand Down Expand Up @@ -156,7 +130,7 @@ IList<IActuator> actuators
var inferenceEvent = new InferenceEvent();

// Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
inferenceEvent.BehaviorName = AnalyticsUtils.Hash(k_VendorKey, behaviorName);
inferenceEvent.BehaviorName = AnalyticsUtils.Hash(AnalyticsConstants.k_VendorKey, behaviorName);

inferenceEvent.SentisModelVersion = sentisModelInfo.Version;
inferenceEvent.SentisModelProducer = sentisModel.ProducerName;
Expand Down
57 changes: 9 additions & 48 deletions com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,35 +23,9 @@ namespace Unity.MLAgents.Analytics
{
internal static class TrainingAnalytics
{
const string k_VendorKey = "unity.ml-agents";
const string k_TrainingEnvironmentInitializedEventName = "ml_agents_training_environment_initialized";
const string k_TrainingBehaviorInitializedEventName = "ml_agents_training_behavior_initialized";
const string k_RemotePolicyInitializedEventName = "ml_agents_remote_policy_initialized";

private static readonly string[] s_EventNames =
{
k_TrainingEnvironmentInitializedEventName,
k_TrainingBehaviorInitializedEventName,
k_RemotePolicyInitializedEventName
};

/// <summary>
/// Hourly limit for this event name
/// </summary>
const int k_MaxEventsPerHour = 1000;

/// <summary>
/// Maximum number of items in this event.
/// </summary>
const int k_MaxNumberOfElements = 1000;

private static bool s_SentEnvironmentInitialized;

#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS
/// <summary>
/// Whether or not we've registered this particular event yet
/// </summary>
static bool s_EventsRegistered;

/// <summary>
/// Behaviors that we've already sent events for.
Expand All @@ -69,19 +43,6 @@ internal static class TrainingAnalytics
internal static bool EnableAnalytics()
{
#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS
if (s_EventsRegistered)
{
return true;
}
foreach (var eventName in s_EventNames)
{
AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(eventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey);
if (result != AnalyticsResult.Ok)
{
return false;
}
}
s_EventsRegistered = true;

if (s_SentRemotePolicyInitialized == null)
{
Expand All @@ -90,7 +51,7 @@ internal static bool EnableAnalytics()
s_TrainingSessionGuid = Guid.NewGuid();
}

return s_EventsRegistered;
return true;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

EnableAnalytics() always returns true if the conditions apply, i.e. UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS. Is that OK? We don't RegisterEvent anymore in the new API.

#else
return false;
#endif // MLA_UNITY_ANALYTICS_MODULE
Expand Down Expand Up @@ -137,12 +98,12 @@ public static void TrainingEnvironmentInitialized(TrainingEnvironmentInitialized

// Note - to debug, use JsonUtility.ToJson on the event.
// Debug.Log(
// $"Would send event {k_TrainingEnvironmentInitializedEventName} with body {JsonUtility.ToJson(tbiEvent, true)}"
// $"Would send event ml_agents_training_environment_initialized with body {JsonUtility.ToJson(tbiEvent, true)}"
// );
#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS
if (AnalyticsUtils.s_SendEditorAnalytics)
{
EditorAnalytics.SendEventWithLimit(k_TrainingEnvironmentInitializedEventName, tbiEvent);
EditorAnalytics.SendAnalytic(tbiEvent);
}
#endif
}
Expand Down Expand Up @@ -175,11 +136,11 @@ IList<IActuator> actuators
var data = GetEventForRemotePolicy(behaviorName, sensors, actionSpec, actuators);
// Note - to debug, use JsonUtility.ToJson on the event.
// Debug.Log(
// $"Would send event {k_RemotePolicyInitializedEventName} with body {JsonUtility.ToJson(data, true)}"
// $"Would send event ml_agents_remote_policy_initialized with body {JsonUtility.ToJson(data, true)}"
// );
if (AnalyticsUtils.s_SendEditorAnalytics)
{
EditorAnalytics.SendEventWithLimit(k_RemotePolicyInitializedEventName, data);
EditorAnalytics.SendAnalytic(data);
}
#endif
}
Expand All @@ -203,7 +164,7 @@ internal static TrainingBehaviorInitializedEvent SanitizeTrainingBehaviorInitial
// Context: The config field was added at the same time as trainer side hashing, so messages including it should already be hashed.
if (tbiEvent.Config.Length == 0 || tbiEvent.BehaviorName.Length != 64)
{
tbiEvent.BehaviorName = AnalyticsUtils.Hash(k_VendorKey, tbiEvent.BehaviorName);
tbiEvent.BehaviorName = AnalyticsUtils.Hash(AnalyticsConstants.k_VendorKey, tbiEvent.BehaviorName);
}

return tbiEvent;
Expand Down Expand Up @@ -233,11 +194,11 @@ public static void TrainingBehaviorInitialized(TrainingBehaviorInitializedEvent

// Note - to debug, use JsonUtility.ToJson on the event.
// Debug.Log(
// $"Would send event {k_TrainingBehaviorInitializedEventName} with body {JsonUtility.ToJson(tbiEvent, true)}"
// $"Would send event ml_agents_training_behavior_initialized with body {JsonUtility.ToJson(tbiEvent, true)}"
// );
if (AnalyticsUtils.s_SendEditorAnalytics)
{
EditorAnalytics.SendEventWithLimit(k_TrainingBehaviorInitializedEventName, tbiEvent);
EditorAnalytics.SendAnalytic(tbiEvent);
}
#endif
}
Expand All @@ -252,7 +213,7 @@ IList<IActuator> actuators
var remotePolicyEvent = new RemotePolicyInitializedEvent();

// Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
remotePolicyEvent.BehaviorName = AnalyticsUtils.Hash(k_VendorKey, behaviorName);
remotePolicyEvent.BehaviorName = AnalyticsUtils.Hash(AnalyticsConstants.k_VendorKey, behaviorName);

remotePolicyEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString();
remotePolicyEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ public string TestTrainingBehaviorInitialized(string stringToMaybeHash)
[Test]
public void TestEnableAnalytics()
{
#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE
Assert.IsTrue(EditorAnalytics.enabled == TrainingAnalytics.EnableAnalytics());
#if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS
Assert.IsTrue(TrainingAnalytics.EnableAnalytics());
#else
Assert.IsFalse(TrainingAnalytics.EnableAnalytics());
#endif
Expand Down