From fe2cc6aca85569876578cbc1348f07a2b85cacb8 Mon Sep 17 00:00:00 2001 From: "James Truher [MSFT]" Date: Thu, 15 Aug 2019 16:13:53 -0700 Subject: [PATCH] Additional Telemetry - Implementation of `RFC0036` (#10336) --- build.psm1 | 4 +- .../Microsoft.PowerShell.ConsoleHost.csproj | 2 - .../host/msh/ConsoleHost.cs | 15 +- .../host/msh/Telemetry.cs | 102 ----- .../CoreCLR/CorePsPlatform.cs | 15 + .../System.Management.Automation.csproj | 2 + .../CommandCompletion/CommandCompletion.cs | 9 +- .../ExperimentalFeature.cs | 3 + .../engine/GetCommandCommand.cs | 10 +- .../engine/Modules/ImportModuleCommand.cs | 21 +- .../engine/hostifaces/PowerShell.cs | 3 + .../engine/pipeline.cs | 5 + .../client/RemoteRunspacePoolInternal.cs | 3 + .../utils/Telemetry.cs | 397 ++++++++++++++++++ test/common/markdown/markdown-link.tests.ps1 | 9 +- test/powershell/Host/Startup.Tests.ps1 | 3 - .../engine/Basic/Telemetry.Tests.ps1 | 131 ++++++ 17 files changed, 610 insertions(+), 124 deletions(-) delete mode 100644 src/Microsoft.PowerShell.ConsoleHost/host/msh/Telemetry.cs create mode 100644 src/System.Management.Automation/utils/Telemetry.cs create mode 100644 test/powershell/engine/Basic/Telemetry.Tests.ps1 diff --git a/build.psm1 b/build.psm1 index f8b6696f03..9169d1cbda 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1012,7 +1012,7 @@ function Start-PSPester { # All concatenated commands/arguments are suffixed with the delimiter (space) # Disable telemetry for all startups of pwsh in tests - $command = "`$env:POWERSHELL_TELEMETRY_OPTOUT = 1;" + $command = "`$env:POWERSHELL_TELEMETRY_OPTOUT = 'yes';" if ($Terse) { $command += "`$ProgressPreference = 'silentlyContinue'; " @@ -1153,7 +1153,7 @@ function Start-PSPester { try { $originalModulePath = $env:PSModulePath $originalTelemetry = $env:POWERSHELL_TELEMETRY_OPTOUT - $env:POWERSHELL_TELEMETRY_OPTOUT = 1 + $env:POWERSHELL_TELEMETRY_OPTOUT = 'yes' if ($Unelevate) { Start-UnelevatedProcess -process $powershell -arguments ($PSFlags + "-c $Command") diff --git a/src/Microsoft.PowerShell.ConsoleHost/Microsoft.PowerShell.ConsoleHost.csproj b/src/Microsoft.PowerShell.ConsoleHost/Microsoft.PowerShell.ConsoleHost.csproj index 77ef5c5bf1..156834ea6f 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/Microsoft.PowerShell.ConsoleHost.csproj +++ b/src/Microsoft.PowerShell.ConsoleHost/Microsoft.PowerShell.ConsoleHost.csproj @@ -8,8 +8,6 @@ - - diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs index 6b461fa603..9ba3d11899 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs +++ b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs @@ -24,6 +24,7 @@ using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.PowerShell.Telemetry; using ConsoleHandle = Microsoft.Win32.SafeHandles.SafeFileHandle; using Dbg = System.Management.Automation.Diagnostics; @@ -123,12 +124,8 @@ namespace Microsoft.PowerShell try { - string profileDir; -#if UNIX - profileDir = Platform.SelectProductNameForDirectory(Platform.XDG_Type.CACHE); -#else - profileDir = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + @"\Microsoft\PowerShell"; - + string profileDir = Platform.CacheDirectory; +#if ! UNIX if (!Directory.Exists(profileDir)) { Directory.CreateDirectory(profileDir); @@ -200,12 +197,14 @@ namespace Microsoft.PowerShell // First check for and handle PowerShell running in a server mode. if (s_cpp.ServerMode) { + ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("ServerMode"); ProfileOptimization.StartProfile("StartupProfileData-ServerMode"); System.Management.Automation.Remoting.Server.OutOfProcessMediator.Run(s_cpp.InitialCommand); exitCode = 0; } else if (s_cpp.NamedPipeServerMode) { + ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("NamedPipe"); ProfileOptimization.StartProfile("StartupProfileData-NamedPipeServerMode"); System.Management.Automation.Remoting.RemoteSessionNamedPipeServer.RunServerMode( s_cpp.ConfigurationName); @@ -213,12 +212,14 @@ namespace Microsoft.PowerShell } else if (s_cpp.SSHServerMode) { + ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("SSHServer"); ProfileOptimization.StartProfile("StartupProfileData-SSHServerMode"); System.Management.Automation.Remoting.Server.SSHProcessMediator.Run(s_cpp.InitialCommand); exitCode = 0; } else if (s_cpp.SocketServerMode) { + ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("SocketServerMode"); ProfileOptimization.StartProfile("StartupProfileData-SocketServerMode"); System.Management.Automation.Remoting.Server.HyperVSocketMediator.Run(s_cpp.InitialCommand, s_cpp.ConfigurationName); @@ -242,7 +243,7 @@ namespace Microsoft.PowerShell PSHost.IsStdOutputRedirected = Console.IsOutputRedirected; // Send startup telemetry for ConsoleHost startup - ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry(); + ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("Normal"); exitCode = s_theConsoleHost.Run(s_cpp, false); } diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/Telemetry.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/Telemetry.cs deleted file mode 100644 index be20aa8cf6..0000000000 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/Telemetry.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Management.Automation; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Security.Cryptography; - -using Microsoft.ApplicationInsights; -using Microsoft.ApplicationInsights.DataContracts; -using Microsoft.ApplicationInsights.Extensibility; - -namespace Microsoft.PowerShell -{ - /// - /// Send up telemetry for startup. - /// - internal static class ApplicationInsightsTelemetry - { - // If this env var is true, yes, or 1, telemetry will NOT be sent. - private const string TelemetryOptoutEnvVar = "POWERSHELL_TELEMETRY_OPTOUT"; - - // Telemetry client to be reused when we start sending more telemetry - private static TelemetryClient _telemetryClient = null; - - // Set this to true to reduce the latency of sending the telemetry - private static bool _developerMode = false; - - // PSCoreInsight2 telemetry key - private const string _psCoreTelemetryKey = "ee4b2115-d347-47b0-adb6-b19c2c763808"; - - static ApplicationInsightsTelemetry() - { - TelemetryConfiguration.Active.InstrumentationKey = _psCoreTelemetryKey; - TelemetryConfiguration.Active.TelemetryChannel.DeveloperMode = _developerMode; - } - - private static bool GetEnvironmentVariableAsBool(string name, bool defaultValue) - { - var str = Environment.GetEnvironmentVariable(name); - if (string.IsNullOrEmpty(str)) - { - return defaultValue; - } - - switch (str.ToLowerInvariant()) - { - case "true": - case "1": - case "yes": - return true; - case "false": - case "0": - case "no": - return false; - default: - return defaultValue; - } - } - - /// - /// Send the telemetry. - /// - private static void SendTelemetry(string eventName, Dictionary payload) - { - try - { - var enabled = !GetEnvironmentVariableAsBool(name: TelemetryOptoutEnvVar, defaultValue: false); - - if (!enabled) - { - return; - } - - if (_telemetryClient == null) - { - _telemetryClient = new TelemetryClient(); - } - - _telemetryClient.TrackEvent(eventName, payload, null); - } - catch (Exception) - { - ; // Do nothing, telemetry can't be sent - } - } - - /// - /// Create the startup payload and send it up. - /// - internal static void SendPSCoreStartupTelemetry() - { - var properties = new Dictionary(); - properties.Add("GitCommitID", PSVersionInfo.GitCommitId); - properties.Add("OSDescription", RuntimeInformation.OSDescription); - SendTelemetry("ConsoleHostStartup", properties); - } - } -} diff --git a/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs b/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs index c569c724da..f26b3cd7e6 100644 --- a/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs +++ b/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs @@ -140,6 +140,21 @@ namespace System.Management.Automation } } + /// + /// Gets the location for the various caches. + /// + internal static string CacheDirectory + { + get + { +#if UNIX + return Platform.SelectProductNameForDirectory(Platform.XDG_Type.CACHE); +#else + return Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + @"\Microsoft\PowerShell"; +#endif + } + } + #if !UNIX private static bool? _isNanoServer = null; private static bool? _isIoT = null; diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index 4f02fa988b..6700e342a5 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -13,6 +13,8 @@ + + diff --git a/src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs b/src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs index 00e9ebb4bc..3297f8bf15 100644 --- a/src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs +++ b/src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs @@ -515,9 +515,12 @@ namespace System.Management.Automation // This is the start of the real implementation of autocomplete/intellisense/tab completion private static CommandCompletion CompleteInputImpl(Ast ast, Token[] tokens, IScriptPosition positionOfCursor, Hashtable options) { +#if LEGACYTELEMETRY + // We could start collecting telemetry at a later date. + // We will leave the #if to remind us that we did this once. var sw = new Stopwatch(); sw.Start(); - +#endif using (var powershell = PowerShell.Create(RunspaceMode.CurrentRunspace)) { var context = LocalPipeline.GetExecutionContextFromTLS(); @@ -590,8 +593,10 @@ namespace System.Management.Automation } var completionResults = results ?? EmptyCompletionResult; - sw.Stop(); + #if LEGACYTELEMETRY + // no telemetry here. We don't capture tab completion performance. + sw.Stop(); TelemetryAPI.ReportTabCompletionTelemetry(sw.ElapsedMilliseconds, completionResults.Count, completionResults.Count > 0 ? completionResults[0].ResultType : CompletionResultType.Text); #endif diff --git a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs index 7bf4678116..676c7decea 100644 --- a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs +++ b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs @@ -9,6 +9,7 @@ using System.Management.Automation.Configuration; using System.Management.Automation.Internal; using System.Management.Automation.Tracing; using System.Runtime.CompilerServices; +using Microsoft.PowerShell.Telemetry; namespace System.Management.Automation { @@ -150,6 +151,7 @@ namespace System.Management.Automation if (IsModuleFeatureName(name)) { list.Add(name); + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ExperimentalModuleFeatureActivation, name); } else if (IsEngineFeatureName(name)) { @@ -157,6 +159,7 @@ namespace System.Management.Automation { feature.Enabled = true; list.Add(name); + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ExperimentalEngineFeatureActivation, name); } else { diff --git a/src/System.Management.Automation/engine/GetCommandCommand.cs b/src/System.Management.Automation/engine/GetCommandCommand.cs index fdfc7f507a..53c0827e12 100644 --- a/src/System.Management.Automation/engine/GetCommandCommand.cs +++ b/src/System.Management.Automation/engine/GetCommandCommand.cs @@ -369,8 +369,9 @@ namespace Microsoft.PowerShell.Commands /// protected override void BeginProcessing() { +#if LEGACYTELEMETRY _timer.Start(); - +#endif base.BeginProcessing(); if (ShowCommandInfo.IsPresent && Syntax.IsPresent) @@ -552,9 +553,11 @@ namespace Microsoft.PowerShell.Commands } } +#if LEGACYTELEMETRY _timer.Stop(); -#if LEGACYTELEMETRY + // No telemetry here - capturing the name of a command which we are not familiar with + // may be confidential customer information // We want telementry on commands people look for but don't exist - this should give us an idea // what sort of commands people expect but either don't exist, or maybe should be installed by default. // The StartsWith is to avoid logging telemetry when suggestion mode checks the @@ -1443,8 +1446,9 @@ namespace Microsoft.PowerShell.Commands private Collection _nounPatterns; private Collection _modulePatterns; +#if LEGACYTELEMETRY private Stopwatch _timer = new Stopwatch(); - +#endif #endregion #region ShowCommandInfo support diff --git a/src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs b/src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs index 7bafebc37d..f2ce89726f 100644 --- a/src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs +++ b/src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs @@ -10,6 +10,7 @@ using System.IO; using System.Linq; using System.Management.Automation; using System.Management.Automation.Internal; +using System.Management.Automation.Language; using System.Management.Automation.Runspaces; using System.Reflection; using System.Security; @@ -17,13 +18,13 @@ using System.Threading; using Microsoft.Management.Infrastructure; using Microsoft.PowerShell.Cmdletization; +using Microsoft.PowerShell.Telemetry; using Dbg = System.Management.Automation.Diagnostics; - -using System.Management.Automation.Language; using Parser = System.Management.Automation.Language.Parser; using ScriptBlock = System.Management.Automation.ScriptBlock; using Token = System.Management.Automation.Language.Token; + #if LEGACYTELEMETRY using Microsoft.PowerShell.Telemetry.Internal; #endif @@ -825,6 +826,12 @@ namespace Microsoft.PowerShell.Commands } } + // Send telemetry on the imported modules + foreach (PSModuleInfo moduleInfo in remotelyImportedModules) + { + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, moduleInfo.Name); + } + return remotelyImportedModules; } @@ -1251,6 +1258,7 @@ namespace Microsoft.PowerShell.Commands foreach (RemoteDiscoveryHelper.CimModule remoteCimModule in remotePsCimModules) { ImportModule_RemotelyViaCimModuleData(importModuleOptions, remoteCimModule, cimSession); + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, remoteCimModule.ModuleName); } } @@ -1743,6 +1751,7 @@ namespace Microsoft.PowerShell.Commands // of doing Get-Module -list foreach (PSModuleInfo module in ModuleInfo) { + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, module.Name); RemoteDiscoveryHelper.DispatchModuleInfoProcessing( module, localAction: delegate () @@ -1772,6 +1781,7 @@ namespace Microsoft.PowerShell.Commands { foreach (Assembly suppliedAssembly in Assembly) { + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, suppliedAssembly.GetName().Name); ImportModule_ViaAssembly(importModuleOptions, suppliedAssembly); } } @@ -1785,6 +1795,8 @@ namespace Microsoft.PowerShell.Commands { SetModuleBaseForEngineModules(foundModule.Name, this.Context); + // Telemetry here - report module load + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, foundModule.Name); #if LEGACYTELEMETRY TelemetryAPI.ReportModuleLoad(foundModule); #endif @@ -1809,6 +1821,7 @@ namespace Microsoft.PowerShell.Commands BaseGuid = modulespec.Guid; PSModuleInfo foundModule = ImportModule_LocallyViaName(importModuleOptions, modulespec.Name); + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, modulespec.Name); if (foundModule != null) { SetModuleBaseForEngineModules(foundModule.Name, this.Context); @@ -1818,6 +1831,10 @@ namespace Microsoft.PowerShell.Commands else if (this.ParameterSetName.Equals(ParameterSet_FQName_ViaPsrpSession, StringComparison.OrdinalIgnoreCase)) { ImportModule_RemotelyViaPsrpSession(importModuleOptions, null, FullyQualifiedName, this.PSSession); + foreach (ModuleSpecification modulespec in FullyQualifiedName) + { + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ModuleLoad, modulespec.Name); + } } else { diff --git a/src/System.Management.Automation/engine/hostifaces/PowerShell.cs b/src/System.Management.Automation/engine/hostifaces/PowerShell.cs index c948b26b79..54ddd650e3 100644 --- a/src/System.Management.Automation/engine/hostifaces/PowerShell.cs +++ b/src/System.Management.Automation/engine/hostifaces/PowerShell.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; @@ -16,6 +17,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Management.Infrastructure; +using Microsoft.PowerShell.Telemetry; using Dbg = System.Management.Automation.Diagnostics; @@ -638,6 +640,7 @@ namespace System.Management.Automation Streams = new PSDataStreams(this); _endInvokeMethod = EndInvoke; _endStopMethod = EndStop; + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.PowerShellCreate, "create"); } /// diff --git a/src/System.Management.Automation/engine/pipeline.cs b/src/System.Management.Automation/engine/pipeline.cs index e8f566b871..4c50997030 100644 --- a/src/System.Management.Automation/engine/pipeline.cs +++ b/src/System.Management.Automation/engine/pipeline.cs @@ -7,6 +7,7 @@ using System.Management.Automation.Runspaces; using System.Management.Automation.Tracing; using System.Reflection; using System.Runtime.ExceptionServices; +using Microsoft.PowerShell.Telemetry; using Dbg = System.Management.Automation.Diagnostics; @@ -1025,6 +1026,10 @@ namespace System.Management.Automation.Internal CommandState.Started, commandProcessor.Command.MyInvocation); + // Telemetry here + // the type of command should be sent along + // commandProcessor.CommandInfo.CommandType + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ApplicationType, commandProcessor.Command.CommandInfo.CommandType.ToString()); #if LEGACYTELEMETRY Microsoft.PowerShell.Telemetry.Internal.TelemetryAPI.TraceExecutedCommand(commandProcessor.Command.CommandInfo, commandProcessor.Command.CommandOrigin); #endif diff --git a/src/System.Management.Automation/engine/remoting/client/RemoteRunspacePoolInternal.cs b/src/System.Management.Automation/engine/remoting/client/RemoteRunspacePoolInternal.cs index ed2ac19361..3c89fe5ac7 100644 --- a/src/System.Management.Automation/engine/remoting/client/RemoteRunspacePoolInternal.cs +++ b/src/System.Management.Automation/engine/remoting/client/RemoteRunspacePoolInternal.cs @@ -10,6 +10,7 @@ using System.Management.Automation.Remoting; using System.Management.Automation.Remoting.Client; using System.Management.Automation.Tracing; using System.Threading; +using Microsoft.PowerShell.Telemetry; using Dbg = System.Management.Automation.Diagnostics; #if LEGACYTELEMETRY @@ -854,6 +855,8 @@ namespace System.Management.Automation.Runspaces.Internal PSEtwLog.LogOperationalVerbose(PSEventId.RunspacePoolOpen, PSOpcode.Open, PSTask.CreateRunspace, PSKeyword.UseAlwaysOperational); + // Telemetry here - remote session + ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.RemoteSessionOpen, isAsync.ToString()); #if LEGACYTELEMETRY TelemetryAPI.ReportRemoteSessionCreated(_connectionInfo); #endif diff --git a/src/System.Management.Automation/utils/Telemetry.cs b/src/System.Management.Automation/utils/Telemetry.cs new file mode 100644 index 0000000000..c7d40c1377 --- /dev/null +++ b/src/System.Management.Automation/utils/Telemetry.cs @@ -0,0 +1,397 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Management.Automation; +using System.Runtime.InteropServices; +using System.Threading; + +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; + +namespace Microsoft.PowerShell.Telemetry +{ + /// + /// The category of telemetry. + /// + internal enum TelemetryType + { + /// + /// Telemetry of the application type (cmdlet, script, etc). + /// + ApplicationType, + + /// + /// Send telemetry when we load a module, only module names in the s_knownModules list + /// will be reported, otherwise it will be "anonymous". + /// + ModuleLoad, + + /// + /// Send telemetry for experimental module feature activation. + /// All experimental engine features will be have telemetry. + /// + ExperimentalEngineFeatureActivation, + + /// + /// Send telemetry for experimental module feature activation. + /// Experimental module features will send telemetry based on the module it is in. + /// If we send telemetry for the module, we will also do so for any experimental feature + /// in that module. + /// + ExperimentalModuleFeatureActivation, + + /// + /// Send telemetry for each PowerShell.Create API. + /// + PowerShellCreate, + + /// + /// Remote session creation. + /// + RemoteSessionOpen, + } + + /// + /// Send up telemetry for startup. + /// + public static class ApplicationInsightsTelemetry + { + // If this env var is true, yes, or 1, telemetry will NOT be sent. + private const string _telemetryOptoutEnvVar = "POWERSHELL_TELEMETRY_OPTOUT"; + + // PSCoreInsight2 telemetry key + // private const string _psCoreTelemetryKey = "ee4b2115-d347-47b0-adb6-b19c2c763808"; // Production + private const string _psCoreTelemetryKey = "d26a5ef4-d608-452c-a6b8-a4a55935f70d"; // V7 Preview 3 + + // Use "anonymous" as the string to return when you can't report a name + private const string _anonymous = "anonymous"; + + // the telemetry failure string + private const string _telemetryFailure = "TELEMETRY_FAILURE"; + + // Telemetry client to be reused when we start sending more telemetry + private static TelemetryClient s_telemetryClient { get; set; } + + // the unique identifier for the user, when we start we + private static string s_uniqueUserIdentifier { get; set; } + + // the session identifier + private static string s_sessionId { get; set; } + + /// Use a hashset for quick lookups. + /// We send telemetry only a known set of modules. + /// If it's not in the list (initialized in the static constructor), then we report anonymous. + private static HashSet s_knownModules; + + /// Gets a value indicating whether telemetry can be sent. + public static bool CanSendTelemetry { get; private set; } + + /// + /// Initializes static members of the class. + /// Static constructor determines whether telemetry is to be sent, and then + /// sets the telemetry key and set the telemetry delivery mode. + /// Creates the session ID and initializes the HashSet of known module names. + /// Gets or constructs the unique identifier. + /// + static ApplicationInsightsTelemetry() + { + // If we can't send telemetry, there's no reason to do any of this + CanSendTelemetry = !GetEnvironmentVariableAsBool(name: _telemetryOptoutEnvVar, defaultValue: false); + if (CanSendTelemetry) + { + s_telemetryClient = new TelemetryClient(); + TelemetryConfiguration.Active.InstrumentationKey = _psCoreTelemetryKey; + // Set this to true to reduce latency during development + TelemetryConfiguration.Active.TelemetryChannel.DeveloperMode = false; + s_sessionId = Guid.NewGuid().ToString(); + + // use a hashset when looking for module names, it should be quicker than a string comparison + s_knownModules = new HashSet(StringComparer.OrdinalIgnoreCase) { + "Microsoft.PowerShell.Archive", + "Microsoft.PowerShell.Host", + "Microsoft.PowerShell.Management", + "Microsoft.PowerShell.Security", + "Microsoft.PowerShell.Utility", + "PackageManagement", + "Pester", + "PowerShellGet", + "PSDesiredStateConfiguration", + "PSReadLine", + "ThreadJob", + }; + s_uniqueUserIdentifier = GetUniqueIdentifier().ToString(); + } + } + + /// + /// Determine whether the environment variable is set and how. + /// + /// The name of the environment variable. + /// If the environment variable is not set, use this as the default value. + /// A boolean representing the value of the environment variable. + private static bool GetEnvironmentVariableAsBool(string name, bool defaultValue) + { + var str = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrEmpty(str)) + { + return defaultValue; + } + + switch (str.ToLowerInvariant()) + { + case "true": + case "1": + case "yes": + return true; + case "false": + case "0": + case "no": + return false; + default: + return defaultValue; + } + } + + /// + /// Send telemetry as a metric. + /// + /// The type of telemetry that we'll be sending. + /// The specific details about the telemetry. + internal static void SendTelemetryMetric(TelemetryType metricId, string data) + { + if (!CanSendTelemetry) + { + return; + } + + string metricName = metricId.ToString(); + try + { + switch (metricId) + { + case TelemetryType.ApplicationType: + case TelemetryType.PowerShellCreate: + case TelemetryType.RemoteSessionOpen: + case TelemetryType.ExperimentalEngineFeatureActivation: + s_telemetryClient.GetMetric(metricName, "uuid", "SessionId", "Detail").TrackValue(metricValue: 1.0, s_uniqueUserIdentifier, s_sessionId, data); + break; + case TelemetryType.ExperimentalModuleFeatureActivation: + string experimentalFeatureName = GetExperimentalFeatureName(data); + s_telemetryClient.GetMetric(metricName, "uuid", "SessionId", "Detail").TrackValue(metricValue: 1.0, s_uniqueUserIdentifier, s_sessionId, experimentalFeatureName); + break; + case TelemetryType.ModuleLoad: + string moduleName = GetModuleName(data); // This will return anonymous if the modulename is not on the report list + s_telemetryClient.GetMetric(metricName, "uuid", "SessionId", "Detail").TrackValue(metricValue: 1.0, s_uniqueUserIdentifier, s_sessionId, moduleName); + break; + } + } + catch + { + // do nothing, telemetry can't be sent + // don't send the panic telemetry as if we have failed above, it will likely fail here. + } + + } + + // Get the experimental feature name. If we can report it, we'll return the name of the feature, otherwise, we'll return "anonymous" + private static string GetExperimentalFeatureName(string featureNameToValidate) + { + // An experimental feature in a module is guaranteed to start with the module name + // we can strip out the text past the last '.' as the text before that will be the ModuleName + int lastDotIndex = featureNameToValidate.LastIndexOf('.'); + string moduleName = featureNameToValidate.Substring(0, lastDotIndex); + if (s_knownModules.Contains(moduleName)) + { + return featureNameToValidate; + } + + return _anonymous; + } + + // Get the module name. If we can report it, we'll return the name, otherwise, we'll return "anonymous" + private static string GetModuleName(string moduleNameToValidate) + { + if (s_knownModules.Contains(moduleNameToValidate)) + { + return moduleNameToValidate; + } + + return _anonymous; + } + + /// + /// Create the startup payload and send it up. + /// This is done only once during for the console host. + /// + /// The "mode" of the startup. + internal static void SendPSCoreStartupTelemetry(string mode) + { + if (!CanSendTelemetry) + { + return; + } + var properties = new Dictionary(); + // The variable POWERSHELL_DISTRIBUTION_CHANNEL is set in our docker images. + // This allows us to track the actual docker OS as OSDescription provides only "linuxkit" + // which has limited usefulness + var channel = Environment.GetEnvironmentVariable("POWERSHELL_DISTRIBUTION_CHANNEL"); + + properties.Add("SessionId", s_sessionId); + properties.Add("UUID", s_uniqueUserIdentifier); + properties.Add("GitCommitID", PSVersionInfo.GitCommitId); + properties.Add("OSDescription", RuntimeInformation.OSDescription); + properties.Add("OSChannel", string.IsNullOrEmpty(channel) ? "unknown" : channel); + properties.Add("StartMode", string.IsNullOrEmpty(mode) ? "unknown" : mode); + try + { + s_telemetryClient.TrackEvent("ConsoleHostStartup", properties, null); + } + catch + { + // do nothing, telemetry cannot be sent + } + } + + /// + /// Try to read the file and collect the guid. + /// + /// The path to the telemetry file. + /// The newly created id. + /// + /// The method returns a bool indicating success or failure of creating the id. + /// + private static bool TryGetIdentifier(string telemetryFilePath, out Guid id) + { + if (File.Exists(telemetryFilePath)) + { + // attempt to read the persisted identifier + const int GuidSize = 16; + byte[] buffer = new byte[GuidSize]; + try + { + using (FileStream fs = new FileStream(telemetryFilePath, FileMode.Open, FileAccess.Read)) + { + // if the read is invalid, or wrong size, we return it + int n = fs.Read(buffer, 0, GuidSize); + if (n == GuidSize) + { + // it's possible this could through + id = new Guid(buffer); + if (id != Guid.Empty) + { + return true; + } + } + } + } + catch + { + // something went wrong, the file may not exist or not have enough bytes, so return false + } + } + + id = Guid.Empty; + return false; + } + + /// + /// Try to create a unique identifier and persist it to the telemetry.uuid file. + /// + /// The path to the persisted telemetry.uuid file. + /// The created identifier. + /// + /// The method returns a bool indicating success or failure of creating the id. + /// + private static bool TryCreateUniqueIdentifierAndFile(string telemetryFilePath, out Guid id) + { + // one last attempt to retrieve before creating incase we have a lot of simultaneous entry into the mutex. + id = Guid.Empty; + if (TryGetIdentifier(telemetryFilePath, out id)) + { + return true; + } + + // The directory may not exist, so attempt to create it + // CreateDirectory will simply return the directory if exists + try + { + Directory.CreateDirectory(Path.GetDirectoryName(telemetryFilePath)); + } + catch + { + // send a telemetry indicating a problem with the cache dir + // it's likely something is seriously wrong so we should at least report it. + // We don't want to provide reasons here, that's not the point, but we + // would like to know if we're having a generalized problem which we can trace statistically + CanSendTelemetry = false; + s_telemetryClient.GetMetric(_telemetryFailure, "Detail").TrackValue(1, "cachedir"); + return false; + } + + // Create and save the new identifier, and if there's a problem, disable telemetry + try + { + id = Guid.NewGuid(); + File.WriteAllBytes(telemetryFilePath, id.ToByteArray()); + return true; + } + catch + { + // another bit of telemetry to notify us about a problem with saving the unique id. + s_telemetryClient.GetMetric(_telemetryFailure, "Detail").TrackValue(1, "saveuuid"); + } + + return false; + } + + /// + /// Retrieve the unique identifier from the persisted file, if it doesn't exist create it. + /// Generate a guid which will be used as the UUID. + /// + /// A guid which represents the unique identifier. + private static Guid GetUniqueIdentifier() + { + // Try to get the unique id. If this returns false, we'll + // create/recreate the telemetry.uuid file to persist for next startup. + Guid id = Guid.Empty; + string uuidPath = Path.Join(Platform.CacheDirectory, "telemetry.uuid"); + if (TryGetIdentifier(uuidPath, out id)) + { + return id; + } + + // Multiple processes may start simultaneously so we need a system wide + // way to control access to the file in the case (although remote) when we have + // simulataneous shell starts without the persisted file which attempt to create the file. + using (var m = new Mutex(true, "CreateUniqueUserId")) + { + // TryCreateUniqueIdentifierAndFile shouldn't throw, but the mutex might + try + { + m.WaitOne(); + if (TryCreateUniqueIdentifierAndFile(uuidPath, out id)) + { + return id; + } + } + catch (Exception) + { + // Any problem in generating a uuid will result in no telemetry being sent. + // Try to send the failure in telemetry, but it will have no unique id. + s_telemetryClient.GetMetric(_telemetryFailure, "Detail").TrackValue(1, "mutex"); + } + finally + { + m.ReleaseMutex(); + } + } + + // something bad happened, turn off telemetry since the unique id wasn't set. + CanSendTelemetry = false; + return id; + } + } +} diff --git a/test/common/markdown/markdown-link.tests.ps1 b/test/common/markdown/markdown-link.tests.ps1 index deb9d73e29..c60592e537 100644 --- a/test/common/markdown/markdown-link.tests.ps1 +++ b/test/common/markdown/markdown-link.tests.ps1 @@ -95,6 +95,11 @@ Describe "Verify Markdown Links" { it " should work" -TestCases $trueFailures { param($url) + # there could be multiple reasons why a failure is ok + # check against the allowed failures + # 503 = service temporarily unavailable + $allowedFailures = @( 503 ) + $prefix = $url.Substring(0,7) # Logging for diagnosability. Azure DevOps sometimes redacts the full url. @@ -108,7 +113,9 @@ Describe "Verify Markdown Links" { } catch { - throw "retry of URL failed with error: $($_.Message)" + if ( $allowedFailures -notcontains $_.Exception.Response.StatusCode ) { + throw "retry of URL failed with error: $($_.Exception.Message)" + } } } else { diff --git a/test/powershell/Host/Startup.Tests.ps1 b/test/powershell/Host/Startup.Tests.ps1 index 62fba40764..41c1749c57 100644 --- a/test/powershell/Host/Startup.Tests.ps1 +++ b/test/powershell/Host/Startup.Tests.ps1 @@ -43,7 +43,6 @@ Describe "Validate start of console host" -Tag CI { 'System.Private.CoreLib.dll' 'System.Private.Uri.dll' 'System.Private.Xml.dll' - 'System.Private.Xml.Linq.dll' 'System.Reflection.Emit.ILGeneration.dll' 'System.Reflection.Emit.Lightweight.dll' 'System.Reflection.Primitives.dll' @@ -67,9 +66,7 @@ Describe "Validate start of console host" -Tag CI { 'System.Threading.Tasks.Parallel.dll' 'System.Threading.Thread.dll' 'System.Threading.ThreadPool.dll' - 'System.Threading.Timer.dll' 'System.Xml.ReaderWriter.dll' - 'System.Xml.XDocument.dll' ) if ($IsWindows) { diff --git a/test/powershell/engine/Basic/Telemetry.Tests.ps1 b/test/powershell/engine/Basic/Telemetry.Tests.ps1 new file mode 100644 index 0000000000..0bc233bb74 --- /dev/null +++ b/test/powershell/engine/Basic/Telemetry.Tests.ps1 @@ -0,0 +1,131 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# unit tests for telemetry +# these tests aren't going to check that telemetry is being sent +# only that we're not treating the telemetry.uuid file correctly + +Describe "Telemetry for shell startup" -Tag CI { + BeforeAll { + # if the telemetry file exists, move it out of the way + # the member is internal, but we can retrieve it via reflection + $cacheDir = [System.Management.Automation.Platform].GetMember("CacheDirectory","NonPublic,Static").GetMethod.Invoke($null, $null) + $uuidPath = Join-Path -Path $cacheDir -ChildPath telemetry.uuid + $uuidFileExists = Test-Path -Path $uuidPath + if ( $uuidFileExists ) { + $originalBytes = Get-Content -AsByteStream -Path $uuidPath + Rename-Item -Path $uuidPath -NewName "${uuidPath}.original" + } + + $PWSH = (Get-Process -Id $PID).MainModule.FileName + $telemetrySet = Test-Path -Path env:POWERSHELL_TELEMETRY_OPTOUT + $SendingTelemetry = $env:POWERSHELL_TELEMETRY_OPTOUT + } + + AfterAll { + # check and reset the telemetry.uuid file + if ( $uuidFileExists ) { + if ( Test-Path -Path "${uuidPath}.original" ) { + Rename-Item -NewName $uuidPath -Path "${uuidPath}.original" -Force + } + else { + [System.IO.File]::WriteAllBytes($uuidPath, $originalBytes) + } + } + if ( $telemetrySet ) { + $env:POWERSHELL_TELEMETRY_OPTOUT = $SendingTelemetry + } + } + + AfterEach { + if ( Test-Path -Path $uuidPath ) { + Remove-Item -Path $uuidPath + } + if ( Test-Path -Path env:POWERSHELL_TELEMETRY_OPTOUT ) { + Remove-Item env:POWERSHELL_TELEMETRY_OPTOUT + } + } + + It "Should not create a uuid file if telemetry is opted out" { + $env:POWERSHELL_TELEMETRY_OPTOUT = 1 + & $PWSH -NoProfile -Command "exit" + $uuidPath | Should -Not -Exist + } + + It "Should create a uuid file if telemetry is opted in" { + $env:POWERSHELL_TELEMETRY_OPTOUT = "no" + & $PWSH -NoProfile -Command "exit" + $uuidPath | Should -Exist + } + + It "Should create a uuid file by default" { + if ( Test-Path env:POWERSHELL_TELEMETRY_OPTOUT ) { Remove-Item -Path env:POWERSHELL_TELEMETRY_OPTOUT } + & $PWSH -NoProfile -Command "exit" + $uuidPath | Should -Exist + } + + It "Should create a property uuid file when telemetry is sent" { + $env:POWERSHELL_TELEMETRY_OPTOUT = "no" + & $PWSH -NoProfile -Command "exit" + $uuidPath | Should -Exist + (Get-ChildItem -Path $uuidPath).Length | Should -Be 16 + [byte[]]$newBytes = Get-Content -AsByteStream -Path $uuidPath + [System.Guid]::New($newBytes) | Should -BeOfType [System.Guid] + } + + It "Should not create a telemetry file if one already exists and telemetry is opted in" { + [byte[]]$bytes = [System.Guid]::NewGuid().ToByteArray() + [System.IO.File]::WriteAllBytes($uuidPath, $bytes) + & $PWSH -NoProfile -Command "exit" + [byte[]]$newBytes = Get-Content -AsByteStream -Path $uuidPath + Compare-Object -ReferenceObject $bytes -DifferenceObject $newBytes | Should -BeNullOrEmpty + } + + It "Should create a new telemetry file if the current one is 00000000-0000-0000-0000-000000000000" { + [byte[]]$zeroGuid = [System.Guid]::Empty.ToByteArray() + [System.IO.File]::WriteAllBytes($uuidPath, $zeroGuid) + & $PWSH -NoProfile -Command "exit" + [byte[]]$newBytes = Get-Content -AsByteStream -Path $uuidPath + # we could legitimately have zeros in the new guid, so we can't check for that + # we're just making sure that there *is* a difference + Compare-Object -ReferenceObject $zeroGuid -DifferenceObject $newBytes | Should -Not -BeNullOrEmpty + } + + It "Should create a new telemetry file if the current one is smaller than 16 bytes" { + $badBytes = [byte[]]::new(8); + [System.IO.File]::WriteAllBytes($uuidPath, $badBytes) + & $PWSH -NoProfile -Command "exit" + [byte[]]$nb = Get-Content -AsByteStream -Path $uuidPath + [System.Guid]::New($nb) | Should -BeOfType [System.Guid] + } + + It "Should not create a new telemetry file if the current one has a valid guid and is larger than 16 bytes" { + $g = [Guid]::newGuid() + $tooManyBytes = $g.ToByteArray() * 2 + [System.IO.File]::WriteAllBytes($uuidPath, $tooManyBytes) + [byte[]]$nb = Get-Content -Path $uuidPath -AsByteStream | Select-Object -First 16 + $ng = [System.Guid]::new($nb) + $g | Should -Be $ng + } + + It "Should properly set whether telemetry is sent based on when environment variable is not set" { + $result = & $PWSH -NoProfile -Command '[Microsoft.PowerShell.Telemetry.ApplicationInsightsTelemetry]::CanSendTelemetry' + $result | Should -Be "True" + } + + $telemetryIsSetData = @( + @{ name = "set to no"; Value = "no" ; expectedValue = "True" } + @{ name = "set to 0"; Value = "0"; expectedValue = "True" } + @{ name = "set to false"; Value = "false"; expectedValue = "True" } + @{ name = "set to yes"; Value = "yes"; expectedValue = "False" } + @{ name = "set to 1"; Value = "1"; expectedValue = "False" } + @{ name = "set to true"; Value = "true"; expectedValue = "False" } + ) + + It "Should properly set whether telemetry is sent based on environment variable when " -TestCases $telemetryIsSetData { + param ( [string]$name, [string]$value, [string]$expectedValue ) + $env:POWERSHELL_TELEMETRY_OPTOUT = $value + $result = & $PWSH -NoProfile -Command '[Microsoft.PowerShell.Telemetry.ApplicationInsightsTelemetry]::CanSendTelemetry' + $result | Should -Be $expectedValue + } +}