Add telemetry for when user triggers testing from CLI

src/client/telemetry/constants.ts

@@ -49,6 +49,7 @@ export enum EventName {
     UNITTEST_RUN = 'UNITTEST.RUN',
     UNITTEST_RUN_ALL_FAILED = 'UNITTEST.RUN_ALL_FAILED',
     UNITTEST_DISABLED = 'UNITTEST.DISABLED',
+    UNITTEST_RUN_CLI = 'UNITTEST.RUN.CLI',
 
     PYTHON_EXPERIMENTS_INIT_PERFORMANCE = 'PYTHON_EXPERIMENTS_INIT_PERFORMANCE',
     PYTHON_EXPERIMENTS_LSP_NOTEBOOKS = 'PYTHON_EXPERIMENTS_LSP_NOTEBOOKS',

src/client/telemetry/index.ts

@@ -2174,6 +2174,13 @@ export interface IEventNamePropertyMapping {
        "unittest.disabled" : { "owner": "eleanorjboyd" }
      */
     [EventName.UNITTEST_DISABLED]: never | undefined;
+    /**
+     * Telemetry event sent when a user runs tests via the CLI in terminal.
+     */
+    /* __GDPR__
+       "unittest.run.cli" : { "owner": "eleanorjboyd" }
+     */
+    [EventName.UNITTEST_RUN_CLI]: never | undefined;
     /*
     Telemetry event sent to provide information on whether we have successfully identify the type of shell used.
     This information is useful in determining how well we identify shells on users machines.

src/client/terminals/codeExecution/terminalReplWatcher.ts

@@ -3,8 +3,20 @@ import { onDidStartTerminalShellExecution } from '../../common/vscodeApis/window
 import { sendTelemetryEvent } from '../../telemetry';
 import { EventName } from '../../telemetry/constants';
 
-function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runningScript` {
+function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runningScript` | 'runningTest' {
     const lower = command.toLowerCase().trimStart();
+
+    // Check for test commands
+    if (
+        lower.includes('pytest') ||
+        (lower.startsWith('python') && (lower.includes(' -m pytest') || lower.includes(' -m unittest'))) ||
+        (lower.startsWith('py ') && (lower.includes(' -m pytest') || lower.includes(' -m unittest'))) ||
+        lower.includes('py.test')
+    ) {
+        return 'runningTest';
+    }
+
+    // Regular Python commands
     if (lower.startsWith('python') || lower.startsWith('py ')) {
         const parts = lower.split(' ');
         if (parts.length === 1) {
@@ -20,7 +32,12 @@ export function registerTriggerForTerminalREPL(disposables: Disposable[]): void
         onDidStartTerminalShellExecution(async (e: TerminalShellExecutionStartEvent) => {
             const replType = checkREPLCommand(e.execution.commandLine.value);
             if (e.execution.commandLine.isTrusted && replType) {
-                sendTelemetryEvent(EventName.REPL, undefined, { replType });
+                // Send test-specific telemetry if it's a test command
+                if (replType === 'runningTest') {
+                    sendTelemetryEvent(EventName.UNITTEST_RUN_CLI);
+                } else {
+                    sendTelemetryEvent(EventName.REPL, undefined, { replType });
+                }
             }
         }),
     );

src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts

@@ -0,0 +1,122 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { expect } from 'chai';
+import * as sinon from 'sinon';
+import * as windowApis from '../../../client/common/vscodeApis/windowApis';
+import * as telemetryModule from '../../../client/telemetry';
+import { EventName } from '../../../client/telemetry/constants';
+import { registerTriggerForTerminalREPL } from '../../../client/terminals/codeExecution/terminalReplWatcher';
+
+suite('Terminal REPL Watcher', () => {
+    let windowApisStub: sinon.SinonStub;
+    let telemetryStub: sinon.SinonStub;
+
+    setup(() => {
+        windowApisStub = sinon.stub(windowApis, 'onDidStartTerminalShellExecution').returns({
+            dispose: () => {
+                // Do nothing
+            },
+        });
+        telemetryStub = sinon.stub(telemetryModule, 'sendTelemetryEvent');
+    });
+
+    teardown(() => {
+        sinon.restore();
+    });
+
+    test('Should send REPL telemetry when Python is invoked', () => {
+        windowApisStub.callsFake((callback) => {
+            callback({
+                execution: {
+                    commandLine: {
+                        value: 'python script.py',
+                        isTrusted: true,
+                    },
+                },
+            });
+            return {
+                dispose: () => {
+                    // Do nothing
+                },
+            };
+        });
+
+        registerTriggerForTerminalREPL([]);
+
+        expect(telemetryStub.calledOnce).to.be.true;
+        expect(telemetryStub.args[0][0]).to.equal(EventName.REPL);
+        expect(telemetryStub.args[0][2]).to.deep.equal({ replType: 'runningScript' });
+    });
+
+    test('Should send unittest CLI telemetry when pytest is invoked', () => {
+        windowApisStub.callsFake((callback) => {
+            callback({
+                execution: {
+                    commandLine: {
+                        value: 'python -m pytest',
+                        isTrusted: true,
+                    },
+                },
+            });
+            return {
+                dispose: () => {
+                    // Do nothing
+                },
+            };
+        });
+
+        registerTriggerForTerminalREPL([]);
+
+        expect(telemetryStub.calledOnce).to.be.true;
+        expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI);
+    });
+
+    test('Should send unittest CLI telemetry when unittest is invoked', () => {
+        windowApisStub.callsFake((callback) => {
+            callback({
+                execution: {
+                    commandLine: {
+                        value: 'python -m unittest discover',
+                        isTrusted: true,
+                    },
+                },
+            });
+            return {
+                dispose: () => {
+                    // Do nothing
+                },
+            };
+        });
+
+        registerTriggerForTerminalREPL([]);
+
+        expect(telemetryStub.calledOnce).to.be.true;
+        expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI);
+    });
+
+    test('Should send unittest CLI telemetry when py.test is invoked', () => {
+        windowApisStub.callsFake((callback) => {
+            callback({
+                execution: {
+                    commandLine: {
+                        value: 'py.test',
+                        isTrusted: true,
+                    },
+                },
+            });
+            return {
+                dispose: () => {
+                    // Do nothing
+                },
+            };
+        });
+
+        registerTriggerForTerminalREPL([]);
+
+        expect(telemetryStub.calledOnce).to.be.true;
+        expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI);
+    });
+});