diff --git a/pythonFiles/normalizeSelection.py b/pythonFiles/normalizeSelection.py deleted file mode 100644 index 8ba3a4b966b..00000000000 --- a/pythonFiles/normalizeSelection.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import ast -import json -import re -import sys -import textwrap - - -def split_lines(source): - """ - Split selection lines in a version-agnostic way. - - Python grammar only treats \r, \n, and \r\n as newlines. - But splitlines() in Python 3 has a much larger list: for example, it also includes \v, \f. - As such, this function will split lines across all Python versions. - """ - return re.split(r"[\n\r]+", source) - - -def _get_statements(selection): - """ - Process a multiline selection into a list of its top-level statements. - This will remove empty newlines around and within the selection, dedent it, - and split it using the result of `ast.parse()`. - """ - - # Remove blank lines within the selection to prevent the REPL from thinking the block is finished. - lines = (line for line in split_lines(selection) if line.strip() != "") - - # Dedent the selection and parse it using the ast module. - # Note that leading comments in the selection will be discarded during parsing. - source = textwrap.dedent("\n".join(lines)) - tree = ast.parse(source) - - # We'll need the dedented lines to rebuild the selection. - lines = split_lines(source) - - # Get the line ranges for top-level blocks returned from parsing the dedented text - # and split the selection accordingly. - # tree.body is a list of AST objects, which we rely on to extract top-level statements. - # If we supported Python 3.8+ only we could use the lineno and end_lineno attributes of each object - # to get the boundaries of each block. - # However, earlier Python versions only have the lineno attribute, which is the range start position (1-indexed). - # Therefore, to retrieve the end line of each block in a version-agnostic way we need to do - # `end = next_block.lineno - 1` - # for all blocks except the last one, which will will just run until the last line. - ends = [] - for node in tree.body[1:]: - line_end = node.lineno - 1 - # Special handling of decorators: - # In Python 3.8 and higher, decorators are not taken into account in the value returned by lineno, - # and we have to use the length of the decorator_list array to compute the actual start line. - # Before that, lineno takes into account decorators, so this offset check is unnecessary. - # Also, not all AST objects can have decorators. - if hasattr(node, "decorator_list") and sys.version_info >= (3, 8): - # Using getattr instead of node.decorator_list or pyright will complain about an unknown member. - line_end -= len(getattr(node, "decorator_list")) - ends.append(line_end) - ends.append(len(lines)) - for node, end in zip(tree.body, ends): - # Given this selection: - # 1: if (m > 0 and - # 2: n < 3): - # 3: print('foo') - # 4: value = 'bar' - # - # The first block would have lineno = 1,and the second block lineno = 4 - start = node.lineno - 1 - if start == end: - # "a=3; b=4" - continue - # Special handling of decorators similar to what's above. - if hasattr(node, "decorator_list") and sys.version_info >= (3, 8): - # Using getattr instead of node.decorator_list or pyright will complain about an unknown member. - start -= len(getattr(node, "decorator_list")) - block = "\n".join(lines[start:end]) - - # If the block is multiline, add an extra newline character at its end. - # This way, when joining blocks back together, there will be a blank line between each multiline statement - # and no blank lines between single-line statements, or it would look like this: - # >>> x = 22 - # >>> - # >>> total = x + 30 - # >>> - # Note that for the multiline parentheses case this newline is redundant, - # since the closing parenthesis terminates the statement already. - # This means that for this pattern we'll end up with: - # >>> x = [ - # ... 1 - # ... ] - # >>> - # >>> y = [ - # ... 2 - # ...] - if end - start > 1: - block += "\n" - - yield block - - -def normalize_lines(selection): - """ - Normalize the text selection received from the extension. - - If it is a single line selection, dedent it and append a newline and - send it back to the extension. - Otherwise, sanitize the multiline selection before returning it: - split it in a list of top-level statements - and add newlines between each of them so the REPL knows where each block ends. - """ - try: - # Parse the selection into a list of top-level blocks. - # We don't differentiate between single and multiline statements - # because it's not a perf bottleneck, - # and the overhead from splitting and rejoining strings in the multiline case is one-off. - statements = _get_statements(selection) - - # Insert a newline between each top-level statement, and append a newline to the selection. - source = "\n".join(statements) + "\n" - except Exception: - # If there's a problem when parsing statements, - # append a blank line to end the block and send it as-is. - source = selection + "\n\n" - - return source - - -if __name__ == "__main__": - # Content is being sent from the extension as a JSON object. - # Decode the data from the raw bytes. - stdin = sys.stdin if sys.version_info < (3,) else sys.stdin.buffer - raw = stdin.read() - contents = json.loads(raw.decode("utf-8")) - - normalized = normalize_lines(contents["code"]) - - # Send the normalized code back to the extension in a JSON object. - data = json.dumps({"normalized": normalized}) - - stdout = sys.stdout if sys.version_info < (3,) else sys.stdout.buffer - stdout.write(data.encode("utf-8")) - stdout.close() diff --git a/src/interactive-window/editor-integration/codewatcher.ts b/src/interactive-window/editor-integration/codewatcher.ts index 7aee63217b3..e1c2425ee2e 100644 --- a/src/interactive-window/editor-integration/codewatcher.ts +++ b/src/interactive-window/editor-integration/codewatcher.ts @@ -283,16 +283,7 @@ export class CodeWatcher implements ICodeWatcher { return; } - const normalize = - this.configService.getSettings(activeEditor.document.uri).normalizeSelectionForInteractiveWindow ?? - true; - const normalizedCode = normalize - ? await this.executionHelper.normalizeLines(codeToExecute!) - : codeToExecute; - if (!normalizedCode || normalizedCode.trim().length === 0) { - return; - } - await this.addCode(iw, normalizedCode, this.document.uri, activeEditor.selection.start.line); + await this.addCode(iw, codeToExecute, this.document.uri, activeEditor.selection.start.line); } } diff --git a/src/interactive-window/editor-integration/codewatcher.unit.test.ts b/src/interactive-window/editor-integration/codewatcher.unit.test.ts index d0e0638cd72..64ffa1887d0 100644 --- a/src/interactive-window/editor-integration/codewatcher.unit.test.ts +++ b/src/interactive-window/editor-integration/codewatcher.unit.test.ts @@ -836,7 +836,6 @@ testing2`; ) ) .returns(() => 'testing2'); - helper.setup((h) => h.normalizeLines(TypeMoq.It.isAny())).returns((x: string) => Promise.resolve(x)); // Set up our expected calls to add code activeInteractiveWindow @@ -881,7 +880,6 @@ testing2`; ) ) .returns(() => 'testing2'); - helper.setup((h) => h.normalizeLines(TypeMoq.It.isAny())).returns((x: string) => Promise.resolve(x)); // Set up our expected calls to add code activeInteractiveWindow diff --git a/src/platform/common/configSettings.ts b/src/platform/common/configSettings.ts index ad714fcfa9c..15385ad1757 100644 --- a/src/platform/common/configSettings.ts +++ b/src/platform/common/configSettings.ts @@ -45,7 +45,6 @@ export class JupyterSettings implements IWatchableJupyterSettings { public notebookFileRoot: string = ''; public useDefaultConfigForJupyter: boolean = false; public sendSelectionToInteractiveWindow: boolean = false; - public normalizeSelectionForInteractiveWindow: boolean = true; public splitRunFileIntoCells: boolean = true; public markdownRegularExpression: string = ''; public codeRegularExpression: string = ''; diff --git a/src/platform/common/constants.ts b/src/platform/common/constants.ts index 6e2e6f44fe2..6d718d90fa5 100644 --- a/src/platform/common/constants.ts +++ b/src/platform/common/constants.ts @@ -360,7 +360,6 @@ export enum Telemetry { PandasTooOld = 'DS_INTERNAL.SHOW_DATA_PANDAS_TOO_OLD', PandasOK = 'DS_INTERNAL.SHOW_DATA_PANDAS_OK', PandasInstallCanceled = 'DS_INTERNAL.SHOW_DATA_PANDAS_INSTALL_CANCELED', - DataScienceSettings = 'DS_INTERNAL.SETTINGS', VariableExplorerVariableCount = 'DS_INTERNAL.VARIABLE_EXPLORER_VARIABLE_COUNT', AddCellBelow = 'DATASCIENCE.ADD_CELL_BELOW', GetPasswordFailure = 'DS_INTERNAL.GET_PASSWORD_FAILURE', diff --git a/src/platform/common/types.ts b/src/platform/common/types.ts index 0418e6e7fb1..9fd66c36ae9 100644 --- a/src/platform/common/types.ts +++ b/src/platform/common/types.ts @@ -53,7 +53,6 @@ export interface IJupyterSettings { readonly useDefaultConfigForJupyter: boolean; readonly enablePythonKernelLogging: boolean; readonly sendSelectionToInteractiveWindow: boolean; - readonly normalizeSelectionForInteractiveWindow: boolean; readonly splitRunFileIntoCells: boolean; readonly markdownRegularExpression: string; readonly codeRegularExpression: string; diff --git a/src/platform/interpreter/internal/scripts/index.node.ts b/src/platform/interpreter/internal/scripts/index.node.ts index 340825d33cd..92e40292096 100644 --- a/src/platform/interpreter/internal/scripts/index.node.ts +++ b/src/platform/interpreter/internal/scripts/index.node.ts @@ -6,42 +6,3 @@ import { EXTENSION_ROOT_DIR } from '../../../constants.node'; // It is simpler to hard-code it instead of using vscode.ExtensionContext.extensionPath. export const _SCRIPTS_DIR = path.join(EXTENSION_ROOT_DIR, 'pythonFiles'); -const SCRIPTS_DIR = _SCRIPTS_DIR; - -// "scripts" contains everything relevant to the scripts found under -// the top-level "pythonFiles" directory. Each of those scripts has -// a function in this module which matches the script's filename. -// Each function provides the commandline arguments that should be -// used when invoking a Python executable, whether through spawn/exec -// or a terminal. -// -// Where relevant (nearly always), the function also returns a "parse" -// function that may be used to deserialize the stdout of the script -// into the corresponding object or objects. "parse()" takes a single -// string as the stdout text and returns the relevant data. -// -// Some of the scripts are located in subdirectories of "pythonFiles". -// For each of those subdirectories there is a sub-module where -// those scripts' functions may be found. -// -// In some cases one or more types related to a script are exported -// from the same module in which the script's function is located. -// These types typically relate to the return type of "parse()". -// -// ignored scripts: -// * install_debugpy.py (used only for extension development) - -//============================ -// normalizeSelection.py - -export function normalizeSelection(): [string[], (out: string) => string] { - const script = path.join(SCRIPTS_DIR, 'normalizeSelection.py'); - const args = [script]; - - function parse(out: string) { - // The text will be used as-is. - return out; - } - - return [args, parse]; -} diff --git a/src/platform/terminals/codeExecution/codeExecutionHelper.node.ts b/src/platform/terminals/codeExecution/codeExecutionHelper.node.ts deleted file mode 100644 index 167513100f2..00000000000 --- a/src/platform/terminals/codeExecution/codeExecutionHelper.node.ts +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { inject, injectable } from 'inversify'; -import { Uri } from 'vscode'; - -import { logger } from '../../logging'; -import * as internalScripts from '../../interpreter/internal/scripts/index.node'; -import { getFilePath } from '../../common/platform/fs-paths'; -import { CodeExecutionHelperBase } from './codeExecutionHelper'; -import { IProcessServiceFactory } from '../../common/process/types.node'; -import { IInterpreterService } from '../../interpreter/contracts'; -import { IServiceContainer } from '../../ioc/types'; -import { splitLines } from '../../common/helpers'; -import { IDisposable } from '../../common/types'; -import { dispose } from '../../common/utils/lifecycle'; - -/** - * Node version of the code execution helper. Node version is necessary because we can't create processes in the web version. - */ -@injectable() -export class CodeExecutionHelper extends CodeExecutionHelperBase { - private readonly interpreterService: IInterpreterService; - private readonly processServiceFactory: IProcessServiceFactory; - - constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { - super(); - this.processServiceFactory = serviceContainer.get(IProcessServiceFactory); - this.interpreterService = serviceContainer.get(IInterpreterService); - } - - public override async normalizeLines(code: string, resource?: Uri): Promise { - const disposables: IDisposable[] = []; - try { - const codeTrimmed = code.trim(); - if (codeTrimmed.length === 0) { - return ''; - } - // On windows cr is not handled well by python when passing in/out via stdin/stdout. - // So just remove cr from the input. - code = code.replace(new RegExp('\\r', 'g'), ''); - if (codeTrimmed.indexOf('\n') === -1) { - // the input is a single line, maybe indented, without terminator - return codeTrimmed + '\n'; - } - - const interpreter = await this.interpreterService.getActiveInterpreter(resource); - const processService = await this.processServiceFactory.create(resource); - - const [args, parse] = internalScripts.normalizeSelection(); - const observable = processService.execObservable(getFilePath(interpreter?.uri) || 'python', args, { - throwOnStdErr: true - }); - - // Read result from the normalization script from stdout, and resolve the promise when done. - let normalized = ''; - observable.out.onDidChange( - (output) => { - if (output.source === 'stdout') { - normalized += output.out; - } - }, - this, - disposables - ); - - // The normalization script expects a serialized JSON object, with the selection under the "code" key. - // We're using a JSON object so that we don't have to worry about encoding, or escaping non-ASCII characters. - const input = JSON.stringify({ code }); - observable.proc?.stdin?.write(input); - observable.proc?.stdin?.end(); - - // We expect a serialized JSON object back, with the normalized code under the "normalized" key. - await observable.out.done; - const result = normalized; - const object = JSON.parse(result); - - const normalizedLines = parse(object.normalized); - // Python will remove leading empty spaces, add them back. - const indexOfFirstNonEmptyLineInOriginalCode = splitLines(code, { - trim: true, - removeEmptyEntries: false - }).findIndex((line) => line.length); - const indexOfFirstNonEmptyLineInNormalizedCode = splitLines(normalizedLines, { - trim: true, - removeEmptyEntries: false - }).findIndex((line) => line.length); - if (indexOfFirstNonEmptyLineInOriginalCode > indexOfFirstNonEmptyLineInNormalizedCode) { - // Some white space has been trimmed, add them back. - const trimmedLineCount = - indexOfFirstNonEmptyLineInOriginalCode - indexOfFirstNonEmptyLineInNormalizedCode; - return `${'\n'.repeat(trimmedLineCount)}${normalizedLines}`; - } - return normalizedLines; - } catch (ex) { - logger.error(ex, 'Python: Failed to normalize code for execution in Interactive Window'); - return code; - } finally { - dispose(disposables); - } - } -} diff --git a/src/platform/terminals/codeExecution/codeExecutionHelper.ts b/src/platform/terminals/codeExecution/codeExecutionHelper.ts index d7df606143e..a797ae1d18e 100644 --- a/src/platform/terminals/codeExecution/codeExecutionHelper.ts +++ b/src/platform/terminals/codeExecution/codeExecutionHelper.ts @@ -11,10 +11,7 @@ import { dedentCode } from '../../common/helpers'; /** * Handles trimming code sent to a terminal so it actually runs. */ -export class CodeExecutionHelperBase implements ICodeExecutionHelper { - public async normalizeLines(code: string, _resource?: Uri): Promise { - return code; - } +export class CodeExecutionHelper implements ICodeExecutionHelper { public async getFileToExecute(): Promise { const activeEditor = window.activeTextEditor; diff --git a/src/platform/terminals/codeExecution/codeExecutionHelper.unit.test.ts b/src/platform/terminals/codeExecution/codeExecutionHelper.unit.test.ts index 203d342f93e..df4387e6952 100644 --- a/src/platform/terminals/codeExecution/codeExecutionHelper.unit.test.ts +++ b/src/platform/terminals/codeExecution/codeExecutionHelper.unit.test.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import * as assert from 'assert'; -import { CodeExecutionHelperBase } from './codeExecutionHelper'; +import { CodeExecutionHelper } from './codeExecutionHelper'; import { MockEditor } from '../../../test/datascience/mockTextEditor'; import { Uri, Selection } from 'vscode'; import { MockDocumentManager } from '../../../test/datascience/mockDocumentManager'; @@ -28,49 +28,49 @@ if (true): suite('Normalize selected text for execution', () => { test('Normalize first line including newline', () => { const editor = initializeMockTextEditor(inputText, new Selection(0, 0, 1, 0)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal('print(1)', text); }); test('Normalize several lines', () => { const editor = initializeMockTextEditor(inputText, new Selection(0, 0, 7, 0)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal(inputText.trimEnd(), text); }); test('Normalize indented lines', () => { const editor = initializeMockTextEditor(inputText, new Selection(3, 0, 5, 0)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal('print(2)\nprint(3)', text); }); test('Normalize indented lines but first line partially selected', () => { const editor = initializeMockTextEditor(inputText, new Selection(3, 3, 5, 0)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal('print(2)\nprint(3)', text); }); test('Normalize single indented line', () => { const editor = initializeMockTextEditor(inputText, new Selection(3, 4, 3, 12)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal('print(2)', text); }); test('Normalize indented line including leading newline', () => { const editor = initializeMockTextEditor(inputText, new Selection(3, 12, 4, 12)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal('\nprint(3)', text); }); test('Normalize a multi-line string', () => { const editor = initializeMockTextEditor(inputText, new Selection(5, 0, 7, 0)); - const helper = new CodeExecutionHelperBase(); + const helper = new CodeExecutionHelper(); const text = helper.getSelectedTextToExecute(editor); assert.equal("print('''a multiline\nstring''')", text); }); diff --git a/src/platform/terminals/codeExecution/codeExecutionHelper.web.ts b/src/platform/terminals/codeExecution/codeExecutionHelper.web.ts deleted file mode 100644 index 0da173ba3fd..00000000000 --- a/src/platform/terminals/codeExecution/codeExecutionHelper.web.ts +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { injectable } from 'inversify'; -import { CodeExecutionHelperBase } from './codeExecutionHelper'; - -@injectable() -export class CodeExecutionHelper extends CodeExecutionHelperBase {} diff --git a/src/platform/terminals/serviceRegistry.node.ts b/src/platform/terminals/serviceRegistry.node.ts index 003448ea5f6..12c6d45f606 100644 --- a/src/platform/terminals/serviceRegistry.node.ts +++ b/src/platform/terminals/serviceRegistry.node.ts @@ -3,8 +3,8 @@ import { interfaces } from 'inversify'; import { ClassType } from '../ioc/types'; -import { CodeExecutionHelper } from './codeExecution/codeExecutionHelper.node'; import { ICodeExecutionHelper } from './types'; +import { CodeExecutionHelper } from './codeExecution/codeExecutionHelper'; interface IServiceRegistry { addSingleton( diff --git a/src/platform/terminals/serviceRegistry.web.ts b/src/platform/terminals/serviceRegistry.web.ts index f6545b65743..12c6d45f606 100644 --- a/src/platform/terminals/serviceRegistry.web.ts +++ b/src/platform/terminals/serviceRegistry.web.ts @@ -3,8 +3,8 @@ import { interfaces } from 'inversify'; import { ClassType } from '../ioc/types'; -import { CodeExecutionHelper } from './codeExecution/codeExecutionHelper.web'; import { ICodeExecutionHelper } from './types'; +import { CodeExecutionHelper } from './codeExecution/codeExecutionHelper'; interface IServiceRegistry { addSingleton( diff --git a/src/platform/terminals/types.ts b/src/platform/terminals/types.ts index 57cba326f7a..d91d48122b8 100644 --- a/src/platform/terminals/types.ts +++ b/src/platform/terminals/types.ts @@ -6,7 +6,6 @@ import { TextEditor, Uri } from 'vscode'; export const ICodeExecutionHelper = Symbol('ICodeExecutionHelper'); export interface ICodeExecutionHelper { - normalizeLines(code: string): Promise; getFileToExecute(): Promise; saveFileIfDirty(file: Uri): Promise; getSelectedTextToExecute(textEditor: TextEditor): string | undefined;