From d7290b417af19b2092bacf58d905b32917b1c578 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 1 Feb 2024 11:15:45 +1100 Subject: [PATCH 1/3] Use TextEncoder/TextDecoder and not Buffer --- .../execution/cellExecutionMessageHandler.ts | 4 +- .../cellExecutionMessageHandler.unit.test.ts | 128 ++++++++++++++---- src/kernels/execution/helpers.ts | 24 +--- src/kernels/kernelCrashMonitor.unit.test.ts | 12 +- src/notebooks/export/exportBase.web.ts | 2 +- .../common/platform/fileSystem.node.ts | 6 +- src/platform/common/platform/fileSystem.ts | 8 +- src/platform/common/platform/types.ts | 2 +- src/platform/common/utils/string.ts | 25 ++++ src/platform/errors/errorUtils.unit.test.ts | 4 +- src/test/common.web.ts | 2 +- src/test/datascience/mockFileSystem.ts | 100 +------------- .../notebook/executionService.vscode.test.ts | 2 +- src/test/datascience/notebook/helper.ts | 6 +- .../standardWidgets.vscode.common.test.ts | 8 +- .../plotView/plotSaveHandler.ts | 2 +- .../plotView/plotViewHandler.ts | 14 +- .../plotView/plotViewHandler.unit.test.ts | 2 +- .../plotting/plotViewer.node.ts | 3 +- .../extension-side/plotting/plotViewer.ts | 3 +- 20 files changed, 173 insertions(+), 184 deletions(-) create mode 100644 src/platform/common/utils/string.ts diff --git a/src/kernels/execution/cellExecutionMessageHandler.ts b/src/kernels/execution/cellExecutionMessageHandler.ts index a8fdc534d98..6bd51078ff1 100644 --- a/src/kernels/execution/cellExecutionMessageHandler.ts +++ b/src/kernels/execution/cellExecutionMessageHandler.ts @@ -706,7 +706,7 @@ export class CellExecutionMessageHandler implements IDisposable { return true; } if (mime === WIDGET_MIMETYPE) { - const data: WidgetData = JSON.parse(Buffer.from(outputItem.data).toString()); + const data: WidgetData = JSON.parse(new TextDecoder().decode(outputItem.data)); // Jupyter Output widgets cannot be rendered properly by the widget manager, // We need to render that. if (typeof data.model_id === 'string' && this.commIdsMappedToWidgetOutputModels.has(data.model_id)) { @@ -763,7 +763,7 @@ export class CellExecutionMessageHandler implements IDisposable { return false; } try { - const value = JSON.parse(Buffer.from(outputItem.data).toString()) as { model_id?: string }; + const value = JSON.parse(new TextDecoder().decode(outputItem.data)) as { model_id?: string }; return value.model_id === expectedModelId; } catch (ex) { traceWarning(`Failed to deserialize the widget data`, ex); diff --git a/src/kernels/execution/cellExecutionMessageHandler.unit.test.ts b/src/kernels/execution/cellExecutionMessageHandler.unit.test.ts index ce158197e5f..f2a4d06d775 100644 --- a/src/kernels/execution/cellExecutionMessageHandler.unit.test.ts +++ b/src/kernels/execution/cellExecutionMessageHandler.unit.test.ts @@ -418,7 +418,10 @@ suite(`Cell Execution Message Handler`, () => { }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Hello'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Hello' + ); // Update the display data again. await executeCellWithOutput(notebook.cellAt(1), codeToUpdateDisplayDataWorld, 1, (producer) => { return [ @@ -429,7 +432,10 @@ suite(`Cell Execution Message Handler`, () => { }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'World'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'World' + ); }); test('Execute cell and add Display output (even if Cell DOM has not yet been updated) ', async () => { @@ -471,7 +477,10 @@ suite(`Cell Execution Message Handler`, () => { }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Hello'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Hello' + ); // Update the display data again. await executeCellWithOutput(notebook.cellAt(1), codeToUpdateDisplayDataWorld, 1, (producer) => { return [ @@ -482,7 +491,10 @@ suite(`Cell Execution Message Handler`, () => { }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'World'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'World' + ); }); test('Updates to two separate display updates in the same cell output', async () => { const notebook = createNotebook([ @@ -520,8 +532,14 @@ suite(`Cell Execution Message Handler`, () => { assert.strictEqual((output1.transient as any).display_id, display_id); const output2 = translateCellDisplayOutput(notebook.cellAt(0).outputs[1]); assert.strictEqual((output2.transient as any).display_id, display_id2); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Hello'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'World'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Hello' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'World' + ); // Update the first display data. await executeCellWithOutput(notebook.cellAt(1), codeToUpdateDisplayData1ILike, 2, (producer) => { @@ -534,8 +552,14 @@ suite(`Cell Execution Message Handler`, () => { ]; }); // await executeAndUpdateDisplayData(notebook, codeToUpdateDisplayData1ILike, 2, outputsFromILikeUpdate); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'I Like'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'World'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'I Like' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'World' + ); // Update the second display data. await executeCellWithOutput(notebook.cellAt(2), codeToUpdateDisplayData1Pizza, 3, (producer) => { @@ -548,8 +572,14 @@ suite(`Cell Execution Message Handler`, () => { ]; }); // await executeAndUpdateDisplayData(notebook, codeToUpdateDisplayData1Pizza, 3, outputsFromPizzaUpdate); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'I Like'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'Pizza'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'I Like' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'Pizza' + ); }); test('Updates to two separate display updates in the same cell output (update second display update)', async () => { @@ -589,8 +619,14 @@ suite(`Cell Execution Message Handler`, () => { assert.strictEqual((output1.transient as any).display_id, display_id); const output2 = translateCellDisplayOutput(notebook.cellAt(0).outputs[1]); assert.strictEqual((output2.transient as any).display_id, display_id2); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Hello'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'World'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Hello' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'World' + ); // Update the second display data. await executeCellWithOutput(notebook.cellAt(1), codeToUpdateDisplayData1Pizza, 2, (producer) => { @@ -604,8 +640,14 @@ suite(`Cell Execution Message Handler`, () => { }); // await executeAndUpdateDisplayData(notebook, codeToUpdateDisplayData1Pizza, 2, outputsFromPizzaUpdate); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Hello'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'Pizza'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Hello' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'Pizza' + ); // Update the first display data. await executeCellWithOutput(notebook.cellAt(2), codeToUpdateDisplayData1ILike, 3, (producer) => { @@ -618,8 +660,14 @@ suite(`Cell Execution Message Handler`, () => { ]; }); // await executeAndUpdateDisplayData(notebook, codeToUpdateDisplayData1ILike, 3, outputsFromILikeUpdate); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'I Like'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'Pizza'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'I Like' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'Pizza' + ); }); test('Updates to two separate display updates in the same cell output (even if Cell DOM has not yet been updated)', async () => { @@ -667,8 +715,14 @@ suite(`Cell Execution Message Handler`, () => { }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Hello'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'Pizza'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Hello' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'Pizza' + ); // Update the first display data. await executeCellWithOutput(notebook.cellAt(2), codeToUpdateDisplayData1ILike, 3, (producer) => { @@ -680,8 +734,14 @@ suite(`Cell Execution Message Handler`, () => { }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'I Like'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'Pizza'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'I Like' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), + 'Pizza' + ); }); test('Updates display updates in the same cell output within the same execution (even if Cell DOM has not yet been updated) (Issue 12755, 13105, 13163)', async () => { @@ -741,11 +801,23 @@ suite(`Cell Execution Message Handler`, () => { producer.stream({ name: 'stdout', text: 'Pizza\n' }) ]; }); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[0].items[0].data).toString(), 'Touch me not\n'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'C'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[2].items[0].data).toString(), 'Hello\n'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[2].items[1].data).toString(), 'World\n'); - assert.strictEqual(Buffer.from(notebook.cellAt(0).outputs[2].items[2].data).toString(), 'Pizza\n'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[0].items[0].data).toString(), + 'Touch me not\n' + ); + assert.strictEqual(new TextDecoder().decode(notebook.cellAt(0).outputs[1].items[0].data).toString(), 'C'); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[2].items[0].data).toString(), + 'Hello\n' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[2].items[1].data).toString(), + 'World\n' + ); + assert.strictEqual( + new TextDecoder().decode(notebook.cellAt(0).outputs[2].items[2].data).toString(), + 'Pizza\n' + ); }); }); @@ -792,7 +864,7 @@ suite(`Cell Execution Message Handler`, () => { for (let index = 0; index < cell.outputs[0].items.length; index++) { const item = cell.outputs[0].items[index]; assert.strictEqual(item.mime, 'application/vnd.code.notebook.stdout'); - assert.strictEqual(Buffer.from(item.data).toString(), `${index}\n`); + assert.strictEqual(new TextDecoder().decode(item.data).toString(), `${index}\n`); } // Now assume we closed VS Code, and then opened it again. @@ -828,9 +900,9 @@ suite(`Cell Execution Message Handler`, () => { const item = cell.outputs[0].items[index]; assert.strictEqual(item.mime, 'application/vnd.code.notebook.stdout'); if (index >= 50) { - assert.strictEqual(Buffer.from(item.data).toString(), `${25 + index}\n`); + assert.strictEqual(new TextDecoder().decode(item.data).toString(), `${25 + index}\n`); } else { - assert.strictEqual(Buffer.from(item.data).toString(), `${index}\n`); + assert.strictEqual(new TextDecoder().decode(item.data).toString(), `${index}\n`); } } } diff --git a/src/kernels/execution/helpers.ts b/src/kernels/execution/helpers.ts index 968ac14040a..f9187eb44c1 100644 --- a/src/kernels/execution/helpers.ts +++ b/src/kernels/execution/helpers.ts @@ -31,6 +31,7 @@ import { import { StopWatch } from '../../platform/common/utils/stopWatch'; import { getExtensionSpecifcStack } from '../../platform/errors/errors'; import { getVersion } from '../../platform/interpreter/helpers'; +import { base64ToUint8Array, uint8ArrayToBase64 } from '../../platform/common/utils/string'; export enum CellOutputMimeTypes { error = 'application/vnd.code.notebook.error', @@ -112,7 +113,7 @@ const orderOfMimeTypes = [ function isEmptyVendoredMimeType(outputItem: NotebookCellOutputItem) { if (outputItem.mime.startsWith('application/vnd.')) { try { - return Buffer.from(outputItem.data).toString().length === 0; + return new TextDecoder().decode(outputItem.data).length === 0; } catch {} } return false; @@ -372,7 +373,7 @@ export function translateCellErrorOutput(output: NotebookCellOutput): nbformat.I }; } const originalError: undefined | nbformat.IError = output.metadata?.originalError; - const value: Error = JSON.parse(Buffer.from(firstItem.data as Uint8Array).toString('utf8')); + const value: Error = JSON.parse(new TextDecoder().decode(firstItem.data)); return { output_type: 'error', ename: value.name, @@ -400,17 +401,7 @@ function convertOutputMimeToJupyterOutput(mime: string, value: Uint8Array) { } else if (mime.startsWith('image/') && mime !== 'image/svg+xml') { // Images in Jupyter are stored in base64 encoded format. // VS Code expects bytes when rendering images. - if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') { - return Buffer.from(value).toString('base64'); - } else { - // https://developer.mozilla.org/en-US/docs/Glossary/Base64#solution_1_%E2%80%93_escaping_the_string_before_encoding_it - const stringValue = textDecoder.decode(value); - return btoa( - encodeURIComponent(stringValue).replace(/%([0-9A-F]{2})/g, function (_match, p1) { - return String.fromCharCode(Number.parseInt('0x' + p1)); - }) - ); - } + return uint8ArrayToBase64(value); } else if ( mime.toLowerCase().startsWith('application/vnd.holoviews_load.v') && mime.toLowerCase().endsWith('+json') @@ -450,12 +441,7 @@ function convertJupyterOutputToBuffer(mime: string, value: unknown): NotebookCel } else if (mime.startsWith('image/') && typeof value === 'string' && mime !== 'image/svg+xml') { // Images in Jupyter are stored in base64 encoded format. // VS Code expects bytes when rendering images. - if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') { - return new NotebookCellOutputItem(Buffer.from(value, 'base64'), mime); - } else { - const data = Uint8Array.from(atob(value), (c) => c.charCodeAt(0)); - return new NotebookCellOutputItem(data, mime); - } + return new NotebookCellOutputItem(base64ToUint8Array(value), mime); } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) { return NotebookCellOutputItem.text(JSON.stringify(value), mime); } else { diff --git a/src/kernels/kernelCrashMonitor.unit.test.ts b/src/kernels/kernelCrashMonitor.unit.test.ts index 9d2a52ec9a2..9760caf52e4 100644 --- a/src/kernels/kernelCrashMonitor.unit.test.ts +++ b/src/kernels/kernelCrashMonitor.unit.test.ts @@ -109,10 +109,10 @@ suite('Kernel Crash Monitor', () => { const execution = controller.createNotebookCellExecution(cell); execution.start(); - const expectedErrorMessage = Buffer.from( + const expectedErrorMessage = new TextDecoder().decode( createOutputWithErrorMessageForDisplay(DataScience.kernelCrashedDueToCodeInCurrentOrPreviousCell)?.items[0]! .data! - ).toString(); + ); when(kernel.status).thenReturn('dead'); onKernelStatusChanged.fire({ status: 'dead', kernel: instance(kernel) }); @@ -127,7 +127,7 @@ suite('Kernel Crash Monitor', () => { assert.strictEqual(cell.outputs.length, 1); assert.strictEqual(cell.outputs[0].items.length, 1); const outputItem = cell.outputs[0].items[0]; - assert.include(Buffer.from(outputItem.data).toString(), expectedErrorMessage); + assert.include(new TextDecoder().decode(outputItem.data), expectedErrorMessage); }); test('Error message displayed and Cell output updated with error message (jupyter kernel)', async () => { when(kernelSession.kind).thenReturn('localJupyter'); @@ -139,10 +139,10 @@ suite('Kernel Crash Monitor', () => { const execution = controller.createNotebookCellExecution(cell); execution.start(); - const expectedErrorMessage = Buffer.from( + const expectedErrorMessage = new TextDecoder().decode( createOutputWithErrorMessageForDisplay(DataScience.kernelCrashedDueToCodeInCurrentOrPreviousCell)?.items[0]! .data! - ).toString(); + ); when(kernel.status).thenReturn('autorestarting'); when(kernelSession.status).thenReturn('autorestarting'); @@ -160,6 +160,6 @@ suite('Kernel Crash Monitor', () => { assert.strictEqual(cell.outputs.length, 1); assert.strictEqual(cell.outputs[0].items.length, 1); const outputItem = cell.outputs[0].items[0]; - assert.include(Buffer.from(outputItem.data).toString(), expectedErrorMessage); + assert.include(new TextDecoder().decode(outputItem.data), expectedErrorMessage); }); }); diff --git a/src/notebooks/export/exportBase.web.ts b/src/notebooks/export/exportBase.web.ts index 8a973d451c3..e406074ae4c 100644 --- a/src/notebooks/export/exportBase.web.ts +++ b/src/notebooks/export/exportBase.web.ts @@ -149,7 +149,7 @@ export class ExportBase implements IExportBase { }); const bytes = this.b64toBlob(content.content, 'application/pdf'); const buffer = await bytes.arrayBuffer(); - await this.fs.writeFile(target!, Buffer.from(buffer)); + await this.fs.writeFile(target!, new Uint8Array(buffer)); } else { const content = await contentsManager.get(tempTarget, { type: 'file', diff --git a/src/platform/common/platform/fileSystem.node.ts b/src/platform/common/platform/fileSystem.node.ts index a45f5d5eb08..277e1bb53b0 100644 --- a/src/platform/common/platform/fileSystem.node.ts +++ b/src/platform/common/platform/fileSystem.node.ts @@ -109,13 +109,13 @@ export class FileSystem extends FileSystemBase implements IFileSystemNode { await this.vscfs.createDirectory(uri); } } - override async writeFile(uri: Uri, text: string | Buffer): Promise { + override async writeFile(uri: Uri, text: string | Uint8Array): Promise { if (isLocalFile(uri)) { const filename = getFilePath(uri); await fs.ensureDir(path.dirname(filename)); - return fs.writeFile(filename, text); + return fs.writeFile(filename, typeof text === 'string' ? Buffer.from(text) : text); } else { - await this.vscfs.writeFile(uri, typeof text === 'string' ? Buffer.from(text) : text); + await this.vscfs.writeFile(uri, typeof text === 'string' ? new TextEncoder().encode(text) : text); } } override async copy(source: Uri, destination: Uri, options?: { overwrite: boolean }): Promise { diff --git a/src/platform/common/platform/fileSystem.ts b/src/platform/common/platform/fileSystem.ts index c235f1066c9..e3dd9194bd2 100644 --- a/src/platform/common/platform/fileSystem.ts +++ b/src/platform/common/platform/fileSystem.ts @@ -48,17 +48,15 @@ export class FileSystem implements IFileSystem { async readFile(uri: vscode.Uri): Promise { const result = await this.vscfs.readFile(uri); - const data = Buffer.from(result); - return data.toString(ENCODING); + return new TextDecoder().decode(result); } async stat(uri: vscode.Uri): Promise { return this.vscfs.stat(uri); } - async writeFile(uri: vscode.Uri, text: string | Buffer): Promise { - const data = typeof text === 'string' ? Buffer.from(text) : text; - return this.vscfs.writeFile(uri, data); + async writeFile(uri: vscode.Uri, text: string | Uint8Array): Promise { + return this.vscfs.writeFile(uri, typeof text === 'string' ? new TextEncoder().encode(text) : text); } async exists( diff --git a/src/platform/common/platform/types.ts b/src/platform/common/platform/types.ts index b42eb713445..e0a0671e51b 100644 --- a/src/platform/common/platform/types.ts +++ b/src/platform/common/platform/types.ts @@ -42,7 +42,7 @@ export interface IFileSystem { delete(uri: vscode.Uri): Promise; readFile(uri: vscode.Uri): Promise; stat(uri: vscode.Uri): Promise; - writeFile(uri: vscode.Uri, text: string | Buffer): Promise; + writeFile(uri: vscode.Uri, text: string | Uint8Array): Promise; getFiles(dir: vscode.Uri): Promise; exists(uri: vscode.Uri, fileType?: vscode.FileType): Promise; getFileHash(filename: vscode.Uri): Promise; diff --git a/src/platform/common/utils/string.ts b/src/platform/common/utils/string.ts new file mode 100644 index 00000000000..53c86f1914a --- /dev/null +++ b/src/platform/common/utils/string.ts @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export function base64ToUint8Array(base64: string): Uint8Array { + if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') { + return Buffer.from(base64, 'base64'); + } else { + return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)); + } +} + +const textDecoder = new TextDecoder(); +export function uint8ArrayToBase64(buffer: Uint8Array): string { + if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') { + return Buffer.from(buffer).toString('base64'); + } else { + // https://developer.mozilla.org/en-US/docs/Glossary/Base64#solution_1_%E2%80%93_escaping_the_string_before_encoding_it + const stringValue = textDecoder.decode(buffer); + return btoa( + encodeURIComponent(stringValue).replace(/%([0-9A-F]{2})/g, function (_match, p1) { + return String.fromCharCode(Number.parseInt('0x' + p1)); + }) + ); + } +} diff --git a/src/platform/errors/errorUtils.unit.test.ts b/src/platform/errors/errorUtils.unit.test.ts index 624536a0a9a..812419cb4b6 100644 --- a/src/platform/errors/errorUtils.unit.test.ts +++ b/src/platform/errors/errorUtils.unit.test.ts @@ -8,7 +8,9 @@ suite('Error Utils', () => { suite('Markdown links to Hrefs', () => { function getHtmlMessage(markdown: string) { const output = createOutputWithErrorMessageForDisplay(markdown); - const { stack } = JSON.parse(Buffer.from(output!.items[0].data).toString()) as { stack: string }; + const { stack } = JSON.parse(new TextDecoder().decode(output!.items[0].data).toString()) as { + stack: string; + }; return stack.replace('\u001b[1;31m', ''); } test('Markdown links to Hrefs', () => { diff --git a/src/test/common.web.ts b/src/test/common.web.ts index cec369c11dd..805d2884831 100644 --- a/src/test/common.web.ts +++ b/src/test/common.web.ts @@ -45,7 +45,7 @@ export function initializeCommonWebApi() { const file = Uri.joinPath(tmpDir, `${uuid()}${extension}`); const contents = options.contents || ''; - await workspace.fs.writeFile(file, Buffer.from(contents)); + await workspace.fs.writeFile(file, new TextEncoder().encode(contents)); return { file, dispose: () => { diff --git a/src/test/datascience/mockFileSystem.ts b/src/test/datascience/mockFileSystem.ts index f1d9466117d..df0a4d93ae8 100644 --- a/src/test/datascience/mockFileSystem.ts +++ b/src/test/datascience/mockFileSystem.ts @@ -4,9 +4,7 @@ /* eslint-disable local-rules/dont-use-fspath */ import * as fsextra from 'fs-extra'; -import * as path from '../../platform/vscode-path/path'; -import { FileStat, FileType, Uri } from 'vscode'; -import { createDeferred } from '../../platform/common/utils/async'; +import { FileStat, FileType } from 'vscode'; export function convertStat(old: fsextra.Stats, filetype: FileType): FileStat { return { @@ -20,99 +18,3 @@ export function convertStat(old: fsextra.Stats, filetype: FileType): FileStat { mtime: Math.round(old.mtimeMs) }; } - -// This is necessary for unit tests and functional tests, since they -// do not run under VS Code so they do not have access to the actual -// "vscode" namespace. -export class FakeVSCodeFileSystemAPI { - public isWritableFileSystem(_scheme: string): boolean | undefined { - return; - } - public async readFile(uri: Uri): Promise { - return fsextra.readFile(uri.fsPath); - } - public async writeFile(uri: Uri, content: Uint8Array): Promise { - await fsextra.mkdirs(path.dirname(uri.fsPath)); - return fsextra.writeFile(uri.fsPath, Buffer.from(content)); - } - public async delete(uri: Uri, _options?: { recursive: boolean; useTrash: boolean }): Promise { - return ( - fsextra - // Make sure the file exists before deleting. - .stat(uri.fsPath) - .then(() => fsextra.remove(uri.fsPath)) - ); - } - public async stat(uri: Uri): Promise { - const filename = uri.fsPath; - - let filetype = FileType.Unknown; - let stat = await fsextra.lstat(filename); - if (stat.isSymbolicLink()) { - filetype = FileType.SymbolicLink; - stat = await fsextra.stat(filename); - } - if (stat.isFile()) { - filetype |= FileType.File; - } else if (stat.isDirectory()) { - filetype |= FileType.Directory; - } - return convertStat(stat, filetype); - } - public async readDirectory(uri: Uri): Promise<[string, FileType][]> { - const names: string[] = await fsextra.readdir(uri.fsPath); - const promises = names.map((name) => { - const filename = path.join(uri.fsPath, name); - return ( - fsextra - // Get the lstat info and deal with symlinks if necessary. - .lstat(filename) - .then(async (stat) => { - let filetype = FileType.Unknown; - if (stat.isFile()) { - filetype = FileType.File; - } else if (stat.isDirectory()) { - filetype = FileType.Directory; - } else if (stat.isSymbolicLink()) { - filetype = FileType.SymbolicLink; - stat = await fsextra.stat(filename); - if (stat.isFile()) { - filetype |= FileType.File; - } else if (stat.isDirectory()) { - filetype |= FileType.Directory; - } - } - return [name, filetype] as [string, FileType]; - }) - .catch(() => [name, FileType.Unknown] as [string, FileType]) - ); - }); - return Promise.all(promises); - } - public async createDirectory(uri: Uri): Promise { - return fsextra.mkdirp(uri.fsPath); - } - public async copy(src: Uri, dest: Uri): Promise { - const deferred = createDeferred(); - const rs = fsextra - // Set an error handler on the stream. - .createReadStream(src.fsPath) - .on('error', (err) => { - deferred.reject(err); - }); - const ws = fsextra - .createWriteStream(dest.fsPath) - // Set an error & close handler on the stream. - .on('error', (err) => { - deferred.reject(err); - }) - .on('close', () => { - deferred.resolve(); - }); - rs.pipe(ws); - return deferred.promise; - } - public async rename(src: Uri, dest: Uri): Promise { - return fsextra.rename(src.fsPath, dest.fsPath); - } -} diff --git a/src/test/datascience/notebook/executionService.vscode.test.ts b/src/test/datascience/notebook/executionService.vscode.test.ts index 32f22882bf7..eb7fa425d7a 100644 --- a/src/test/datascience/notebook/executionService.vscode.test.ts +++ b/src/test/datascience/notebook/executionService.vscode.test.ts @@ -143,7 +143,7 @@ suite('Kernel Execution @kernelCore', function () { await kernelExecution.executeCell(cell); assert.isAtLeast(cell.executionSummary?.executionOrder || 0, 1); - assert.strictEqual(Buffer.from(cell.outputs[0].items[0].data).toString().trim(), '123412341234'); + assert.strictEqual(new TextDecoder().decode(cell.outputs[0].items[0].data).toString().trim(), '123412341234'); assert.isTrue(cell.executionSummary?.success); }); test('Test __vsc_ipynb_file__ defined in cell using VSCode Kernel', async () => { diff --git a/src/test/datascience/notebook/helper.ts b/src/test/datascience/notebook/helper.ts index 0939d81a0a3..60fe9d57a47 100644 --- a/src/test/datascience/notebook/helper.ts +++ b/src/test/datascience/notebook/helper.ts @@ -192,7 +192,7 @@ async function createTemporaryNotebookFromNotebook( prefix?: string ) { const uri = await generateTemporaryFilePath('.ipynb', disposables, rootFolder, prefix); - await workspace.fs.writeFile(uri, Buffer.from(JSON.stringify(notebook))); + await workspace.fs.writeFile(uri, new TextEncoder().encode(JSON.stringify(notebook))); return uri; } @@ -1145,7 +1145,7 @@ function getOutputText(output: NotebookCellOutputItem) { ) { return ''; } - return Buffer.from(output.data).toString('utf8'); + return new TextDecoder().decode(output.data); } function hasTextOutputValue(output: NotebookCellOutputItem, value: string, isExactMatch = true) { if ( @@ -1158,7 +1158,7 @@ function hasTextOutputValue(output: NotebookCellOutputItem, value: string, isExa return false; } try { - const haystack = Buffer.from(output.data).toString('utf8'); + const haystack = new TextDecoder().decode(output.data); return isExactMatch ? haystack === value || haystack.trim() === value : haystack.toLowerCase().includes(value.toLowerCase()); diff --git a/src/test/datascience/widgets/standardWidgets.vscode.common.test.ts b/src/test/datascience/widgets/standardWidgets.vscode.common.test.ts index 2b7db1f0053..a0a7e0dd0fc 100644 --- a/src/test/datascience/widgets/standardWidgets.vscode.common.test.ts +++ b/src/test/datascience/widgets/standardWidgets.vscode.common.test.ts @@ -560,10 +560,10 @@ suite('Standard IPyWidget Tests @widgets', function () { for (let output of cell.outputs) { for (let item of output.items) { if (item.mime === 'application/vnd.custom') { - mimeValues.push(Buffer.from(item.data).toString().trim()); + mimeValues.push(new TextDecoder().decode(item.data).toString().trim()); } if (item.mime === 'application/vnd.code.notebook.stdout') { - stdOut = Buffer.from(item.data).toString().trim(); + stdOut = new TextDecoder().decode(item.data).toString().trim(); } } } @@ -583,10 +583,10 @@ suite('Standard IPyWidget Tests @widgets', function () { for (let output of cell.outputs) { for (let item of output.items) { if (item.mime === 'application/vnd.custom') { - mimeValues.push(Buffer.from(item.data).toString().trim()); + mimeValues.push(new TextDecoder().decode(item.data).toString().trim()); } if (item.mime === 'application/vnd.code.notebook.stdout') { - stdOut = Buffer.from(item.data).toString().trim(); + stdOut = new TextDecoder().decode(item.data).toString().trim(); } } } diff --git a/src/webviews/extension-side/plotView/plotSaveHandler.ts b/src/webviews/extension-side/plotView/plotSaveHandler.ts index 56ac90bd813..603778b1bef 100644 --- a/src/webviews/extension-side/plotView/plotSaveHandler.ts +++ b/src/webviews/extension-side/plotView/plotSaveHandler.ts @@ -83,7 +83,7 @@ export class PlotSaveHandler implements IPlotSaveHandler { ); } - await this.fs.writeFile(target, Buffer.from(data.data)); + await this.fs.writeFile(target, data.data); } protected async saveAsPdf(_output: NotebookCellOutput, _target: Uri) { diff --git a/src/webviews/extension-side/plotView/plotViewHandler.ts b/src/webviews/extension-side/plotView/plotViewHandler.ts index 9af7c2f4bbb..6ac86110619 100644 --- a/src/webviews/extension-side/plotView/plotViewHandler.ts +++ b/src/webviews/extension-side/plotView/plotViewHandler.ts @@ -6,6 +6,7 @@ import { NotebookCellOutputItem, NotebookDocument } from 'vscode'; import { traceError } from '../../../platform/logging'; import { getDisplayPath } from '../../../platform/common/platform/fs-paths'; import { IPlotViewerProvider } from '../plotting/types'; +import { uint8ArrayToBase64 } from '../../../platform/common/utils/string'; const svgMimeType = 'image/svg+xml'; const pngMimeType = 'image/png'; @@ -56,8 +57,8 @@ function getOutputItem( // Wrap our PNG data into an SVG element so what we can display it in the current plot viewer function convertPngToSvg(pngOutput: NotebookCellOutputItem): string { - const imageBuffer = Buffer.from(pngOutput.data); - const imageData = imageBuffer.toString('base64'); + const imageBuffer = pngOutput.data; + const imageData = uint8ArrayToBase64(imageBuffer); const dims = getPngDimensions(imageBuffer); // Of note here, we want the dims on the SVG element, and the image at 100% this is due to how the SVG control @@ -71,20 +72,21 @@ function convertPngToSvg(pngOutput: NotebookCellOutputItem): string { `; } -export function getPngDimensions(buffer: Buffer): { width: number; height: number } { +export function getPngDimensions(buffer: Uint8Array): { width: number; height: number } { // Verify this is a PNG if (!isPng(buffer)) { throw new Error('The buffer is not a valid png'); } // The dimensions of a PNG are the first 8 bytes (width then height) of the IHDR chunk. The // IHDR chunk starts at offset 8. + const view = new DataView(buffer.buffer) return { - width: buffer.readUInt32BE(16), - height: buffer.readUInt32BE(20) + width: view.getUint32(16, false), + height: view.getUint32(20, false) }; } -function isPng(buffer: Buffer): boolean { +function isPng(buffer: Uint8Array): boolean { // The first eight bytes of a PNG datastream always contain the following (decimal) values: // 137 80 78 71 13 10 26 10 return ( diff --git a/src/webviews/extension-side/plotView/plotViewHandler.unit.test.ts b/src/webviews/extension-side/plotView/plotViewHandler.unit.test.ts index 5dc6a40d657..6754b7335b1 100644 --- a/src/webviews/extension-side/plotView/plotViewHandler.unit.test.ts +++ b/src/webviews/extension-side/plotView/plotViewHandler.unit.test.ts @@ -21,7 +21,7 @@ suite('PlotViewHandler', () => { // Rest of IHDR and other 0x08, 0x02, 0x00, 0x00, 0x00, 0xc5, 0x6b, 0x38 ]); - deepStrictEqual(getPngDimensions(Buffer.from(t)), { + deepStrictEqual(getPngDimensions(t), { height: 1067, width: 1066 }); diff --git a/src/webviews/extension-side/plotting/plotViewer.node.ts b/src/webviews/extension-side/plotting/plotViewer.node.ts index 7625b1a66c6..102994afbb7 100644 --- a/src/webviews/extension-side/plotting/plotViewer.node.ts +++ b/src/webviews/extension-side/plotting/plotViewer.node.ts @@ -13,6 +13,7 @@ import { noop } from '../../../platform/common/utils/misc'; import { PlotViewer as PlotViewerBase } from './plotViewer'; import { IWebviewPanelProvider } from '../../../platform/common/application/types'; import { IConfigurationService, IExtensionContext } from '../../../platform/common/types'; +import { base64ToUint8Array } from '../../../platform/common/utils/string'; @injectable() export class PlotViewer extends PlotViewerBase { @@ -46,7 +47,7 @@ export class PlotViewer extends PlotViewerBase { break; case '.png': - const buffer = Buffer.from(payload.png.replace('data:image/png;base64', ''), 'base64'); + const buffer = base64ToUint8Array(payload.png.replace('data:image/png;base64', '')); await this.fs.writeFile(file, buffer); break; diff --git a/src/webviews/extension-side/plotting/plotViewer.ts b/src/webviews/extension-side/plotting/plotViewer.ts index 066e4563cee..38a67ac6eeb 100644 --- a/src/webviews/extension-side/plotting/plotViewer.ts +++ b/src/webviews/extension-side/plotting/plotViewer.ts @@ -17,6 +17,7 @@ import { joinPath } from '../../../platform/vscode-path/resources'; import { noop } from '../../../platform/common/utils/misc'; import { sendTelemetryEvent, Telemetry } from '../../../telemetry'; import { StopWatch } from '../../../platform/common/utils/stopWatch'; +import { base64ToUint8Array } from '../../../platform/common/utils/string'; @injectable() export class PlotViewer extends WebviewPanelHost implements IPlotViewer, IDisposable { @@ -135,7 +136,7 @@ export class PlotViewer extends WebviewPanelHost implements const ext = path.extname(file.path); switch (ext.toLowerCase()) { case '.png': - const buffer = Buffer.from(payload.png.replace('data:image/png;base64', ''), 'base64'); + const buffer = base64ToUint8Array(payload.png.replace('data:image/png;base64', '')); await this.fs.writeFile(file, buffer); break; From 207412e46459c68e094bb1f6f4ac4bfd20673e0b Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 1 Feb 2024 11:23:44 +1100 Subject: [PATCH 2/3] oops --- src/webviews/extension-side/plotView/plotViewHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webviews/extension-side/plotView/plotViewHandler.ts b/src/webviews/extension-side/plotView/plotViewHandler.ts index 6ac86110619..4bba688df11 100644 --- a/src/webviews/extension-side/plotView/plotViewHandler.ts +++ b/src/webviews/extension-side/plotView/plotViewHandler.ts @@ -79,7 +79,7 @@ export function getPngDimensions(buffer: Uint8Array): { width: number; height: n } // The dimensions of a PNG are the first 8 bytes (width then height) of the IHDR chunk. The // IHDR chunk starts at offset 8. - const view = new DataView(buffer.buffer) + const view = new DataView(buffer.buffer); return { width: view.getUint32(16, false), height: view.getUint32(20, false) From 241b78bda05b5a5e5cb7a854d5956a31f3a91f06 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 1 Feb 2024 12:24:55 +1100 Subject: [PATCH 3/3] Fix tests --- src/webviews/extension-side/plotView/plotViewHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webviews/extension-side/plotView/plotViewHandler.ts b/src/webviews/extension-side/plotView/plotViewHandler.ts index 4bba688df11..e1d8914121a 100644 --- a/src/webviews/extension-side/plotView/plotViewHandler.ts +++ b/src/webviews/extension-side/plotView/plotViewHandler.ts @@ -79,7 +79,7 @@ export function getPngDimensions(buffer: Uint8Array): { width: number; height: n } // The dimensions of a PNG are the first 8 bytes (width then height) of the IHDR chunk. The // IHDR chunk starts at offset 8. - const view = new DataView(buffer.buffer); + const view = new DataView(new Uint8Array(buffer).buffer); return { width: view.getUint32(16, false), height: view.getUint32(20, false)