From c764b968d1aaa04bc20643c2fe53a258941e2b0d Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Tue, 2 Apr 2024 15:59:21 +0200 Subject: [PATCH 1/7] fix nested object serialization and deserialization --- packages/core/src/codecs/octetstream-codec.ts | 26 ++++-- packages/core/test/ContentSerdesTest.ts | 88 ++++++++++++++++++- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/packages/core/src/codecs/octetstream-codec.ts b/packages/core/src/codecs/octetstream-codec.ts index b90f78759..1d9321255 100644 --- a/packages/core/src/codecs/octetstream-codec.ts +++ b/packages/core/src/codecs/octetstream-codec.ts @@ -214,7 +214,20 @@ export default class OctetstreamCodec implements ContentCodec { const sortedProperties = Object.getOwnPropertyNames(schema.properties); for (const propertyName of sortedProperties) { const propertySchema = schema.properties[propertyName]; - result[propertyName] = this.bytesToValue(bytes, propertySchema, parameters); + if (propertySchema.type === "object") { + const bitLength = parseInt(propertySchema["ex:bitLength"]); + const bitOffset = + propertySchema["ex:bitOffset"] !== undefined ? parseInt(propertySchema["ex:bitOffset"]) : 0; + const length = isNaN(bitLength) ? bytes.length : Math.ceil(bitLength / 8); + const buf = Buffer.alloc(length); + this.copyBits(bytes, bitOffset, buf, 0, length * 8); + result[propertyName] = this.objectToValue(buf, propertySchema, { + ...parameters, + length: length.toString(), + }); + } else { + result[propertyName] = this.bytesToValue(bytes, propertySchema, parameters); + } } return result; } @@ -542,22 +555,25 @@ export default class OctetstreamCodec implements ContentCodec { throw new Error("Missing 'length' parameter necessary for write"); } + const offset = schema["ex:bitOffset"] !== undefined ? parseInt(schema["ex:bitOffset"]) : 0; result = result ?? Buffer.alloc(parseInt(parameters.length)); for (const propertyName in schema.properties) { if (Object.hasOwnProperty.call(value, propertyName) === false) { - throw new Error(`Missing property '${propertyName}'`); - } + throw new Error(`Missing property '${propertyName}'`); + } const propertySchema = schema.properties[propertyName]; const propertyValue = value[propertyName]; const propertyOffset = parseInt(propertySchema["ex:bitOffset"]); const propertyLength = parseInt(propertySchema["ex:bitLength"]); let buf: Buffer; if (propertySchema.type === "object") { - buf = this.valueToObject(propertyValue, propertySchema, parameters, result); + const length = Math.ceil(propertyLength / 8).toString(); + + buf = this.valueToObject(propertyValue, propertySchema, { ...parameters, length }, result); } else { buf = this.valueToBytes(propertyValue, propertySchema, parameters); } - this.copyBits(buf, propertyOffset, result, propertyOffset, propertyLength); + this.copyBits(buf, propertyOffset, result, offset + propertyOffset, propertyLength); } return result; } diff --git a/packages/core/test/ContentSerdesTest.ts b/packages/core/test/ContentSerdesTest.ts index 9f88a47ea..f48ca71c3 100644 --- a/packages/core/test/ContentSerdesTest.ts +++ b/packages/core/test/ContentSerdesTest.ts @@ -444,6 +444,59 @@ class SerdesOctetTests { }, } ); + + checkStreamToValue( + [0x0e, 0x10, 0x10, 0x10, 0x0e], + { + flags1: { flag1: false, flag2: true }, + flags2: { flag1: true, flag2: false }, + }, + "object", + { + type: "object", + properties: { + flags1: { + type: "object", + "ex:bitOffset": 0, + "ex:bitLength": 8, + properties: { + flag1: { + type: "boolean", + title: "Bit 1", + "ex:bitOffset": 3, + "ex:bitLength": 1, + }, + flag2: { + type: "boolean", + title: "Bit 2", + "ex:bitOffset": 4, + "ex:bitLength": 1, + }, + }, + }, + flags2: { + type: "object", + "ex:bitOffset": 8, + "ex:bitLength": 8, + properties: { + flag1: { + type: "boolean", + title: "Bit 1", + "ex:bitOffset": 3, + "ex:bitLength": 1, + }, + flag2: { + type: "boolean", + title: "Bit 2", + "ex:bitOffset": 4, + "ex:bitLength": 1, + }, + }, + }, + }, + }, + { length: "5" } + ); } @test async "OctetStream to value should throw"() { @@ -515,7 +568,7 @@ class SerdesOctetTests { ).to.throw(Error, "Missing schema for object"); expect(() => ContentSerdes.contentToValue( - { type: "application/octet-stream", body: Buffer.from([0x36, 0x30]) }, + { type: "application/octet-stream;length=2", body: Buffer.from([0x36, 0x30]) }, { type: "object", properties: { @@ -763,6 +816,39 @@ class SerdesOctetTests { ); body = await content.toBuffer(); expect(body).to.deep.equal(Buffer.from([0xc0])); + + content = ContentSerdes.valueToContent( + { + flags1: { flag1: false, flag2: true }, + flags2: { flag1: true, flag2: false }, + }, + { + type: "object", + properties: { + flags1: { + type: "object", + properties: { + flag1: { type: "boolean", "ex:bitOffset": 3, "ex:bitLength": 1 }, + flag2: { type: "boolean", "ex:bitOffset": 4, "ex:bitLength": 1 }, + }, + "ex:bitLength": 8, + }, + flags2: { + type: "object", + properties: { + flag1: { type: "boolean", "ex:bitOffset": 3, "ex:bitLength": 1 }, + flag2: { type: "boolean", "ex:bitOffset": 4, "ex:bitLength": 1 }, + }, + "ex:bitOffset": 8, + "ex:bitLength": 8, + }, + }, + "ex:bitLength": 16, + }, + "application/octet-stream;length=2;" + ); + body = await content.toBuffer(); + expect(body).to.deep.equal(Buffer.from([0x08, 0x10])); } @test "value to OctetStream should throw"() { From 3ad5f545d08e06e527171cf831abded17fc35f15 Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Tue, 2 Apr 2024 16:01:42 +0200 Subject: [PATCH 2/7] fix format --- packages/core/src/codecs/octetstream-codec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/codecs/octetstream-codec.ts b/packages/core/src/codecs/octetstream-codec.ts index 1d9321255..32910d495 100644 --- a/packages/core/src/codecs/octetstream-codec.ts +++ b/packages/core/src/codecs/octetstream-codec.ts @@ -559,8 +559,8 @@ export default class OctetstreamCodec implements ContentCodec { result = result ?? Buffer.alloc(parseInt(parameters.length)); for (const propertyName in schema.properties) { if (Object.hasOwnProperty.call(value, propertyName) === false) { - throw new Error(`Missing property '${propertyName}'`); - } + throw new Error(`Missing property '${propertyName}'`); + } const propertySchema = schema.properties[propertyName]; const propertyValue = value[propertyName]; const propertyOffset = parseInt(propertySchema["ex:bitOffset"]); From 162a37b1abf377ef3d6077840f2b2093f3f1c6d7 Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Tue, 2 Apr 2024 16:18:14 +0200 Subject: [PATCH 3/7] remove unused parameter --- packages/core/test/ContentSerdesTest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/test/ContentSerdesTest.ts b/packages/core/test/ContentSerdesTest.ts index f48ca71c3..95355c286 100644 --- a/packages/core/test/ContentSerdesTest.ts +++ b/packages/core/test/ContentSerdesTest.ts @@ -568,7 +568,7 @@ class SerdesOctetTests { ).to.throw(Error, "Missing schema for object"); expect(() => ContentSerdes.contentToValue( - { type: "application/octet-stream;length=2", body: Buffer.from([0x36, 0x30]) }, + { type: "application/octet-stream", body: Buffer.from([0x36, 0x30]) }, { type: "object", properties: { From ac364397b8128261981a79270cce217d148f702f Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Tue, 2 Apr 2024 16:43:34 +0200 Subject: [PATCH 4/7] clean up --- packages/core/src/codecs/octetstream-codec.ts | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/packages/core/src/codecs/octetstream-codec.ts b/packages/core/src/codecs/octetstream-codec.ts index 32910d495..6b9f5571e 100644 --- a/packages/core/src/codecs/octetstream-codec.ts +++ b/packages/core/src/codecs/octetstream-codec.ts @@ -214,20 +214,8 @@ export default class OctetstreamCodec implements ContentCodec { const sortedProperties = Object.getOwnPropertyNames(schema.properties); for (const propertyName of sortedProperties) { const propertySchema = schema.properties[propertyName]; - if (propertySchema.type === "object") { - const bitLength = parseInt(propertySchema["ex:bitLength"]); - const bitOffset = - propertySchema["ex:bitOffset"] !== undefined ? parseInt(propertySchema["ex:bitOffset"]) : 0; - const length = isNaN(bitLength) ? bytes.length : Math.ceil(bitLength / 8); - const buf = Buffer.alloc(length); - this.copyBits(bytes, bitOffset, buf, 0, length * 8); - result[propertyName] = this.objectToValue(buf, propertySchema, { - ...parameters, - length: length.toString(), - }); - } else { - result[propertyName] = this.bytesToValue(bytes, propertySchema, parameters); - } + const length = bytes.length.toString(); + result[propertyName] = this.bytesToValue(bytes, propertySchema, {...parameters, length}); } return result; } @@ -567,9 +555,7 @@ export default class OctetstreamCodec implements ContentCodec { const propertyLength = parseInt(propertySchema["ex:bitLength"]); let buf: Buffer; if (propertySchema.type === "object") { - const length = Math.ceil(propertyLength / 8).toString(); - - buf = this.valueToObject(propertyValue, propertySchema, { ...parameters, length }, result); + buf = this.valueToObject(propertyValue, propertySchema, parameters, result); } else { buf = this.valueToBytes(propertyValue, propertySchema, parameters); } From 7415c624bc9a03b273b48e77e07bb3818c833cd0 Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Tue, 2 Apr 2024 16:53:37 +0200 Subject: [PATCH 5/7] fix format --- packages/core/src/codecs/octetstream-codec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/codecs/octetstream-codec.ts b/packages/core/src/codecs/octetstream-codec.ts index 6b9f5571e..1b659d920 100644 --- a/packages/core/src/codecs/octetstream-codec.ts +++ b/packages/core/src/codecs/octetstream-codec.ts @@ -215,7 +215,7 @@ export default class OctetstreamCodec implements ContentCodec { for (const propertyName of sortedProperties) { const propertySchema = schema.properties[propertyName]; const length = bytes.length.toString(); - result[propertyName] = this.bytesToValue(bytes, propertySchema, {...parameters, length}); + result[propertyName] = this.bytesToValue(bytes, propertySchema, { ...parameters, length }); } return result; } From d4a3888f6588fa0f3a1080c603a776efefd04d13 Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Thu, 4 Apr 2024 15:00:45 +0200 Subject: [PATCH 6/7] add check to ensure `ex:bitOffset` is valid --- packages/core/src/codecs/octetstream-codec.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/core/src/codecs/octetstream-codec.ts b/packages/core/src/codecs/octetstream-codec.ts index 1b659d920..c9b8f523b 100644 --- a/packages/core/src/codecs/octetstream-codec.ts +++ b/packages/core/src/codecs/octetstream-codec.ts @@ -543,8 +543,18 @@ export default class OctetstreamCodec implements ContentCodec { throw new Error("Missing 'length' parameter necessary for write"); } + const length = parseInt(parameters.length); const offset = schema["ex:bitOffset"] !== undefined ? parseInt(schema["ex:bitOffset"]) : 0; - result = result ?? Buffer.alloc(parseInt(parameters.length)); + + if (isNaN(offset) || offset < 0) { + throw new Error("ex:bitOffset must be a non-negative number"); + } + + if (offset > length * 8) { + throw new Error(`ex:bitOffset ${offset} exceeds length ${length}`); + } + + result = result ?? Buffer.alloc(length); for (const propertyName in schema.properties) { if (Object.hasOwnProperty.call(value, propertyName) === false) { throw new Error(`Missing property '${propertyName}'`); From 1d867505d778eb2f915df884dbc4de9171385d46 Mon Sep 17 00:00:00 2001 From: Thomas Wehr Date: Fri, 5 Apr 2024 09:22:39 +0200 Subject: [PATCH 7/7] add checks for more parameters --- packages/core/src/codecs/octetstream-codec.ts | 77 +++++++++++++++---- packages/core/test/ContentSerdesTest.ts | 72 +++++++++++++++++ 2 files changed, 135 insertions(+), 14 deletions(-) diff --git a/packages/core/src/codecs/octetstream-codec.ts b/packages/core/src/codecs/octetstream-codec.ts index c9b8f523b..e1bb481c6 100644 --- a/packages/core/src/codecs/octetstream-codec.ts +++ b/packages/core/src/codecs/octetstream-codec.ts @@ -59,14 +59,41 @@ export default class OctetstreamCodec implements ContentCodec { debug("OctetstreamCodec parsing", bytes); debug("Parameters", parameters); - const bigEndian = !(parameters.byteSeq?.includes(Endianness.LITTLE_ENDIAN) === true); // default to big endian - let signed = parameters.signed !== "false"; // default to signed + const length = + parameters.length != null + ? parseInt(parameters.length) + : (warn("Missing 'length' parameter necessary for write. I'll do my best"), undefined); + + if (length !== undefined) { + if (isNaN(length) || length < 0) { + throw new Error("'length' parameter must be a non-negative number"); + } + if (length !== bytes.length) { + throw new Error(`Lengths do not match, required: ${length} provided: ${bytes.length}`); + } + } + + let signed = true; // default to signed + if (parameters.signed !== undefined) { + if (parameters.signed !== "true" && parameters.signed !== "false") { + throw new Error("'signed' parameter must be 'true' or 'false'"); + } + signed = parameters.signed === "true"; + } + + let bitLength = schema?.["ex:bitLength"] !== undefined ? parseInt(schema["ex:bitLength"]) : bytes.length * 8; + + if (isNaN(bitLength) || bitLength < 0) { + throw new Error("'ex:bitLength' must be a non-negative number"); + } + const offset = schema?.["ex:bitOffset"] !== undefined ? parseInt(schema["ex:bitOffset"]) : 0; - if (parameters.length != null && parseInt(parameters.length) !== bytes.length) { - throw new Error("Lengths do not match, required: " + parameters.length + " provided: " + bytes.length); + + if (isNaN(offset) || offset < 0) { + throw new Error("'ex:bitOffset' must be a non-negative number"); } - let bitLength: number = - schema?.["ex:bitLength"] !== undefined ? parseInt(schema["ex:bitLength"]) : bytes.length * 8; + + const bigEndian = !(parameters.byteSeq?.includes(Endianness.LITTLE_ENDIAN) === true); // default to big endian let dataType: string = schema?.type; if (!dataType) { @@ -223,16 +250,38 @@ export default class OctetstreamCodec implements ContentCodec { valueToBytes(value: unknown, schema?: DataSchema, parameters: { [key: string]: string | undefined } = {}): Buffer { debug(`OctetstreamCodec serializing '${value}'`); - if (parameters.length == null) { - warn("Missing 'length' parameter necessary for write. I'll do my best"); + const bigEndian = !(parameters.byteSeq?.includes(Endianness.LITTLE_ENDIAN) === true); // default to big endian + + let signed = true; // default to true + + if (parameters.signed !== undefined) { + if (parameters.signed !== "true" && parameters.signed !== "false") { + throw new Error("'signed' parameter must be 'true' or 'false'"); + } + signed = parameters.signed === "true"; + } + + let length = + parameters.length != null + ? parseInt(parameters.length) + : (warn("Missing 'length' parameter necessary for write. I'll do my best"), undefined); + + if (length !== undefined && (isNaN(length) || length < 0)) { + throw new Error("'length' parameter must be a non-negative number"); } - const bigEndian = !(parameters.byteSeq?.includes(Endianness.LITTLE_ENDIAN) === true); // default to big endian - let signed = parameters.signed !== "false"; // default to signed - // byte length of the buffer to be returned - let length = parameters.length != null ? parseInt(parameters.length) : undefined; let bitLength = schema?.["ex:bitLength"] !== undefined ? parseInt(schema["ex:bitLength"]) : undefined; + + if (bitLength !== undefined && (isNaN(bitLength) || bitLength < 0)) { + throw new Error("'ex:bitLength' must be a non-negative number"); + } + const offset = schema?.["ex:bitOffset"] !== undefined ? parseInt(schema["ex:bitOffset"]) : 0; + + if (isNaN(offset) || offset < 0) { + throw new Error("'ex:bitOffset' must be a non-negative number"); + } + let dataType: string = schema?.type ?? undefined; if (value === undefined) { @@ -547,11 +596,11 @@ export default class OctetstreamCodec implements ContentCodec { const offset = schema["ex:bitOffset"] !== undefined ? parseInt(schema["ex:bitOffset"]) : 0; if (isNaN(offset) || offset < 0) { - throw new Error("ex:bitOffset must be a non-negative number"); + throw new Error("'ex:bitOffset' must be a non-negative number"); } if (offset > length * 8) { - throw new Error(`ex:bitOffset ${offset} exceeds length ${length}`); + throw new Error(`'ex:bitOffset' ${offset} exceeds 'length' ${length}`); } result = result ?? Buffer.alloc(length); diff --git a/packages/core/test/ContentSerdesTest.ts b/packages/core/test/ContentSerdesTest.ts index 95355c286..d67b460a7 100644 --- a/packages/core/test/ContentSerdesTest.ts +++ b/packages/core/test/ContentSerdesTest.ts @@ -584,6 +584,55 @@ class SerdesOctetTests { { type: "uint8" } ) ).to.throw(Error, "Type is unsigned but 'signed' is true"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream;length=test`, body: Buffer.from([0x36]) }, + { type: "integer" } + ) + ).to.throw(Error, "'length' parameter must be a non-negative number"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream;length=-1`, body: Buffer.from([0x36]) }, + { type: "integer" } + ) + ).to.throw(Error, "'length' parameter must be a non-negative number"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream;signed=invalid`, body: Buffer.from([0x36]) }, + { type: "integer" } + ) + ).to.throw(Error, "'signed' parameter must be 'true' or 'false'"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream`, body: Buffer.from([0x36]) }, + { type: "integer", "ex:bitOffset": "invalid" } + ) + ).to.throw(Error, "'ex:bitOffset' must be a non-negative number"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream`, body: Buffer.from([0x36]) }, + { type: "integer", "ex:bitOffset": -1 } + ) + ).to.throw(Error, "'ex:bitOffset' must be a non-negative number"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream`, body: Buffer.from([0x36]) }, + { type: "integer", "ex:bitLength": "invalid" } + ) + ).to.throw(Error, "'ex:bitLength' must be a non-negative number"); + + expect(() => + ContentSerdes.contentToValue( + { type: `application/octet-stream`, body: Buffer.from([0x36]) }, + { type: "integer", "ex:bitLength": -1 } + ) + ).to.throw(Error, "'ex:bitLength' must be a non-negative number"); } @test async "value to OctetStream"() { @@ -937,6 +986,29 @@ class SerdesOctetTests { Error, "Missing 'type' property in schema" ); + expect(() => ContentSerdes.valueToContent(10, { type: "int8" }, "application/octet-stream;signed=8")).to.throw( + Error, + "'signed' parameter must be 'true' or 'false'" + ); + expect(() => + ContentSerdes.valueToContent(10, { type: "int8" }, "application/octet-stream;length=-1;") + ).to.throw(Error, "'length' parameter must be a non-negative number"); + expect(() => ContentSerdes.valueToContent(10, { type: "int8" }, "application/octet-stream;length=x;")).to.throw( + Error, + "'length' parameter must be a non-negative number" + ); + expect(() => + ContentSerdes.valueToContent(10, { type: "integer", "ex:bitOffset": -16 }, "application/octet-stream") + ).to.throw(Error, "'ex:bitOffset' must be a non-negative number"); + expect(() => + ContentSerdes.valueToContent(10, { type: "integer", "ex:bitOffset": "foo" }, "application/octet-stream") + ).to.throw(Error, "'ex:bitOffset' must be a non-negative number"); + expect(() => + ContentSerdes.valueToContent(10, { type: "integer", "ex:bitLength": -8 }, "application/octet-stream") + ).to.throw(Error, "'ex:bitLength' must be a non-negative number"); + expect(() => + ContentSerdes.valueToContent(10, { type: "integer", "ex:bitLength": "foo" }, "application/octet-stream") + ).to.throw(Error, "'ex:bitLength' must be a non-negative number"); } }