add loop and restBuffer types

Author: extremeheat

src/compiler.js

@@ -4,6 +4,7 @@ const utils = require('./datatypes/utils')
 const conditionalDatatypes = require('./datatypes/compiler-conditional')
 const structuresDatatypes = require('./datatypes/compiler-structures')
 const utilsDatatypes = require('./datatypes/compiler-utils')
+const extrasDatatypes = require('./datatypes/compiler-extras')
 
 const { tryCatch } = require('./utils')
 
@@ -270,6 +271,7 @@ class ReadCompiler extends Compiler {
     this.addTypes(conditionalDatatypes.Read)
     this.addTypes(structuresDatatypes.Read)
     this.addTypes(utilsDatatypes.Read)
+    this.addTypes(extrasDatatypes.Read)
 
     // Add default types
     for (const key in numeric) {
@@ -320,6 +322,7 @@ class WriteCompiler extends Compiler {
     this.addTypes(conditionalDatatypes.Write)
     this.addTypes(structuresDatatypes.Write)
     this.addTypes(utilsDatatypes.Write)
+    this.addTypes(extrasDatatypes.Write)
 
     // Add default types
     for (const key in numeric) {
@@ -370,6 +373,7 @@ class SizeOfCompiler extends Compiler {
     this.addTypes(conditionalDatatypes.SizeOf)
     this.addTypes(structuresDatatypes.SizeOf)
     this.addTypes(utilsDatatypes.SizeOf)
+    this.addTypes(extrasDatatypes.SizeOf)
 
     // Add default types
     for (const key in numeric) {

src/datatypes/compiler-extras.js

@@ -0,0 +1,74 @@
+/* eslint-disable multiline-ternary */
+function validateNT (nt) {
+  if (nt !== null && typeof nt !== 'number') throw new Error('Loop terminator must be a number like 0 or null for EOF')
+  return nt !== null
+}
+
+module.exports = {
+  Read: {
+    loop: ['parametrizable', (compiler, struct) => {
+      const nt = struct.nt
+      const hasTerminator = validateNT(nt)
+
+      return compiler.wrapCode(`
+        const results = []
+        let size = 0
+        while (offset !== buffer.length) {
+          ${hasTerminator ? `
+          const typ = ctx.i8(buffer, offset)
+          if (typ.value === ${nt}) {
+            return { value: results, size: size + 1 }
+          }` : ''}
+          const entry = ${compiler.callType(struct.type)}
+          results.push(entry.value)
+          offset += entry.size
+          size += entry.size
+        }
+        return { value: results, size }
+      `)
+    }],
+    restBuffer: ['native', (buffer, offset) => {
+      return {
+        value: buffer.slice(offset),
+        size: buffer.length - offset
+      }
+    }]
+  },
+  Write: {
+    loop: ['parametrizable', (compiler, struct) => {
+      const nt = struct.nt
+      const hasTerminator = validateNT(nt)
+
+      return compiler.wrapCode(`
+        for (const key in value) {
+          offset = ${compiler.callType('value[key]', struct.type)}
+        }
+        ${hasTerminator ? `offset = ctx.i8(${nt}, buffer, offset)` : ''}
+        return offset
+      `)
+    }],
+    restBuffer: ['native', (value, buffer, offset) => {
+      if (!(value instanceof Buffer)) value = Buffer.from(value)
+      value.copy(buffer, offset)
+      return offset + value.length
+    }]
+  },
+  SizeOf: {
+    loop: ['parametrizable', (compiler, struct) => {
+      const nt = struct.nt
+      const hasTerminator = validateNT(nt)
+
+      return compiler.wrapCode(`
+        let size = ${hasTerminator ? '1' : '0'}
+        for (const key in value) {
+          size += ${compiler.callType('value[key]', struct.type)}
+        }
+        return size
+      `)
+    }],
+    restBuffer: ['native', (value) => {
+      if (!(value instanceof Buffer)) value = Buffer.from(value)
+      return value.length
+    }]
+  }
+}

src/datatypes/extras.js

@@ -0,0 +1,111 @@
+const { PartialReadError } = require('../utils')
+
+module.exports = {
+  loop: [readLoop, writeLoop, sizeOfLoop, require('../../ProtoDef/schemas/extras.json').loop],
+  restBuffer: [readRestBuffer, writeRestBuffer, sizeOfRestBuffer, require('../../ProtoDef/schemas/extras.json').restBuffer]
+}
+
+function readLoop (buffer, offset, typeArgs, rootNode) {
+  if (!typeArgs) {
+    throw new Error('typeArgs is required for loop type')
+  }
+
+  const results = []
+  const startOffset = offset
+  const nt = typeArgs.nt
+  const hasTerminator = nt !== null && typeof nt === 'number'
+
+  while (offset < buffer.length) {
+    // Check for terminator if specified
+    if (hasTerminator) {
+      if (offset >= buffer.length) break
+      const terminatorValue = buffer.readInt8(offset)
+      if (terminatorValue === nt) {
+        // Found terminator, consume it and return
+        return {
+          value: results,
+          size: offset - startOffset + 1
+        }
+      }
+    }
+
+    // Read the next element
+    try {
+      const entry = this.read(buffer, offset, typeArgs.type, rootNode)
+      results.push(entry.value)
+      offset += entry.size
+    } catch (error) {
+      if (error instanceof PartialReadError) {
+        break
+      }
+      throw error
+    }
+  }
+
+  return {
+    value: results,
+    size: offset - startOffset
+  }
+}
+
+function writeLoop (value, buffer, offset, typeArgs, rootNode) {
+  if (!typeArgs) {
+    throw new Error('typeArgs is required for loop type')
+  }
+
+  const nt = typeArgs.nt
+  const hasTerminator = nt !== null && typeof nt === 'number'
+
+  // Write each element in the array
+  for (const item of value) {
+    offset = this.write(item, buffer, offset, typeArgs.type, rootNode)
+  }
+
+  // Write terminator if specified
+  if (hasTerminator) {
+    buffer.writeInt8(nt, offset)
+    offset++
+  }
+
+  return offset
+}
+
+function sizeOfLoop (value, typeArgs, rootNode) {
+  if (!typeArgs) {
+    throw new Error('typeArgs is required for loop type')
+  }
+
+  const nt = typeArgs.nt
+  const hasTerminator = nt !== null && typeof nt === 'number'
+  let size = hasTerminator ? 1 : 0 // 1 byte for terminator if present
+
+  // Calculate size of all elements
+  for (const item of value) {
+    size += this.sizeOf(item, typeArgs.type, rootNode)
+  }
+
+  return size
+}
+
+function readRestBuffer (buffer, offset) {
+  const remainingBuffer = buffer.slice(offset)
+  return {
+    value: remainingBuffer,
+    size: remainingBuffer.length
+  }
+}
+
+function writeRestBuffer (value, buffer, offset) {
+  if (!(value instanceof Buffer)) {
+    value = Buffer.from(value)
+  }
+  value.copy(buffer, offset)
+  return offset + value.length
+}
+
+function sizeOfRestBuffer (value) {
+  if (!(value instanceof Buffer)) {
+    value = Buffer.from(value)
+  }
+  return value.length
+}

src/protodef.js

@@ -52,6 +52,7 @@ class ProtoDef {
     this.addTypes(require('./datatypes/utils'))
     this.addTypes(require('./datatypes/structures'))
     this.addTypes(require('./datatypes/conditional'))
+    this.addTypes(require('./datatypes/extras'))
   }
 
   addProtocol (protocolData, path) {

test/dataTypes/datatypes.js

@@ -55,7 +55,7 @@ function testType (type, values) {
       })
     } else { testValue(type, value.value, value.buffer) }
   })
-  if (type !== 'void') {
+  if (type !== 'void' && type !== 'restBuffer' && !type.startsWith('loop_')) {
     it('reads 0 bytes and throw a PartialReadError', () => {
       try {
         proto.parsePacketBuffer(type, Buffer.alloc(0))

test/dataTypes/prepareTests.js

@@ -20,6 +20,10 @@ const testData = [
   {
     kind: 'utils',
     data: require('../../ProtoDef/test/utils.json')
+  },
+  {
+    kind: 'extras',
+    data: require('../../ProtoDef/test/extras.json')
   }
 ]
 
@@ -30,7 +34,7 @@ function arrayToBuffer (arr) {
 function transformValues (type, values) {
   return values.map(val => {
     let value = val.value
-    if (type.indexOf('buffer') === 0) {
+    if (type.indexOf('buffer') === 0 || type.endsWith('Buffer')) {
       value = arrayToBuffer(value)
     } else if (value) {
       // we cannot use undefined type in JSON so need to convert it here to pass strictEquals test

test_extras.js

@@ -0,0 +1,23 @@
+const ProtoDef = require('./src/index').ProtoDef
+
+const proto = new ProtoDef()
+
+// Test basic types are available
+console.log('Available types:', Object.keys(proto.types))
+
+// Test the restBuffer type
+try {
+  const testBuffer = Buffer.from([1, 2, 3, 4, 5])
+  const result = proto.read(testBuffer, 2, 'restBuffer')
+  console.log('restBuffer result:', result)
+} catch (e) {
+  console.log('restBuffer error:', e.message)
+}
+
+// Test the loop type
+try {
+  const result = proto.read(Buffer.from([1, 2, 3, 0]), 0, ['loop', { type: 'i8', nt: 0 }])
+  console.log('loop result:', result)
+} catch (e) {
+  console.log('loop error:', e.message)
+}