Add primitive variables (#125)

* Add context variables to compiler

* new `addVariable` to ProtodefCompiler, `setVariable` to CompiledProtodef
* / at top of switch field will notate a context variable

* add reserved field

* add variables to interpreter, add example

* add to API docs

* update protodef submodule

* update protodef submodule

* test tests

* Update package.json

Co-authored-by: Romain Beaumont <romain.rom1@gmail.com>

Author: extremeheat

ProtoDef

@@ -26,6 +26,10 @@ The `path` is an array of namespace keys which select a path of namespaces to be
 
 See full_protocol.js for an example of usage.
 
+### ProtoDef.setVariable(name, value)
+
+Sets a primitive variable type for the specified `name`, which can be dynamically updated. Can be refrenced in switch statements with the "/" prefix.
+
 ### ProtoDef.read(buffer, cursor, _fieldInfo, rootNodes)
 
 Read the packet defined by `_fieldInfo` in `buffer` starting from `cursor` using the context `rootNodes`.
@@ -80,6 +84,10 @@ Add types in `protocol` recursively. The protocol object is an object with keys
 
 The `path` is an array of namespace keys which select a path of namespaces to be added to the protodef object.
 
+### ProtoDefCompiler.addVariable(name, value)
+
+Adds a primitive variable type for the specified `name`, which can be dynamically updated. Can be refrenced in switch statements with the "/" prefix.
+
 ### ProtoDefCompiler.compileProtoDefSync(options = { printCode: false })
 
 Compile and return a `ProtoDef` object, optionaly print the generated javascript code.
@@ -94,6 +102,11 @@ sizeOfCtx, writeCtx and readCtx are the compiled version of sizeOf, write and re
 
 It can be used directly for easier debugging/using already compiled js.
 
+### CompiledProtodef.setVariable(name, value)
+
+Sets a primitive variable type for the specified `name`, which can be dynamically updated. Can be refrenced in switch statements with the "/" prefix.
+
+
 ## utils
 
 Some functions that can be useful to build new datatypes reader and writer.

doc/api.md

@@ -0,0 +1,31 @@
+const { ProtoDef } = require('protodef')
+const assert = require('assert')
+
+// Create a protocol where DrawText is sent with a "opacity" field at the end only if the color isn't transparent.
+const protocol = {
+  string: ['pstring', { countType: 'varint' }],
+  ColorPalette: ['container', [{ name: 'palette', type: ['array', { countType: 'i32', type: 'string' }] }]],
+  DrawText: ['container', [{ name: 'color', type: 'u8' }, { name: 'opacity', type: ['switch', { compareTo: 'color', fields: { '/color_transparent': 'void' }, default: 'u8' }] }]]
+}
+
+function test () {
+  // A "palette" here refers to a array of values, identified with their index in the array
+  const palette = ['red', 'green', 'blue', 'transparent']
+  const proto = new ProtoDef()
+  proto.addTypes(protocol)
+  // A "variable" is similar to a type, it's a primitive value that can be used in switch comparisons.
+  proto.setVariable('color_transparent', palette.indexOf('transparent'))
+  // An example usage is sending paletted IDs, with feild serialization based on those IDs
+  proto.createPacketBuffer('ColorPalette', { palette })
+  // Here, "opacity", 0x4 is written *only* if the color isn't transparent. In this case, it is, so 0x4 isn't written.
+  // At the top is 0x3, the index of the "transparent" color.
+  const s = proto.createPacketBuffer('DrawText', { color: palette.indexOf('transparent'), opacity: 4 })
+  assert(s.equals(Buffer.from([3])))
+  console.log(s)
+
+  // Here 4 should be written at the end
+  const t = proto.createPacketBuffer('DrawText', { color: palette.indexOf('blue'), opacity: 4 })
+  assert(t.equals(Buffer.from([2, 4])))
+}
+
+test()

examples/variable.js

@@ -16,7 +16,7 @@
   "dependencies": {
     "lodash.get": "^4.4.2",
     "lodash.reduce": "^4.6.0",
-    "protodef-validator": "^1.2.2",
+    "protodef-validator": "^1.3.0",
     "readable-stream": "^3.0.3"
   },
   "engines": {

package.json

@@ -32,6 +32,12 @@ class ProtoDefCompiler {
     this.sizeOfCompiler.addProtocol(protocolData, path)
   }
 
+  addVariable (key, val) {
+    this.readCompiler.addContextType(key, val)
+    this.writeCompiler.addContextType(key, val)
+    this.sizeOfCompiler.addContextType(key, val)
+  }
+
   compileProtoDefSync (options = { printCode: false }) {
     const sizeOfCode = this.sizeOfCompiler.generate()
     const writeCode = this.writeCompiler.generate()
@@ -70,6 +76,12 @@ class CompiledProtodef {
     return writeFn(value, buffer, cursor)
   }
 
+  setVariable (key, val) {
+    this.sizeOfCtx[key] = val
+    this.readCtx[key] = val
+    this.writeCtx[key] = val
+  }
+
   sizeOf (value, type) {
     const sizeFn = this.sizeOfCtx[type]
     if (!sizeFn) { throw new Error('missing data type: ' + type) }
@@ -186,7 +198,7 @@ class Compiler {
   getField (name) {
     const path = name.split('/')
     let i = this.scopeStack.length - 1
-    const reserved = ['value', 'enum', 'default', 'size']
+    const reserved = ['value', 'enum', 'default', 'size', 'offset']
     while (path.length) {
       const scope = this.scopeStack[i]
       const field = path.shift()

src/compiler.js

@@ -10,7 +10,8 @@ module.exports = {
       let code = `switch (${compare}) {\n`
       for (const key in struct.fields) {
         let val = key
-        if (isNaN(val) && val !== 'true' && val !== 'false') val = `"${val}"`
+        if (val.startsWith('/')) val = 'ctx.' + val.slice(1) // Root context variable
+        else if (isNaN(val) && val !== 'true' && val !== 'false') val = `"${val}"`
         code += compiler.indent(`case ${val}: return ` + compiler.callType(struct.fields[key])) + '\n'
       }
       code += compiler.indent('default: return ' + compiler.callType(struct.default ? struct.default : 'void')) + '\n'
@@ -39,7 +40,8 @@ module.exports = {
       let code = `switch (${compare}) {\n`
       for (const key in struct.fields) {
         let val = key
-        if (isNaN(val) && val !== 'true' && val !== 'false') val = `"${val}"`
+        if (val.startsWith('/')) val = 'ctx.' + val.slice(1) // Root context variable
+        else if (isNaN(val) && val !== 'true' && val !== 'false') val = `"${val}"`
         code += compiler.indent(`case ${val}: return ` + compiler.callType('value', struct.fields[key])) + '\n'
       }
       code += compiler.indent('default: return ' + compiler.callType('value', struct.default ? struct.default : 'void')) + '\n'
@@ -69,7 +71,8 @@ module.exports = {
       let code = `switch (${compare}) {\n`
       for (const key in struct.fields) {
         let val = key
-        if (isNaN(val) && val !== 'true' && val !== 'false') val = `"${val}"`
+        if (val.startsWith('/')) val = 'ctx.' + val.slice(1) // Root context variable
+        else if (isNaN(val) && val !== 'true' && val !== 'false') val = `"${val}"`
         code += compiler.indent(`case ${val}: return ` + compiler.callType('value', struct.fields[key])) + '\n'
       }
       code += compiler.indent('default: return ' + compiler.callType('value', struct.default ? struct.default : 'void')) + '\n'

src/datatypes/compiler-conditional.js

@@ -8,7 +8,12 @@ module.exports = {
 function readSwitch (buffer, offset, { compareTo, fields, compareToValue, default: defVal }, rootNode) {
   compareTo = compareToValue !== undefined ? compareToValue : getField(compareTo, rootNode)
   if (typeof fields[compareTo] === 'undefined' && typeof defVal === 'undefined') { throw new Error(compareTo + ' has no associated fieldInfo in switch') }
-
+  for (const field in fields) {
+    if (field.startsWith('/')) {
+      fields[this.types[field.slice(1)]] = fields[field]
+      delete fields[field]
+    }
+  }
   const caseDefault = typeof fields[compareTo] === 'undefined'
   const resultingType = caseDefault ? defVal : fields[compareTo]
   const fieldInfo = getFieldInfo(resultingType)
@@ -18,7 +23,12 @@ function readSwitch (buffer, offset, { compareTo, fields, compareToValue, defaul
 function writeSwitch (value, buffer, offset, { compareTo, fields, compareToValue, default: defVal }, rootNode) {
   compareTo = compareToValue !== undefined ? compareToValue : getField(compareTo, rootNode)
   if (typeof fields[compareTo] === 'undefined' && typeof defVal === 'undefined') { throw new Error(compareTo + ' has no associated fieldInfo in switch') }
-
+  for (const field in fields) {
+    if (field.startsWith('/')) {
+      fields[this.types[field.slice(1)]] = fields[field]
+      delete fields[field]
+    }
+  }
   const caseDefault = typeof fields[compareTo] === 'undefined'
   const fieldInfo = getFieldInfo(caseDefault ? defVal : fields[compareTo])
   return tryDoc(() => this.write(value, buffer, offset, fieldInfo, rootNode), caseDefault ? 'default' : compareTo)
@@ -27,7 +37,12 @@ function writeSwitch (value, buffer, offset, { compareTo, fields, compareToValue
 function sizeOfSwitch (value, { compareTo, fields, compareToValue, default: defVal }, rootNode) {
   compareTo = compareToValue !== undefined ? compareToValue : getField(compareTo, rootNode)
   if (typeof fields[compareTo] === 'undefined' && typeof defVal === 'undefined') { throw new Error(compareTo + ' has no associated fieldInfo in switch') }
-
+  for (const field in fields) {
+    if (field.startsWith('/')) {
+      fields[this.types[field.slice(1)]] = fields[field]
+      delete fields[field]
+    }
+  }
   const caseDefault = typeof fields[compareTo] === 'undefined'
   const fieldInfo = getFieldInfo(caseDefault ? defVal : fields[compareTo])
   return tryDoc(() => this.sizeOf(value, fieldInfo, rootNode), caseDefault ? 'default' : compareTo)

src/datatypes/conditional.js

@@ -103,6 +103,10 @@ class ProtoDef {
     }
   }
 
+  setVariable (key, val) {
+    this.types[key] = val
+  }
+
   read (buffer, cursor, _fieldInfo, rootNodes) {
     const { type, typeArgs } = getFieldInfo(_fieldInfo)
     const typeFunctions = this.types[type]

src/protodef.js

@@ -28,11 +28,22 @@ function arrayToBuffer (arr) {
 }
 
 function transformValues (type, values) {
-  return values.map(value => ({
-    buffer: arrayToBuffer(value.buffer),
-    value: type.indexOf('buffer') === 0 ? arrayToBuffer(value.value) : value.value,
-    description: value.description
-  }))
+  return values.map(val => {
+    let value = val.value
+    if (type.indexOf('buffer') === 0) {
+      value = arrayToBuffer(value)
+    } else if (value) {
+      // we cannot use undefined type in JSON so need to convert it here to pass strictEquals test
+      for (const key in value) {
+        if (value[key] === 'undefined') value[key] = undefined
+      }
+    }
+    return {
+      buffer: arrayToBuffer(val.buffer),
+      value,
+      description: val.description
+    }
+  })
 }
 
 testData.forEach(tests => {
@@ -47,6 +58,7 @@ testData.forEach(tests => {
         types[type] = subtype.type
         compiler.addTypesToCompile(types)
 
+        subtype.vars?.forEach(([k, v]) => { proto.setVariable(k, v); compiler.addVariable(k, v) })
         subtype.values = transformValues(test.type, subtype.values)
         subtype.type = type
         subTypes.push(subtype)