Skip to content

Commit 757a8bd

Browse files
lorowlorow
authored
feat: Add support for configuring the module via OSC
* Add support for more OSC meessage types * Add support for mapping halved values properly * PoC move updating VRCFT state to Update() instead of affecting it right away TODO: - tweak emulation values - performance profile - config staleness after updating by ETVR App * Rework config updating - fix stale config issue in mappers * Move SmoothStep to utils directory * Add Custom listening IP address to config * Add support for sending string and Ip address values over OSC * Fix emulating eyebrows on v2 mapper crashing * Make emulation opt-in * Fix wonky widen / squeeze emulation on v1 params * Adjust default emulation settings * Remove IP address from updateable fields - wrong idea * Switch default listening address from loopback to Any
1 parent 7833de2 commit 757a8bd

12 files changed

Lines changed: 414 additions & 121 deletions

‎ETVRTrackingModule.cs‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,23 @@ namespace ETVRTrackingModule
88
public class ETVRTrackingModule : ExtTrackingModule
99
{
1010
private OSCManager? _oscManager;
11-
private ExpressionsMapper? _expressionMapper;
11+
private ExpressionsMapperManager? _expressionMapper;
1212
public override (bool SupportsEye, bool SupportsExpression) Supported => (true, false);
1313
public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eyeAvailable, bool expressionAvailable)
1414
{
1515
ModuleInformation.Name = "ETVR Eye Tracking module";
1616
var stream = GetType().Assembly.GetManifestResourceStream("ETVRTrackingModule.Assets.ETVRLogo.png");
1717
ModuleInformation.StaticImages = stream != null? new List<Stream> { stream } : ModuleInformation.StaticImages;
1818

19-
var currentPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
20-
ETVRConfigManager config = new ETVRConfigManager(currentPath, Logger);
21-
config.LoadConfig();
19+
var currentPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
20+
ETVRConfigManager configManager = new ETVRConfigManager(currentPath, Logger);
21+
configManager.LoadConfig();
2222

23-
_expressionMapper = new ExpressionsMapper(Logger, ref config);
24-
_oscManager = new OSCManager(Logger, _expressionMapper, ref config);
23+
_expressionMapper = new ExpressionsMapperManager(Logger, configManager.Config);
24+
_expressionMapper.RegisterSelf(ref configManager);
25+
26+
_oscManager = new OSCManager(Logger, _expressionMapper);
27+
_oscManager.RegisterSelf(ref configManager);
2528
_oscManager.Start();
2629

2730
if (_oscManager.State == OSCState.CONNECTED) return (true, false);
@@ -37,7 +40,8 @@ public override void Teardown()
3740

3841
public override void Update()
3942
{
40-
Thread.Sleep(100);
43+
_expressionMapper!.UpdateVRCFTState();
44+
Thread.Sleep(5);
4145
}
4246
}
4347
}

‎ExpressionStrategies/BaseParamMapper.cs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,29 @@
44

55
namespace ETVRTrackingModule.ExpressionStrategies;
66

7-
public class BaseParamMapper : IMappingStategy
7+
public class BaseParamMapper : IMappingStrategy
88
{
99
protected ILogger _logger;
10-
protected readonly Config _config;
10+
protected Config _config;
1111

1212
protected OneEuroFilter _leftOneEuroFilter;
1313
protected OneEuroFilter _rightOneEuroFilter;
1414

15-
public BaseParamMapper(ILogger logger, ref Config config)
15+
public BaseParamMapper(ILogger logger, Config config)
1616
{
1717
_leftOneEuroFilter = new OneEuroFilter(minCutoff: 0.1f, beta: 15.0f);
1818
_rightOneEuroFilter = new OneEuroFilter(minCutoff: 0.1f, beta: 15.0f);
1919
_logger = logger;
2020
_config = config;
2121
}
2222

23-
public virtual void handleOSCMessage(OSCMessage message)
23+
public void UpdateConfig(Config config)
24+
{
25+
_config = config;
26+
_logger.LogInformation("config update: {}", config.ShouldEmulateEyeWiden);
27+
}
28+
29+
public virtual void HandleOSCMessage(OSCMessage message)
2430
{
2531
}
2632

@@ -30,6 +36,10 @@ protected static string GetParamToMap(string oscAddress)
3036
return oscUrlSplit[^1];
3137
}
3238

39+
public virtual void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes)
40+
{
41+
}
42+
3343
private protected void _emulateEyeBrow(
3444
ref UnifiedExpressionShape[] eyeShapes,
3545
UnifiedExpressions eyebrowExpressionLowerrer,
@@ -45,7 +55,7 @@ private protected void _emulateEyeBrow(
4555
var filteredBaseOpenness = (float)oneEuroFilter.Filter(baseEyeOpenness, 1);
4656
if (filteredBaseOpenness >= riseThreshold)
4757
{
48-
eyeShapes[(int)eyebrowExpressionUpper].Weight = Utils.SmoothStep(
58+
eyeShapes[(int)eyebrowExpressionUpper].Weight = Utils.MathUtils.SmoothStep(
4959
riseThreshold,
5060
1,
5161
filteredBaseOpenness
@@ -54,7 +64,7 @@ private protected void _emulateEyeBrow(
5464

5565
if (filteredBaseOpenness <= lowerThreshold)
5666
{
57-
eyeShapes[(int)eyebrowExpressionLowerrer].Weight = Utils.SmoothStep(
67+
eyeShapes[(int)eyebrowExpressionLowerrer].Weight = Utils.MathUtils.SmoothStep(
5868
lowerThreshold,
5969
1,
6070
filteredBaseOpenness

‎ExpressionStrategies/IExpressionMapper.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
namespace ETVRTrackingModule.ExpressionStrategies;
44

5-
public interface IMappingStategy
5+
public interface IMappingStrategy
66
{
7-
public void handleOSCMessage(OSCMessage message);
7+
public void HandleOSCMessage(OSCMessage message);
88
}

‎ExpressionStrategies/V1Mapper.cs‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using Microsoft.Extensions.Logging;
1+
using System.Security.Cryptography.X509Certificates;
2+
using Microsoft.Extensions.Logging;
23
using VRCFaceTracking;
34
using VRCFaceTracking.Core.Params.Data;
45
using VRCFaceTracking.Core.Params.Expressions;
@@ -17,21 +18,26 @@ public class V1Mapper : BaseParamMapper
1718
{ "EyesY", 0f },
1819
};
1920

20-
public V1Mapper(ILogger logger, ref Config config) : base(logger, ref config)
21-
{
22-
}
21+
public V1Mapper(ILogger logger, Config config) : base(logger, config) {}
2322

24-
public override void handleOSCMessage(OSCMessage message)
23+
public override void HandleOSCMessage(OSCMessage message)
2524
{
2625
var paramToMap = GetParamToMap(message.address);
2726
if (_parameterValues.ContainsKey(paramToMap))
2827
{
29-
_parameterValues[paramToMap] = message.value;
30-
UpdateVRCFTEyeData(ref UnifiedTracking.Data.Eye, ref UnifiedTracking.Data.Shapes);
28+
if (message.value is not OSCFloat oscF)
29+
{
30+
_logger.LogInformation("ParamMapper got passed a wrong type of message: {}", message.value.Type);
31+
return;
32+
}
33+
else
34+
{
35+
_parameterValues[paramToMap] = oscF.value;
36+
}
3137
}
3238
}
3339

34-
private void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes)
40+
public override void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes)
3541
{
3642
HandleEyeGaze(ref eyeData);
3743
HandleEyeOpenness(ref eyeData, ref eyeShapes);
@@ -75,25 +81,34 @@ Config config
7581
if (_config.ShouldEmulateEyeWiden && baseEyeOpenness >= config.WidenThresholdV1[0])
7682
{
7783
eye.Openness = 0.8f;
78-
var widenValue = Utils.SmoothStep(
84+
var widenValue = Utils.MathUtils.SmoothStep(
7985
config.WidenThresholdV1[0],
8086
config.WidenThresholdV1[1],
8187
baseEyeOpenness
8288
) * config.OutputMultiplier;
8389
eyeShapes[(int)widenParam].Weight = widenValue;
84-
eyeShapes[(int)squintParam].Weight = 0;
90+
}
91+
// we gotta reset it manually, otherwise VRCFT will just persist it, leading to wonky behaviour
92+
else
93+
{
94+
eyeShapes[(int)widenParam].Weight = 0;
8595
}
8696

8797
if (_config.ShouldEmulateEyeSquint && baseEyeOpenness <= config.SqueezeThresholdV1[0])
8898
{
8999
eyeShapes[(int)widenParam].Weight = 0;
90-
var squintValue = Utils.SmoothStep(
100+
var squintValue = Utils.MathUtils.SmoothStep(
91101
config.SqueezeThresholdV1[1],
92102
config.SqueezeThresholdV1[0],
93103
baseEyeOpenness
94104
) * config.OutputMultiplier;
95105
eyeShapes[(int)squintParam].Weight = squintValue;
96106
}
107+
else
108+
{
109+
eyeShapes[(int)squintParam].Weight = 0;
110+
}
111+
97112
}
98113

99114
private void EmulateEyeBrows(ref UnifiedExpressionShape[] eyeShapes)

‎ExpressionStrategies/V2Mapper.cs‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -46,32 +46,32 @@ public class V2Mapper : BaseParamMapper
4646
"EyeLidRight",
4747
};
4848

49-
public V2Mapper(ILogger logger, ref Config config) : base(logger, ref config)
50-
{
51-
}
49+
private bool _isSingleEye = false;
50+
51+
public V2Mapper(ILogger logger, Config config) : base(logger, config) {}
5252

53-
public override void handleOSCMessage(OSCMessage message)
53+
public override void HandleOSCMessage(OSCMessage message)
5454
{
5555
string paramToMap = GetParamToMap(message.address);
5656
if (!_parameterValues.ContainsKey(paramToMap))
5757
return;
5858

59-
_parameterValues[paramToMap] = message.value;
60-
var singleEyeMode = _singleEyeParamNames.Contains(paramToMap);
61-
UpdateVRCFTEyeData(ref UnifiedTracking.Data.Eye, ref UnifiedTracking.Data.Shapes, paramToMap, singleEyeMode);
59+
if (message.value is not OSCFloat oscF)
60+
{
61+
_logger.LogInformation("ParamMapper got passed a wrong type of message: {}", message.value.Type);
62+
return;
63+
}
64+
else
65+
_parameterValues[paramToMap] = oscF.value;
66+
67+
_isSingleEye = _singleEyeParamNames.Contains(paramToMap);
6268
}
6369

64-
private void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes,
65-
string parameter, bool isSingleEyeMode = false)
70+
public override void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes)
6671
{
67-
if (_gazeParameters.Contains(parameter))
68-
HandleEyeGaze(ref eyeData, isSingleEyeMode);
69-
70-
if (_opennessParameters.Contains(parameter))
71-
{
72-
HandleEyeOpenness(ref eyeData, ref eyeShapes, isSingleEyeMode);
73-
EmulateEyebrows(ref eyeShapes, isSingleEyeMode);
74-
}
72+
HandleEyeGaze(ref eyeData, _isSingleEye);
73+
HandleEyeOpenness(ref eyeData, ref eyeShapes, _isSingleEye);
74+
EmulateEyebrows(ref eyeShapes, _isSingleEye);
7575
}
7676

7777
private void HandleEyeGaze(ref UnifiedEyeData eyeData, bool isSingleEyeMode)
@@ -119,7 +119,7 @@ Config config
119119
eyeData.Openness = baseOpenness;
120120
if (_config.ShouldEmulateEyeWiden && baseOpenness >= config.WidenThresholdV2[0])
121121
{
122-
var opennessValue = Utils.SmoothStep(
122+
var opennessValue = Utils.MathUtils.SmoothStep(
123123
config.WidenThresholdV2[0],
124124
config.WidenThresholdV2[1],
125125
baseOpenness
@@ -129,7 +129,7 @@ Config config
129129

130130
if (_config.ShouldEmulateEyeSquint && baseOpenness <= config.SqueezeThresholdV2[0])
131131
{
132-
var opennessValue = Utils.SmoothStep(
132+
var opennessValue = Utils.MathUtils.SmoothStep(
133133
config.SqueezeThresholdV2[0],
134134
config.SqueezeThresholdV2[1],
135135
baseOpenness
@@ -166,9 +166,10 @@ private void EmulateEyebrows(ref UnifiedExpressionShape[] eyeShapes, bool isSing
166166

167167
return;
168168
}
169-
170-
var baseRightEyeOpenness = _parameterValues["RightEyeLidExpandedSqueeze"];
171-
var baseLeftEyeOpenness = _parameterValues["LeftEyeLidExpandedSqueeze"];
169+
170+
171+
var baseRightEyeOpenness = _parameterValues["EyeLidLeft"];
172+
var baseLeftEyeOpenness = _parameterValues["EyeLidRight"];
172173

173174
_emulateEyeBrow(
174175
ref eyeShapes,

‎ExpressionsMapper.cs‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,60 @@
11
using ETVRTrackingModule.ExpressionStrategies;
22
using Microsoft.Extensions.Logging;
3+
using VRCFaceTracking;
34

45
namespace ETVRTrackingModule
56
{
6-
public class ExpressionsMapper
7+
public class ExpressionsMapperManager
78
{
89
private V1Mapper _v1Mapper;
910
private V2Mapper _v2Mapper;
10-
11+
private BaseParamMapper _currentMapper;
12+
1113
ILogger _logger;
12-
public ExpressionsMapper(ILogger logger, ref ETVRConfigManager configManager)
14+
public ExpressionsMapperManager(ILogger logger, Config config)
1315
{
14-
var config = configManager.Config;
1516
_logger = logger;
16-
_v1Mapper = new V1Mapper(_logger, ref config);
17-
_v2Mapper = new V2Mapper(_logger, ref config);
17+
_v1Mapper = new V1Mapper(_logger, config);
18+
_v2Mapper = new V2Mapper(_logger, config);
19+
_currentMapper = _v1Mapper;
1820
}
1921

22+
public void RegisterSelf(ref ETVRConfigManager configManager)
23+
{
24+
configManager.RegisterListener(HandleConfigUpdate);
25+
}
26+
27+
private void HandleConfigUpdate(Config config)
28+
{
29+
_v1Mapper.UpdateConfig(config);
30+
_v2Mapper.UpdateConfig(config);
31+
}
32+
2033
public void MapMessage(OSCMessage msg)
2134
{
2235
if (!msg.success)
2336
return;
2437

2538
if (IsV2Param(msg))
2639
{
27-
_v2Mapper.handleOSCMessage(msg);
40+
_currentMapper = _v2Mapper;
41+
_v2Mapper.HandleOSCMessage(msg);
2842
return;
2943
}
3044

31-
_v1Mapper.handleOSCMessage(msg);
45+
_currentMapper = _v1Mapper;
46+
_v1Mapper.HandleOSCMessage(msg);
3247
}
3348

3449
private bool IsV2Param(OSCMessage oscMessage)
3550
{
3651
var isv2Param = oscMessage.address.Contains("/v2/");
3752
return isv2Param;
3853
}
54+
55+
public void UpdateVRCFTState()
56+
{
57+
_currentMapper.UpdateVRCFTEyeData(ref UnifiedTracking.Data.Eye, ref UnifiedTracking.Data.Shapes);
58+
}
3959
}
4060
}

0 commit comments

Comments
 (0)