Skip to content

Commit ac1ed30

Browse files
lorowlorow
authored
Add the ability to customize max widen and squeeze values (#1)
* Add the ability to customize max widen and squeeze values * Simplify limits implementation. Breaking change, config has to be reset * Document MaxWidenSqueezeThresholdV variables, add output multiplier, refactor some parts * Add OneEuroFilter filtering to hopefully fix jittering issues * Redo the threshold limiters, now they feel more intuitive to use * Fix squint/widen emulation in v2, fix squint emulation in v1, tune emulation values, improve documentation * Fix weird twitching issue with V2 parameters
1 parent 6a65e7e commit ac1ed30

6 files changed

Lines changed: 277 additions & 90 deletions

File tree

‎ExpressionStrategies/BaseParamMapper.cs‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,46 +9,55 @@ public class BaseParamMapper : IMappingStategy
99
protected ILogger _logger;
1010
protected readonly Config _config;
1111

12+
protected OneEuroFilter _leftOneEuroFilter;
13+
protected OneEuroFilter _rightOneEuroFilter;
14+
1215
public BaseParamMapper(ILogger logger, ref Config config)
1316
{
17+
_leftOneEuroFilter = new OneEuroFilter(minCutoff: 0.1f, beta: 15.0f);
18+
_rightOneEuroFilter = new OneEuroFilter(minCutoff: 0.1f, beta: 15.0f);
1419
_logger = logger;
1520
_config = config;
1621
}
17-
18-
public virtual void handleOSCMessage(OSCMessage message) {}
22+
23+
public virtual void handleOSCMessage(OSCMessage message)
24+
{
25+
}
1926

2027
protected static string GetParamToMap(string oscAddress)
2128
{
2229
var oscUrlSplit = oscAddress.Split("/");
2330
return oscUrlSplit[^1];
2431
}
25-
32+
2633
private protected void _emulateEyeBrow(
2734
ref UnifiedExpressionShape[] eyeShapes,
2835
UnifiedExpressions eyebrowExpressionLowerrer,
2936
UnifiedExpressions eyebrowExpressionUpper,
37+
ref OneEuroFilter oneEuroFilter,
3038
float baseEyeOpenness,
3139
float riseThreshold,
3240
float lowerThreshold)
3341
{
3442
if (!_config.ShouldEmulateEyebrows)
3543
return;
3644

37-
if (baseEyeOpenness >= riseThreshold)
45+
var filteredBaseOpenness = (float)oneEuroFilter.Filter(baseEyeOpenness, 1);
46+
if (filteredBaseOpenness >= riseThreshold)
3847
{
3948
eyeShapes[(int)eyebrowExpressionUpper].Weight = Utils.SmoothStep(
4049
riseThreshold,
4150
1,
42-
baseEyeOpenness
51+
filteredBaseOpenness
4352
);
4453
}
4554

46-
if (baseEyeOpenness <= lowerThreshold)
55+
if (filteredBaseOpenness <= lowerThreshold)
4756
{
4857
eyeShapes[(int)eyebrowExpressionLowerrer].Weight = Utils.SmoothStep(
4958
lowerThreshold,
5059
1,
51-
baseEyeOpenness
60+
filteredBaseOpenness
5261
);
5362
}
5463
}

‎ExpressionStrategies/V1Mapper.cs‎

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ public class V1Mapper : BaseParamMapper
1616
{ "RightEyeX", 0f },
1717
{ "EyesY", 0f },
1818
};
19-
20-
public V1Mapper(ILogger logger, ref Config config) : base(logger, ref config) { }
19+
20+
public V1Mapper(ILogger logger, ref Config config) : base(logger, ref config)
21+
{
22+
}
2123

2224
public override void handleOSCMessage(OSCMessage message)
2325
{
@@ -41,22 +43,23 @@ private void HandleEyeGaze(ref UnifiedEyeData eyeData)
4143
eyeData.Right.Gaze = new Vector2(_parameterValues["RightEyeX"], _parameterValues["EyesY"]);
4244
eyeData.Left.Gaze = new Vector2(_parameterValues["LeftEyeX"], _parameterValues["EyesY"]);
4345
}
44-
46+
4547
private void HandleEyeOpenness(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes)
4648
{
4749
// so how it works, currently we cannot output values above 1.0 and below 0.0
4850
// which means, we cannot really output whether someone's squeezing their eyes
4951
// or making a surprised face. Therefore, we kinda have to cheat.
5052
// If we detect that the values provided by ETVR are below or above a certain threshold
5153
// we fake the squeeze and widen
52-
var baseRightEyeOpenness = _parameterValues["RightEyeLidExpandedSqueeze"];
53-
var baseLeftEyeOpenness = _parameterValues["LeftEyeLidExpandedSqueeze"];
54+
55+
var baseRightEyeOpenness = (float)_leftOneEuroFilter.Filter(_parameterValues["RightEyeLidExpandedSqueeze"], 1);
56+
var baseLeftEyeOpenness = (float)_rightOneEuroFilter.Filter(_parameterValues["LeftEyeLidExpandedSqueeze"], 1);
5457

5558
_handleSingleEyeOpenness(ref eyeData.Right, ref eyeShapes, UnifiedExpressions.EyeWideRight,
56-
UnifiedExpressions.EyeSquintRight, baseRightEyeOpenness, _config.WidenThreshold, _config.SqueezeThreshold);
59+
UnifiedExpressions.EyeSquintRight, baseRightEyeOpenness, _config);
5760

5861
_handleSingleEyeOpenness(ref eyeData.Left, ref eyeShapes, UnifiedExpressions.EyeWideLeft,
59-
UnifiedExpressions.EyeSquintLeft, baseLeftEyeOpenness, _config.WidenThreshold, _config.SqueezeThreshold);
62+
UnifiedExpressions.EyeSquintLeft, baseLeftEyeOpenness, _config);
6063
}
6164

6265
private void _handleSingleEyeOpenness(
@@ -65,50 +68,54 @@ private void _handleSingleEyeOpenness(
6568
UnifiedExpressions widenParam,
6669
UnifiedExpressions squintParam,
6770
float baseEyeOpenness,
68-
float widenThreshold,
69-
float squeezeThreshold
71+
Config config
7072
)
7173
{
7274
eye.Openness = baseEyeOpenness;
73-
if (_config.ShouldEmulateEyeWiden && baseEyeOpenness >= widenThreshold)
75+
if (_config.ShouldEmulateEyeWiden && baseEyeOpenness >= config.WidenThresholdV1[0])
7476
{
75-
eyeShapes[(int)widenParam].Weight = Utils.SmoothStep(
76-
widenThreshold,
77-
1,
77+
eye.Openness = 0.8f;
78+
var widenValue = Utils.SmoothStep(
79+
config.WidenThresholdV1[0],
80+
config.WidenThresholdV1[1],
7881
baseEyeOpenness
79-
);
82+
) * config.OutputMultiplier;
83+
eyeShapes[(int)widenParam].Weight = widenValue;
8084
eyeShapes[(int)squintParam].Weight = 0;
8185
}
8286

83-
if (_config.ShouldEmulateEyeSquint && baseEyeOpenness <= squeezeThreshold)
87+
if (_config.ShouldEmulateEyeSquint && baseEyeOpenness <= config.SqueezeThresholdV1[0])
8488
{
8589
eyeShapes[(int)widenParam].Weight = 0;
86-
eyeShapes[(int)squintParam].Weight = Utils.SmoothStep(
87-
squeezeThreshold,
88-
0,
90+
var squintValue = Utils.SmoothStep(
91+
config.SqueezeThresholdV1[1],
92+
config.SqueezeThresholdV1[0],
8993
baseEyeOpenness
90-
);
94+
) * config.OutputMultiplier;
95+
eyeShapes[(int)squintParam].Weight = squintValue;
9196
}
9297
}
9398

9499
private void EmulateEyeBrows(ref UnifiedExpressionShape[] eyeShapes)
95100
{
96-
var baseRightEyeOpenness = _parameterValues["RightEyeLidExpandedSqueeze"];
97-
var baseLeftEyeOpenness = _parameterValues["LeftEyeLidExpandedSqueeze"];
98-
101+
var baseRightEyeOpenness = (float)_leftOneEuroFilter.Filter(_parameterValues["RightEyeLidExpandedSqueeze"], 1);
102+
var baseLeftEyeOpenness = (float)_rightOneEuroFilter.Filter(_parameterValues["LeftEyeLidExpandedSqueeze"], 1);
103+
99104
_emulateEyeBrow(
100105
ref eyeShapes,
101106
UnifiedExpressions.BrowLowererRight,
102107
UnifiedExpressions.BrowOuterUpRight,
108+
ref _leftOneEuroFilter,
103109
baseRightEyeOpenness,
104110
_config.EyebrowThresholdRising,
105111
_config.EyebrowThresholdLowering
106112
);
107-
113+
108114
_emulateEyeBrow(
109115
ref eyeShapes,
110116
UnifiedExpressions.BrowLowererLeft,
111117
UnifiedExpressions.BrowOuterUpLeft,
118+
ref _rightOneEuroFilter,
112119
baseLeftEyeOpenness,
113120
_config.EyebrowThresholdRising,
114121
_config.EyebrowThresholdLowering

‎ExpressionStrategies/V2Mapper.cs‎

Lines changed: 59 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,26 @@ public class V2Mapper : BaseParamMapper
2929
{ "EyeLidRight", 1f },
3030
};
3131

32-
public V2Mapper(ILogger logger, ref Config config) : base(logger, ref config) { }
32+
private readonly string[] _gazeParameters =
33+
{
34+
"EyeX",
35+
"EyeY",
36+
"EyeLeftX",
37+
"EyeLeftY",
38+
"EyeRightX",
39+
"EyeRightY",
40+
};
41+
42+
private readonly string[] _opennessParameters =
43+
{
44+
"EyeLid",
45+
"EyeLidLeft",
46+
"EyeLidRight",
47+
};
48+
49+
public V2Mapper(ILogger logger, ref Config config) : base(logger, ref config)
50+
{
51+
}
3352

3453
public override void handleOSCMessage(OSCMessage message)
3554
{
@@ -39,14 +58,20 @@ public override void handleOSCMessage(OSCMessage message)
3958

4059
_parameterValues[paramToMap] = message.value;
4160
var singleEyeMode = _singleEyeParamNames.Contains(paramToMap);
42-
UpdateVRCFTEyeData(ref UnifiedTracking.Data.Eye, ref UnifiedTracking.Data.Shapes, singleEyeMode);
61+
UpdateVRCFTEyeData(ref UnifiedTracking.Data.Eye, ref UnifiedTracking.Data.Shapes, paramToMap, singleEyeMode);
4362
}
4463

45-
private void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes, bool isSingleEyeMode = false)
64+
private void UpdateVRCFTEyeData(ref UnifiedEyeData eyeData, ref UnifiedExpressionShape[] eyeShapes,
65+
string parameter, bool isSingleEyeMode = false)
4666
{
47-
HandleEyeGaze(ref eyeData, isSingleEyeMode);
48-
HandleEyeOpenness(ref eyeData, ref eyeShapes, isSingleEyeMode);
49-
EmulateEyebrows(ref eyeShapes, isSingleEyeMode);
67+
if (_gazeParameters.Contains(parameter))
68+
HandleEyeGaze(ref eyeData, isSingleEyeMode);
69+
70+
if (_opennessParameters.Contains(parameter))
71+
{
72+
HandleEyeOpenness(ref eyeData, ref eyeShapes, isSingleEyeMode);
73+
EmulateEyebrows(ref eyeShapes, isSingleEyeMode);
74+
}
5075
}
5176

5277
private void HandleEyeGaze(ref UnifiedEyeData eyeData, bool isSingleEyeMode)
@@ -68,43 +93,48 @@ private void HandleEyeOpenness(ref UnifiedEyeData eyeData, ref UnifiedExpression
6893
{
6994
if (isSingleEyeMode)
7095
{
71-
var eyeOpenness = _parameterValues["EyeLid"];
96+
var eyeOpenness = (float)_leftOneEuroFilter.Filter(_parameterValues["EyeLid"], 1);
7297

73-
HandleSingleEyeOpenness(ref eyeData.Left, eyeOpenness, _config.WidenThreshold, _config.SqueezeThreshold);
74-
HandleSingleEyeOpenness(ref eyeData.Right, eyeOpenness, _config.WidenThreshold, _config.SqueezeThreshold);
98+
HandleSingleEyeOpenness(ref eyeData.Left, eyeOpenness, _config);
99+
HandleSingleEyeOpenness(ref eyeData.Right, eyeOpenness, _config);
75100
return;
76101
}
77102

78-
HandleSingleEyeOpenness(ref eyeData.Left, _parameterValues["EyeLidLeft"], _config.WidenThreshold,
79-
_config.SqueezeThreshold);
80-
HandleSingleEyeOpenness(ref eyeData.Right, _parameterValues["EyeLidRight"], _config.WidenThreshold,
81-
_config.SqueezeThreshold);
103+
HandleSingleEyeOpenness(
104+
ref eyeData.Left,
105+
(float)_leftOneEuroFilter.Filter(_parameterValues["EyeLidLeft"], 1),
106+
_config);
107+
HandleSingleEyeOpenness(
108+
ref eyeData.Right,
109+
(float)_rightOneEuroFilter.Filter(_parameterValues["EyeLidRight"], 1),
110+
_config);
82111
}
83112

84113
private void HandleSingleEyeOpenness(
85114
ref UnifiedSingleEyeData eyeData,
86115
float baseOpenness,
87-
float widenThreshold,
88-
float squeezeThreshold
116+
Config config
89117
)
90118
{
91119
eyeData.Openness = baseOpenness;
92-
if (_config.ShouldEmulateEyeWiden && baseOpenness >= widenThreshold)
120+
if (_config.ShouldEmulateEyeWiden && baseOpenness >= config.WidenThresholdV2[0])
93121
{
94-
eyeData.Openness = Utils.SmoothStep(
95-
widenThreshold,
96-
1.4f,
122+
var opennessValue = Utils.SmoothStep(
123+
config.WidenThresholdV2[0],
124+
config.WidenThresholdV2[1],
97125
baseOpenness
98-
);
126+
) * config.OutputMultiplier;
127+
eyeData.Openness = baseOpenness + opennessValue;
99128
}
100129

101-
if (_config.ShouldEmulateEyeSquint && baseOpenness <= squeezeThreshold)
130+
if (_config.ShouldEmulateEyeSquint && baseOpenness <= config.SqueezeThresholdV2[0])
102131
{
103-
eyeData.Openness = Utils.SmoothStep(
104-
squeezeThreshold,
105-
-1.4f,
132+
var opennessValue = Utils.SmoothStep(
133+
config.SqueezeThresholdV2[0],
134+
config.SqueezeThresholdV2[1],
106135
baseOpenness
107-
);
136+
) * config.OutputMultiplier;
137+
eyeData.Openness = baseOpenness - opennessValue;
108138
}
109139
}
110140

@@ -118,6 +148,7 @@ private void EmulateEyebrows(ref UnifiedExpressionShape[] eyeShapes, bool isSing
118148
ref eyeShapes,
119149
UnifiedExpressions.BrowLowererRight,
120150
UnifiedExpressions.BrowOuterUpRight,
151+
ref _leftOneEuroFilter,
121152
eyeOpenness,
122153
_config.EyebrowThresholdRising,
123154
_config.EyebrowThresholdLowering
@@ -127,6 +158,7 @@ private void EmulateEyebrows(ref UnifiedExpressionShape[] eyeShapes, bool isSing
127158
ref eyeShapes,
128159
UnifiedExpressions.BrowLowererLeft,
129160
UnifiedExpressions.BrowOuterUpLeft,
161+
ref _rightOneEuroFilter,
130162
eyeOpenness,
131163
_config.EyebrowThresholdRising,
132164
_config.EyebrowThresholdLowering
@@ -142,6 +174,7 @@ private void EmulateEyebrows(ref UnifiedExpressionShape[] eyeShapes, bool isSing
142174
ref eyeShapes,
143175
UnifiedExpressions.BrowLowererRight,
144176
UnifiedExpressions.BrowOuterUpRight,
177+
ref _leftOneEuroFilter,
145178
baseRightEyeOpenness,
146179
_config.EyebrowThresholdRising,
147180
_config.EyebrowThresholdLowering
@@ -151,6 +184,7 @@ private void EmulateEyebrows(ref UnifiedExpressionShape[] eyeShapes, bool isSing
151184
ref eyeShapes,
152185
UnifiedExpressions.BrowLowererLeft,
153186
UnifiedExpressions.BrowOuterUpLeft,
187+
ref _rightOneEuroFilter,
154188
baseLeftEyeOpenness,
155189
_config.EyebrowThresholdRising,
156190
_config.EyebrowThresholdLowering

‎ExpressionsMapper.cs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,30 @@ namespace ETVRTrackingModule
55
{
66
public class ExpressionsMapper
77
{
8-
private IMappingStategy _mappingStrategy;
9-
private Config _config;
8+
private V1Mapper _v1Mapper;
9+
private V2Mapper _v2Mapper;
10+
1011
ILogger _logger;
11-
public ExpressionsMapper(ILogger logger, ref ETVRConfigManager configManager)
12+
public ExpressionsMapper(ILogger logger, ref ETVRConfigManager configManager)
1213
{
14+
var config = configManager.Config;
1315
_logger = logger;
14-
_config = configManager.Config;
15-
_mappingStrategy = new V2Mapper(_logger, ref _config);
16+
_v1Mapper = new V1Mapper(_logger, ref config);
17+
_v2Mapper = new V2Mapper(_logger, ref config);
1618
}
19+
1720
public void MapMessage(OSCMessage msg)
1821
{
1922
if (!msg.success)
2023
return;
21-
22-
var nextStrategy = IsV2Param(msg) ? (IMappingStategy) new V2Mapper(_logger, ref _config) : new V1Mapper(_logger, ref _config);
23-
24-
if (_mappingStrategy.GetType() != nextStrategy.GetType())
24+
25+
if (IsV2Param(msg))
2526
{
26-
_mappingStrategy = nextStrategy;
27+
_v2Mapper.handleOSCMessage(msg);
28+
return;
2729
}
28-
_mappingStrategy.handleOSCMessage(msg);
30+
31+
_v1Mapper.handleOSCMessage(msg);
2932
}
3033

3134
private bool IsV2Param(OSCMessage oscMessage)
@@ -34,4 +37,4 @@ private bool IsV2Param(OSCMessage oscMessage)
3437
return isv2Param;
3538
}
3639
}
37-
}
40+
}

0 commit comments

Comments
 (0)