Skip to content

Commit 5c9357a

Browse files
matasareimatasarei
committed
Face mesh working
1 parent a18c1cd commit 5c9357a

2 files changed

Lines changed: 83 additions & 58 deletions

File tree

‎index.html‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,9 @@
4848
</style>
4949
</head>
5050
<body>
51-
<!-- clmtrack -->
52-
<script src="https://cdnjs.cloudflare.com/ajax/libs/clmtrackr/1.1.2/clmtrackr.min.js"></script>
51+
<!-- MediaPipe Face Mesh and Camera Utils from CDN -->
52+
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/face_mesh.js"></script>
53+
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js"></script>
5354
<div id="tryon">
5455
<video id="camera" loop></video>
5556
<canvas id="overlay"></canvas>

‎js/tryon-face.js‎

Lines changed: 80 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,12 @@ export class TryOnFace {
4646
document.getElementById(this.selector).style.width = this.width + "px";
4747
this.video.setAttribute('width', this.width);
4848
this.video.setAttribute('height', this.height);
49-
this.tracker = new clm.tracker({useWebGL: true});
50-
this.tracker.init();
5149
this.stream = null;
5250
this.position = { x: 0, y: 0, z: 0 };
5351
this.rotation = { x: 0, y: 0 };
5452
this.size = { x: 1, y: 1, z: 1 };
53+
this.faceMesh = null;
54+
this.camera = null;
5555
this.init3D();
5656
}
5757

@@ -61,86 +61,110 @@ export class TryOnFace {
6161
}
6262

6363
start() {
64-
const video = this.video;
65-
const constraints = {
66-
video: {
67-
width: { ideal: this.width },
68-
height: { ideal: this.height }
69-
},
70-
audio: false
71-
};
72-
getCameraStreamProxy(constraints)
73-
.then((stream) => {
74-
this.stream = stream;
75-
attachStreamToVideoProxy(stream, video);
76-
video.play();
77-
this.changeStatus('STATUS_CAMERA_STARTED');
78-
this.tracker.start(video);
79-
this.loop();
80-
})
81-
.catch((err) => {
82-
this.changeStatus('STATUS_CAMERA_ERROR');
83-
console.error('Camera access error:', err && err.message ? err.message : err);
84-
});
64+
this.changeStatus('STATUS_SEARCH');
65+
this.initFaceMesh();
8566
}
8667

8768
stop() {
88-
try {
89-
this.tracker.stop();
90-
} catch (e) {}
91-
if (this.stream) {
92-
stopCameraStreamProxy(this.stream);
93-
this.stream = null;
69+
if (this.camera) {
70+
this.camera.stop();
71+
this.camera = null;
9472
}
9573
this.changeStatus('STATUS_READY');
9674
}
9775

98-
calculateDistanceScale(positions) {
99-
const L = CONFIG.LANDMARKS;
100-
const faceWidth = Math.abs(positions[L.RIGHT_EAR][0] - positions[L.LEFT_EAR][0]);
101-
return CONFIG.DETECTION.REFERENCE_FACE_WIDTH / faceWidth;
76+
initFaceMesh() {
77+
this.faceMesh = new window.FaceMesh({
78+
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`
79+
});
80+
this.faceMesh.setOptions({
81+
maxNumFaces: 1,
82+
refineLandmarks: true,
83+
minDetectionConfidence: 0.5,
84+
minTrackingConfidence: 0.5
85+
});
86+
this.faceMesh.onResults(this.onResults.bind(this));
87+
this.camera = new window.Camera(this.video, {
88+
onFrame: async () => {
89+
await this.faceMesh.send({image: this.video});
90+
},
91+
width: this.width,
92+
height: this.height
93+
});
94+
this.camera.start();
10295
}
10396

104-
calculateYawAngle(positions) {
105-
const L = CONFIG.LANDMARKS;
97+
onResults(results) {
98+
if (!results.multiFaceLandmarks || results.multiFaceLandmarks.length === 0) {
99+
this.changeStatus('STATUS_SEARCH');
100+
this.size.x = 0;
101+
this.size.y = 0;
102+
this.render();
103+
return;
104+
}
105+
this.changeStatus('STATUS_FOUND');
106+
const landmarks = results.multiFaceLandmarks[0];
107+
// Use MediaPipe landmark indices for left/right ear, eyes, nose, etc.
108+
// See: https://github.com/tensorflow/tfjs-models/blob/master/face-landmarks-detection/mesh_map.jpg
109+
// Example indices:
110+
// Left ear: 234, Right ear: 454, Left eye: 33, Right eye: 263, Nose tip: 1, Nose bridge: 168
111+
const L = {
112+
LEFT_EAR: 234,
113+
RIGHT_EAR: 454,
114+
LEFT_EYE: 33,
115+
RIGHT_EYE: 263,
116+
NOSE_TIP: 1,
117+
NOSE_BRIDGE: 168
118+
};
119+
function getXY(idx) {
120+
return [landmarks[idx].x * this.width, landmarks[idx].y * this.height];
121+
}
122+
const positions = {};
123+
Object.keys(L).forEach(key => {
124+
positions[L[key]] = getXY.call(this, L[key]);
125+
});
126+
// Calculate parameters using MediaPipe landmarks
127+
const faceWidth = Math.abs(positions[L.RIGHT_EAR][0] - positions[L.LEFT_EAR][0]);
128+
const distanceScale = CONFIG.DETECTION.REFERENCE_FACE_WIDTH / faceWidth;
106129
const faceCenterX = (positions[L.LEFT_EAR][0] + positions[L.RIGHT_EAR][0]) / 2;
107130
const noseX = positions[L.NOSE_TIP][0];
108131
const horizontalOffset = noseX - faceCenterX;
109-
const faceWidth = positions[L.RIGHT_EAR][0] - positions[L.LEFT_EAR][0];
110132
const normalizedOffset = horizontalOffset / (faceWidth / 2);
111133
const maxRotationRad = Math.PI / 4;
112-
return normalizedOffset * maxRotationRad;
113-
}
114-
115-
calculateRollAngle(positions) {
116-
const L = CONFIG.LANDMARKS;
134+
const yawAngleRad = normalizedOffset * maxRotationRad;
117135
const leftEyeX = positions[L.LEFT_EYE][0];
118136
const leftEyeY = positions[L.LEFT_EYE][1];
119137
const rightEyeX = positions[L.RIGHT_EYE][0];
120138
const rightEyeY = positions[L.RIGHT_EYE][1];
121139
const eyeDeltaX = rightEyeX - leftEyeX;
122140
const eyeDeltaY = rightEyeY - leftEyeY;
123-
return Math.atan2(-eyeDeltaY, eyeDeltaX);
124-
}
125-
126-
calculateGlassesCenter(positions) {
127-
const L = CONFIG.LANDMARKS;
141+
const rollAngleRad = Math.atan2(-eyeDeltaY, eyeDeltaX);
128142
const centerX = positions[L.NOSE_TIP][0];
129143
const weight = CONFIG.GLASSES.BRIDGE_WEIGHT;
130144
const centerY = positions[L.NOSE_BRIDGE][1] * weight + positions[L.NOSE_TIP][1] * (1 - weight);
131-
return { x: centerX, y: centerY };
132-
}
133-
134-
calculateGlassesWidth(positions) {
135-
const L = CONFIG.LANDMARKS;
136-
const faceWidth = positions[L.RIGHT_EAR][0] - positions[L.LEFT_EAR][0];
145+
const center = this.correct(centerX, centerY);
146+
const eyeDistance = rightEyeX - leftEyeX;
137147
const widthByFace = faceWidth * CONFIG.GLASSES.WIDTH_TO_FACE_RATIO;
138-
const eyeDistance = positions[L.RIGHT_EYE][0] - positions[L.LEFT_EYE][0];
139148
const widthByEyes = eyeDistance * CONFIG.GLASSES.WIDTH_TO_EYE_RATIO;
140-
if (!isFinite(eyeDistance) || eyeDistance < 8) {
141-
return widthByFace;
149+
const glassesWidth = (!isFinite(eyeDistance) || eyeDistance < 8)
150+
? widthByFace
151+
: widthByEyes * 0.65 + widthByFace * 0.35;
152+
let frontWidth = 100, frontHeight = 50;
153+
if (this.textures && this.textures['front'] && this.textures['front'].image) {
154+
frontWidth = this.textures['front'].image.width;
155+
frontHeight = this.textures['front'].image.height;
142156
}
143-
return widthByEyes * 0.65 + widthByFace * 0.35;
157+
this.position.x = center.x;
158+
this.position.y = center.y;
159+
this.rotation.y = yawAngleRad * CONFIG.GLASSES.ROTATION_DAMPENING;
160+
this.rotation.z = rollAngleRad * CONFIG.GLASSES.ROTATION_DAMPENING;
161+
this.size.x = glassesWidth;
162+
this.size.y = (this.size.x / frontWidth) * frontHeight;
163+
this.size.z = this.size.x * CONFIG.GLASSES.DEPTH_TO_WIDTH_RATIO;
164+
const absYaw = Math.min(Math.abs(yawAngleRad), maxRotationRad) / maxRotationRad;
165+
const depthDampen = 1 - (absYaw * 0.6);
166+
this.position.z = - (this.size.z / 2) * depthDampen;
167+
this.render();
144168
}
145169

146170
loop() {

0 commit comments

Comments
 (0)