vnc: cycle codec to H.264 when VNC is enabled mid-session

If a browser tab negotiated H.265 before VNC was toggled on, the
capture pipeline keeps emitting H.265 NALs. resolveCodec only
forces H.264 for *new* WebRTC sessions; the in-flight one is not
disturbed. So when a TigerVNC client subsequently connected, it
received H.265 NALs marked as OpenH264 (encoding 50), which the
spec defines as H.264-only — the client's decoder failed silently
and the user kept seeing the (already-rendered) placeholder.

After enabling VNC via JSON-RPC, check the running codec; if it is
not H.264, cycle the pipeline (VideoStop → SetCodecType(0) →
VideoStart). The active WebRTC session will see frames briefly
disappear and then need to refresh — the warning log says so.

Also adds trace-level diagnostics so the user can see what is
happening with `JETKVM_LOG_TRACE=vnc`:

  - codec value at the moment a VNC client connects
  - dropped non-IDR frames during the keyframe wait
  - rect emissions (OpenH264 size/flags or Raw placeholder)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: mcuelenaere

vnc.go

@@ -74,10 +74,42 @@ func rpcSetVNCConfig(cfg VNCConfig) error {
 			vncLogger.Error().Err(err).Msg("failed to start VNC server after config change")
 			return err
 		}
+		// Force the encoder to H.264 if a stream is already running
+		// at H.265. Without this, a browser tab that negotiated H.265
+		// before VNC was enabled would keep the pipeline at H.265,
+		// and any VNC client would receive H.265 NALs marked as
+		// OpenH264 (encoding 50, H.264-only) and silently fail to
+		// decode. Cycling the stream briefly interrupts the active
+		// WebRTC session — the user will need to refresh the browser.
+		forceVideoCodecForVNC()
 	}
 	return nil
 }
 
+// forceVideoCodecForVNC makes sure the running capture pipeline emits
+// H.264 once VNC is enabled. If no consumers are running, the codec
+// gets pinned but the pipeline stays stopped — the next acquirer's
+// VideoStart will pick it up.
+func forceVideoCodecForVNC() {
+	curr, err := nativeInstance.VideoGetCodecType()
+	if err != nil {
+		vncLogger.Warn().Err(err).Msg("could not read current video codec; skipping coercion")
+		return
+	}
+	if curr == 0 {
+		return // already H.264
+	}
+	if !videoStreamHasConsumers() {
+		// Pipeline is idle. Just pin the codec; next start picks it up.
+		_ = nativeInstance.VideoSetCodecType(0)
+		return
+	}
+	vncLogger.Warn().Int("from", curr).Msg("cycling capture pipeline to H.264 for VNC; refresh active browser tabs")
+	_ = nativeInstance.VideoStop()
+	_ = nativeInstance.VideoSetCodecType(0)
+	_ = nativeInstance.VideoStart()
+}
+
 // StartVNCServer starts the VNC TCP listener if VNC is enabled.
 // Idempotent: a second call while running is a no-op.
 func StartVNCServer() error {
@@ -222,6 +254,9 @@ func (s *VNCServer) addClient(c *vncConn) (sps, pps []byte) {
 	if first {
 		acquireVideoStreamWithCodec(vncVideoConsumer, 0) // 0 = H.264
 	}
+	if curr, err := nativeInstance.VideoGetCodecType(); err == nil {
+		c.l.Trace().Int("codec", curr).Msg("video codec at VNC client connect (0=H.264, 1=H.265)")
+	}
 	s.paramsMu.Lock()
 	defer s.paramsMu.Unlock()
 	return append([]byte(nil), s.sps...), append([]byte(nil), s.pps...)

vnc_conn.go

@@ -175,6 +175,7 @@ func (c *vncConn) dispatchLoop() {
 			}
 			c.stateMu.Unlock()
 			if drop {
+				c.l.Trace().Int("size", len(pkt.data)).Msg("dropping non-IDR frame while waiting for keyframe")
 				select {
 				case c.updateNeeded <- struct{}{}:
 				default:
@@ -274,6 +275,7 @@ func (c *vncConn) writeUpdate(pkt vncFramePacket) error {
 			payload = pkt.data
 		}
 
+		c.l.Trace().Uint16("w", w).Uint16("h", h).Int("size", len(payload)).Uint32("flags", flags).Bool("primed", c.primed).Msg("emitting OpenH264 rect")
 		if err := c.conn.WriteOpenH264Rect(
 			rfb.Rect{X: 0, Y: 0, W: w, H: h, Encoding: rfb.EncodingOpenH264},
 			flags, payload,
@@ -282,6 +284,7 @@ func (c *vncConn) writeUpdate(pkt vncFramePacket) error {
 		}
 	} else if !hasH264 {
 		pixels := rfb.PlaceholderImage(int(w), int(h))
+		c.l.Trace().Uint16("w", w).Uint16("h", h).Msg("emitting Raw placeholder rect (client did not advertise OpenH264)")
 		if err := c.conn.WriteRawRect(
 			rfb.Rect{X: 0, Y: 0, W: w, H: h, Encoding: rfb.EncodingRaw},
 			pixels,