Merge pull request #945 from revisit-studies/hotfix-screen-recording-replay-and-sync

Fix screen recording replay and sync with provenance timeline

Author: JackWilb

src/analysis/individualStudy/thinkAloud/ThinkAloudFooter.tsx

@@ -62,9 +62,9 @@ async function getTags(storageEngine: StorageEngine | undefined, type: 'particip
 }
 
 export function ThinkAloudFooter({
-  visibleParticipants, rawTranscript, currentShownTranscription, width, onTimeUpdate, isReplay, editedTranscript, currentTrial, saveProvenance, jumpedToLine = 0, studyId, setHasAudio, storageEngine,
+  visibleParticipants, rawTranscript, currentShownTranscription, width, onTimeUpdate, isReplay, editedTranscript, currentTrial, saveProvenance, jumpedToLine = 0, studyId, setHasAudio, storageEngine, forceMute, onAnalysisIsPlayingChange, onProvenanceTimelineChange,
 } : {
-  visibleParticipants: string[], rawTranscript: TranscribedAudio | null, currentShownTranscription: number | null, width: number, onTimeUpdate: (n: number) => void, isReplay: boolean, editedTranscript?: EditedText[], currentTrial: string, saveProvenance: (prov: unknown) => void, jumpedToLine?: number, studyId: string, setHasAudio: (b: boolean) => void, storageEngine: StorageEngine | undefined,
+  visibleParticipants: string[], rawTranscript: TranscribedAudio | null, currentShownTranscription: number | null, width: number, onTimeUpdate: (n: number) => void, isReplay: boolean, editedTranscript?: EditedText[], currentTrial: string, saveProvenance: (prov: unknown) => void, jumpedToLine?: number, studyId: string, setHasAudio: (b: boolean) => void, storageEngine: StorageEngine | undefined, forceMute?: boolean, onAnalysisIsPlayingChange?: (b: boolean) => void, onProvenanceTimelineChange?: (n: number)=> void,
 }) {
   const auth = useAuth();
 
@@ -80,7 +80,9 @@ export function ThinkAloudFooter({
 
   const { value: allParticipantTags, execute: pullAllParticipantTags } = useAsync(getTags, [storageEngine, 'participant']);
   const [analysisIsPlaying, _setAnalysisIsPlaying] = useState(false);
-  const [isMuted, setIsMuted] = useState(true);
+
+  const [_isMuted, setIsMuted] = useState(true);
+  const isMuted = forceMute ? true : _isMuted;
 
   const [audioUrl, setAudioUrl] = useState<string | null>(null);
   const [screenRecordingUrl, setScreenRecordingUrl] = useState<string | null>(null);
@@ -142,7 +144,9 @@ export function ThinkAloudFooter({
       setIsMuted(true);
     }
     _setAnalysisIsPlaying(playing);
-  }, []);
+
+    onAnalysisIsPlayingChange && onAnalysisIsPlayingChange(playing);
+  }, [onAnalysisIsPlayingChange]);
 
   const [transcriptLines, setTranscriptLines] = useState<TranscriptLinesWithTimes[] | null>(null);
 
@@ -278,15 +282,15 @@ export function ThinkAloudFooter({
     <AppShell.Footer zIndex={101} withBorder={false}>
       <Stack style={{ backgroundColor: 'var(--mantine-color-blue-1)', height: '100%' }} gap={5} justify="center">
 
-        <AudioProvenanceVis isMuted={isMuted} setHasAudio={setHasAudio} jumpedToAudioTime={jumpedToTime} speed={speed} setSpeed={setSpeed} saveProvenance={saveProvenance} analysisIsPlaying={analysisIsPlaying} setAnalysisIsPlaying={setAnalysisIsPlaying} setTime={onTimeUpdate} setTimeString={(_t) => setTimeString(_t)} answers={participant ? participant.answers : {}} taskName={currentTrial} context={isReplay ? 'provenanceVis' : 'audioAnalysis'} />
+        <AudioProvenanceVis isMuted={isMuted} setHasAudio={setHasAudio} jumpedToAudioTime={jumpedToTime} speed={speed} setSpeed={setSpeed} saveProvenance={saveProvenance} analysisIsPlaying={analysisIsPlaying} setAnalysisIsPlaying={setAnalysisIsPlaying} setTime={onTimeUpdate} setTimeString={(_t) => setTimeString(_t)} answers={participant ? participant.answers : {}} taskName={currentTrial} context={isReplay ? 'provenanceVis' : 'audioAnalysis'} onProvenanceTimelineChange={onProvenanceTimelineChange} />
         {xScale && transcriptLines ? <TranscriptSegmentsVis startTime={xScale.domain()[0]} xScale={xScale} transcriptLines={transcriptLines} currentShownTranscription={currentShownTranscription || 0} /> : null }
 
         <Group gap="xs" style={{ width: '100%' }} justify="center" wrap="nowrap">
           <Group wrap="nowrap">
             <Text ff="monospace" style={{ textAlign: 'right' }} mt="lg" c="dimmed">{timeString}</Text>
 
             <Tooltip label="Play">
-              <ActionIcon mt={25} size="xl" variant="light" onClick={() => { setAnalysisIsPlaying(!analysisIsPlaying); setIsMuted(!isMuted); }}>
+              <ActionIcon mt={25} size="xl" variant="light" onClick={() => { setAnalysisIsPlaying(!analysisIsPlaying); setIsMuted(!_isMuted); }}>
                 {analysisIsPlaying ? <IconPlayerPauseFilled /> : <IconPlayerPlayFilled /> }
               </ActionIcon>
             </Tooltip>

src/components/audioAnalysis/AudioProvenanceVis.tsx

@@ -31,8 +31,8 @@ function safe<T>(p: Promise<T>): Promise<T | null> {
 }
 
 export function AudioProvenanceVis({
-  setTimeString, answers, setTime, taskName, context, saveProvenance, analysisIsPlaying, setAnalysisIsPlaying, speed, jumpedToAudioTime, setHasAudio, setSpeed, isMuted,
-}: { setTimeString: (time: string) => void; answers: Record<string, StoredAnswer>, setTime: (time: number) => void, taskName: string, context: 'audioAnalysis' | 'provenanceVis', saveProvenance: ((state: unknown) => void), analysisIsPlaying: boolean, setAnalysisIsPlaying: (b: boolean) => void, speed: number, jumpedToAudioTime: number, setHasAudio: (b: boolean) => void, setSpeed: (n: number) => void, isMuted: boolean }) {
+  setTimeString, answers, setTime, taskName, context, saveProvenance, analysisIsPlaying, setAnalysisIsPlaying, speed, jumpedToAudioTime, setHasAudio, setSpeed, isMuted, onProvenanceTimelineChange,
+}: { setTimeString: (time: string) => void; answers: Record<string, StoredAnswer>, setTime: (time: number) => void, taskName: string, context: 'audioAnalysis' | 'provenanceVis', saveProvenance: ((state: unknown) => void), analysisIsPlaying: boolean, setAnalysisIsPlaying: (b: boolean) => void, speed: number, jumpedToAudioTime: number, setHasAudio: (b: boolean) => void, setSpeed: (n: number) => void, isMuted: boolean, onProvenanceTimelineChange?: (n: number) => void }) {
   const [searchParams, setSearchParams] = useSearchParams();
   const participantId = useMemo(() => searchParams.get('participantId') || '', [searchParams]);
 
@@ -139,6 +139,7 @@ export function AudioProvenanceVis({
     }
 
     setPlayTime(n);
+    onProvenanceTimelineChange && onProvenanceTimelineChange(n - startTime);
 
     if (wavesurfer.current && percent !== undefined && !Number.isNaN(percent)) {
       setTimeout(() => {
@@ -149,7 +150,7 @@ export function AudioProvenanceVis({
     if (setTime) {
       setTime(n);
     }
-  }, [setAnalysisIsPlaying, setTime, startTime, totalAudioLength]);
+  }, [setAnalysisIsPlaying, setTime, startTime, totalAudioLength, onProvenanceTimelineChange]);
 
   useEffect(() => {
     _setPlayTime(startTime + jumpedToAudioTime * 1000 + 1, undefined);

src/components/interface/AnalysisFooter.tsx

@@ -1,13 +1,14 @@
-import { useDispatch } from 'react-redux';
 import { useParams } from 'react-router';
 
+import { useCallback } from 'react';
 import { useAsync } from '../../store/hooks/useAsync';
 
 import { useStorageEngine } from '../../storage/storageEngineHooks';
 import { StorageEngine } from '../../storage/engines/types';
 import { ThinkAloudFooter } from '../../analysis/individualStudy/thinkAloud/ThinkAloudFooter';
 import { useCurrentIdentifier } from '../../routes/utils';
-import { useStoreActions } from '../../store/store';
+import { useStoreActions, useStoreDispatch } from '../../store/store';
+import { useRecordingConfig } from '../../store/hooks/useRecordingConfig';
 
 function getAllParticipantsNames(storageEngine: StorageEngine | undefined) {
   if (storageEngine) {
@@ -23,16 +24,30 @@ export function AnalysisFooter({ setHasAudio }: {setHasAudio: (b: boolean) => vo
 
   const identifier = useCurrentIdentifier();
 
-  const dispatch = useDispatch();
+  const storeDispatch = useStoreDispatch();
+
+  const { currentComponentHasScreenRecording } = useRecordingConfig();
 
   const { studyId } = useParams();
 
   const {
     saveAnalysisState,
   } = useStoreActions();
 
+  const { setAnalysisIsPlaying, setProvenanceJumpTime } = useStoreActions();
+
+  const handleProvenanceTimelineChange = useCallback((n: number) => {
+    storeDispatch(setProvenanceJumpTime(n));
+  }, [storeDispatch, setProvenanceJumpTime]);
+
+  const handleAnalysisIsPlayingChange = useCallback((isPlaying: boolean) => {
+    storeDispatch(setAnalysisIsPlaying(isPlaying));
+  }, [storeDispatch, setAnalysisIsPlaying]);
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const saveProvenance = useCallback((prov: any) => storeDispatch(saveAnalysisState(prov)), [storeDispatch, saveAnalysisState]);
+
   return (
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    <ThinkAloudFooter storageEngine={storageEngine} setHasAudio={setHasAudio} studyId={studyId || ''} currentTrial={identifier} isReplay visibleParticipants={allParticipants || []} rawTranscript={null} currentShownTranscription={null} width={3000} onTimeUpdate={() => {}} saveProvenance={(prov: any) => dispatch(saveAnalysisState(prov))} />
+    <ThinkAloudFooter storageEngine={storageEngine} setHasAudio={setHasAudio} studyId={studyId || ''} currentTrial={identifier} isReplay visibleParticipants={allParticipants || []} rawTranscript={null} currentShownTranscription={null} width={3000} onTimeUpdate={() => {}} saveProvenance={saveProvenance} onProvenanceTimelineChange={handleProvenanceTimelineChange} onAnalysisIsPlayingChange={handleAnalysisIsPlayingChange} forceMute={currentComponentHasScreenRecording} />
   );
 }