// Record screen — real getUserMedia / getDisplayMedia + MediaRecorder pipeline.
// Composition modes (cameraOnly, cameraSlide, slideCamera) drive the live overlay,
// but for v1 we record the raw camera/screen track. Overlay burn-in lands in step #4.
function ScreenRecord({ state, set, nav }) {
const [recState, setRecState] = React.useState('idle'); // idle | recording | paused
const [time, setTime] = React.useState(0);
const [slideIdx, setSlideIdx] = React.useState(0);
const [showGuides, setShowGuides] = React.useState(false);
const [frontCam, setFrontCam] = React.useState(true);
const [source, setSource] = React.useState('camera'); // camera | screen
const [streamReady, setStreamReady] = React.useState(false);
const [permError, setPermError] = React.useState(null);
const [speechFollow, setSpeechFollow] = React.useState(false);
const [burnIn, setBurnIn] = React.useState(true);
const videoRef = React.useRef(null);
const streamRef = React.useRef(null);
const recorderRef = React.useRef(null);
const chunksRef = React.useRef([]);
const recordStartRef = React.useRef(0);
const transcriptLogRef = React.useRef([]); // [{t, text, progress, slide}]
const compositorRef = React.useRef(null); // { stream, stop }
// The compositor draws from this ref each frame so we don't recreate it on
// every state update — just mutate fields and the rAF loop picks them up.
const drawStateRef = React.useRef({
composition: 'cameraOnly',
aspect: '9:16',
mirror: true,
slideText: '',
slideLang: 'auto',
progress: 0,
burnInCaption: true,
});
const slides = state.slides;
const slide = slides[slideIdx];
const fmt = (s) => `${String(Math.floor(s/60)).padStart(2,'0')}:${String(s%60).padStart(2,'0')}`;
// Keep the compositor draw-state in sync each render. The compositor's
// rAF loop reads the ref every frame.
React.useEffect(() => {
const s = drawStateRef.current;
s.composition = state.composition;
s.aspect = state.aspect || '9:16';
s.mirror = source === 'camera' && frontCam;
s.slideText = slide?.text || '';
s.slideLang = slide?.lang || 'auto';
s.burnInCaption = burnIn;
});
// Speech-driven teleprompter alignment
const advanceSlide = React.useCallback(() => {
setSlideIdx(i => Math.min(slides.length - 1, i + 1));
}, [slides.length]);
const speech = useSpeechFollow({
enabled: speechFollow && recState === 'recording',
scriptText: slide?.text || '',
lang: slide?.lang || 'auto',
onAdvance: advanceSlide,
onResult: (t) => {
// Log each interim/final transcript event with a timestamp relative to recording start
const rel = recordStartRef.current ? (Date.now() - recordStartRef.current) : 0;
transcriptLogRef.current.push({ t: rel, text: t.text, progress: t.progress, slide: slideIdx });
// Mirror progress into the draw state so the burn-in highlight tracks speech.
drawStateRef.current.progress = t.progress;
},
});
// Acquire / re-acquire media stream when source or facing changes.
React.useEffect(() => {
let cancelled = false;
async function acquire() {
// Don't tear down a stream while we're actively recording.
if (recorderRef.current && recorderRef.current.state !== 'inactive') return;
stopStream();
setStreamReady(false);
setPermError(null);
try {
const stream = await getMediaStream(source, frontCam);
if (cancelled) {
stream.getTracks().forEach(t => t.stop());
return;
}
streamRef.current = stream;
if (videoRef.current) {
videoRef.current.srcObject = stream;
// Best-effort autoplay; some browsers gate without a gesture.
videoRef.current.play().catch(() => {});
}
// If user ends screen-share via the browser UI, fall back to camera.
stream.getVideoTracks().forEach(t => {
t.onended = () => {
if (source === 'screen') setSource('camera');
};
});
setStreamReady(true);
} catch (err) {
if (!cancelled) setPermError(humanizeMediaError(err, source));
}
}
acquire();
return () => { cancelled = true; };
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [source, frontCam]);
// Wall-clock timer while recording.
React.useEffect(() => {
if (recState !== 'recording') return;
const t = setInterval(() => setTime(s => s + 1), 1000);
return () => clearInterval(t);
}, [recState]);
// Tear everything down on unmount.
React.useEffect(() => () => {
try {
const r = recorderRef.current;
if (r && r.state !== 'inactive') r.stop();
} catch {}
stopStream();
}, []);
function stopStream() {
const s = streamRef.current;
if (s) {
s.getTracks().forEach(t => { try { t.stop(); } catch {} });
streamRef.current = null;
}
if (videoRef.current) videoRef.current.srcObject = null;
}
function startRecording() {
if (!streamRef.current) {
setPermError('No video source. Click "Allow" when prompted.');
return;
}
chunksRef.current = [];
const mimeType = pickRecorderMime();
// Decide the recording stream: composited canvas (with caption burn-in)
// when burnIn is on, else the raw source stream.
let recordedStream = streamRef.current;
if (burnIn) {
try {
compositorRef.current = startCompositor({
sourceVideo: videoRef.current,
sourceStream: streamRef.current,
drawStateRef,
});
recordedStream = compositorRef.current.stream;
} catch (e) {
// Fall back to raw stream if compositor setup fails.
console.warn('[compositor] failed, recording raw stream:', e);
compositorRef.current = null;
}
}
let recorder;
try {
recorder = new MediaRecorder(
recordedStream,
mimeType ? { mimeType, videoBitsPerSecond: 5_000_000 } : undefined
);
} catch (e) {
setPermError('Recording not supported in this browser. ' + (e?.message || ''));
try { compositorRef.current?.stop(); } catch {}
compositorRef.current = null;
return;
}
recorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) chunksRef.current.push(e.data);
};
recorder.onstop = () => {
const type = recorder.mimeType || mimeType || 'video/webm';
const blob = new Blob(chunksRef.current, { type });
chunksRef.current = [];
const url = URL.createObjectURL(blob);
// Tear down the compositor (if any) before navigating.
try { compositorRef.current?.stop(); } catch {}
compositorRef.current = null;
// Persist on app state so the edit/share screens can read it.
if (state.recording?.url) {
try { URL.revokeObjectURL(state.recording.url); } catch {}
}
set({
recording: {
url,
mimeType: type,
durationSec: time,
size: blob.size,
source,
createdAt: Date.now(),
composited: burnIn,
// Captures the smart-follow log for caption export later
transcript: transcriptLogRef.current.slice(),
slideTexts: slides.map(s => ({ text: s.text, lang: s.lang })),
},
});
transcriptLogRef.current = [];
setTime(0);
setRecState('idle');
setTimeout(() => nav.go('edit'), 250);
};
recorder.start(1000); // 1s timeslices keeps the array small for long takes
recorderRef.current = recorder;
recordStartRef.current = Date.now();
transcriptLogRef.current = [];
setRecState('recording');
setTime(0);
}
function stopRecording() {
const r = recorderRef.current;
if (!r) { setRecState('idle'); return; }
try { if (r.state !== 'inactive') r.stop(); } catch {}
}
const startStop = () => {
if (recState === 'idle') startRecording();
else stopRecording();
};
const pauseResume = () => {
const r = recorderRef.current;
if (!r) return;
if (r.state === 'recording') { r.pause(); setRecState('paused'); }
else if (r.state === 'paused') { r.resume(); setRecState('recording'); }
};
const nextSlide = () => setSlideIdx(Math.min(slides.length-1, slideIdx+1));
const prevSlide = () => setSlideIdx(Math.max(0, slideIdx-1));
const cycleSource = () => {
if (recState !== 'idle') return; // can't switch source mid-recording
setSource(source === 'camera' ? 'screen' : 'camera');
};
return (
{/* Live camera/screen video preview */}
{/* Grid guides */}
{showGuides && (
{[1,2].map(i =>
)}
{[1,2].map(i =>
)}
)}
{/* Composition — slide content overlays */}
{state.composition === 'cameraSlide' && (
)}
{state.composition === 'slideCamera' && (
<>
{/* Solid slide background */}
{/* PiP camera */}
)}
{/* TOP BAR */}
nav.go('prepare')} style={{
width:38, height:38, borderRadius:999,
background:'rgba(0,0,0,0.45)', backdropFilter:'blur(10px)',
color:'#fff', display:'inline-flex', alignItems:'center', justifyContent:'center',
border:'1px solid rgba(255,255,255,0.12)'
}}>
{/* Recording state pill */}
{recState === 'recording' && (
)}
{recState === 'paused' && (
)}
{recState === 'idle' && (
)}
{fmt(time)}
{recState === 'recording' &&
}
{/* Right cluster */}
{
if (!speech.supported && speech.provider !== 'pending') {
setPermError('Smart follow needs Chrome, Edge, or Safari (SpeechRecognition API).');
return;
}
setSpeechFollow(v => !v);
}}
active={speechFollow}
title="Smart follow (auto-scroll teleprompter)"
>
recState === 'idle' && setBurnIn(v => !v)}
active={burnIn}
title={burnIn ? 'Burn captions in (compositor on)' : 'Raw stream (no overlays in export)'}
>
{source === 'screen' ? : }
setShowGuides(!showGuides)} active={showGuides}>
{source === 'camera' && (
setFrontCam(!frontCam)}>
)}
{/* Source label / error */}
{(permError || !streamReady) && (
{permError || (source === 'screen' ? 'Pick a screen / window to share…' : 'Requesting camera & mic…')}
)}
{/* TELEPROMPTER (only in cameraOnly mode) */}
{state.composition === 'cameraOnly' && (
SLIDE {String(slideIdx+1).padStart(2,'0')} / {String(slides.length).padStart(2,'0')}
{speechFollow ? (
{speech.error ? 'SR ERR' : (speech.listening ? 'LISTENING' : 'SMART')}
) : slide.lang === 'ja' ? (
FURIGANA
) : null}
{speechFollow && (
)}
)}
{/* Slide dots */}
{slides.map((_,i) => (
))}
{/* BOTTOM CONTROL DECK */}
{/* Prev slide */}
{/* Record button */}
{/* Pause / Next-slide */}
{recState === 'idle' ? (
) : (
{recState === 'recording' ?
)}
{/* Hint */}
{recState === 'idle' && (streamReady ? 'Tap to record · Switch source with the camera/screen icon' : 'Waiting for camera permission…')}
{recState === 'recording' && Recording · Tap ● to stop }
{recState === 'paused' && 'Paused · Tap play to resume'}
{children}
);
}
function SlideBodyText({ slide, large, progress = 0 }) {
const sz = large ? 30 : 20;
const ruby = slide.lang === 'ja' || (slide.lang === 'auto' && /[一-鿿-ゟ゠-ヿ]/.test(slide.text));
if (ruby && slide.text) {
// For ruby (furigana) text we render the mock pairs and dim the unspoken
// tail with a clip-path-based opacity gradient driven by `progress`.
const cutoff = Math.max(0, Math.min(100, progress * 100));
return
0
? `linear-gradient(to right, rgba(0,0,0,1) ${cutoff}%, rgba(0,0,0,0.5) ${cutoff+10}%, rgba(0,0,0,0.5) 100%)`
: 'none',
maskImage: progress > 0
? `linear-gradient(to right, rgba(0,0,0,1) ${cutoff}%, rgba(0,0,0,0.5) ${cutoff+10}%, rgba(0,0,0,0.5) 100%)`
: 'none',
}}>
;
}
// Token-level highlight for plain text. Faded for "not yet spoken", full opacity for spoken.
const tokens = (slide.text || 'Your script appears here.').split(/(\s+)/);
const wordCount = tokens.filter(t => /\S/.test(t)).length || 1;
const spokenWords = Math.round(progress * wordCount);
let wi = 0;
return
{tokens.map((tok, i) => {
if (!/\S/.test(tok)) return {tok} ;
const isSpoken = wi < spokenWords;
wi++;
return 0 ? (isSpoken ? 1 : 0.45) : 1,
color: progress > 0 && isSpoken ? 'var(--accent)' : '#fff',
transition: 'opacity .2s ease, color .2s ease',
}}>{tok} ;
})}
;
}
// Live preview — renders a real from the active stream. Falls back to a
// muted gradient + silhouette while permission is pending or denied.
function LivePreview({ videoRef, ready, source, frontCam, small, error, cloneFromRef }) {
// PiP variant: re-use the already-attached stream by cloning the srcObject
// into a second element on mount.
const localRef = React.useRef(null);
const ref = videoRef || localRef;
React.useEffect(() => {
if (!cloneFromRef || !cloneFromRef.current) return;
const sync = () => {
if (localRef.current && cloneFromRef.current?.srcObject) {
localRef.current.srcObject = cloneFromRef.current.srcObject;
localRef.current.play().catch(() => {});
}
};
sync();
const id = setInterval(sync, 500); // cheap re-sync if the parent stream gets replaced
return () => clearInterval(id);
}, [cloneFromRef]);
const fallbackBg = frontCam
? 'radial-gradient(circle at 45% 40%, #3a4a4e 0%, #1a2024 55%, #0a0d0f 100%)'
: 'radial-gradient(circle at 55% 60%, #4a3a3a 0%, #1e1a18 55%, #0c0a09 100%)';
// Mirror only the front-facing camera, never screen capture.
const mirror = source === 'camera' && frontCam;
return (
{!ready && !error && !small && (
<>
{source === 'screen' ? 'SCREEN · WAITING' : 'CAM · ' + (frontCam?'FRONT':'BACK')}
)}
);
}
// ─────────────────────────────────────────────────────────────
// Capture helpers
// ─────────────────────────────────────────────────────────────
async function getMediaStream(source, frontCam) {
if (source === 'screen') {
if (!navigator.mediaDevices?.getDisplayMedia) {
throw new Error('Screen capture is not supported in this browser.');
}
// Capture the screen with system audio if the OS+browser allow it,
// and combine a separate mic track so voice-overs work everywhere.
const display = await navigator.mediaDevices.getDisplayMedia({
video: { frameRate: 30 },
audio: true,
});
try {
const mic = await navigator.mediaDevices.getUserMedia({ audio: true });
mic.getAudioTracks().forEach(t => display.addTrack(t));
} catch {
// No mic permission — keep going with screen audio (or silent).
}
return display;
}
if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('Camera capture is not supported in this browser.');
}
return navigator.mediaDevices.getUserMedia({
video: {
facingMode: frontCam ? 'user' : { ideal: 'environment' },
width: { ideal: 1280 },
height: { ideal: 720 },
},
audio: true,
});
}
// ─────────────────────────────────────────────────────────────
// Canvas compositor — draws the live video + overlays into an offscreen
// canvas and exposes its captureStream(). The recorded video has the
// teleprompter/captions baked in, so what the viewer sees matches what
// the teacher saw at record time.
// ─────────────────────────────────────────────────────────────
function startCompositor({ sourceVideo, sourceStream, drawStateRef }) {
const aspect = drawStateRef.current.aspect || '9:16';
const { w, h } = aspectToSize(aspect);
const canvas = document.createElement('canvas');
canvas.width = w;
canvas.height = h;
const ctx = canvas.getContext('2d', { alpha: false });
let raf = 0;
let stopped = false;
function tick() {
if (stopped) return;
raf = requestAnimationFrame(tick);
drawFrame(ctx, sourceVideo, drawStateRef.current, w, h);
}
tick();
// Some browsers don't capture from canvas until a frame's been drawn,
// so fire one synchronous draw before captureStream.
drawFrame(ctx, sourceVideo, drawStateRef.current, w, h);
const videoStream = canvas.captureStream(30);
const composited = new MediaStream();
videoStream.getVideoTracks().forEach(t => composited.addTrack(t));
// Audio stays from the source.
sourceStream.getAudioTracks().forEach(t => composited.addTrack(t));
return {
stream: composited,
canvas,
stop() {
stopped = true;
cancelAnimationFrame(raf);
videoStream.getTracks().forEach(t => { try { t.stop(); } catch {} });
},
};
}
function aspectToSize(aspect) {
// 1080p target frames; aspect picks portrait/landscape/square.
switch (aspect) {
case '16:9': return { w: 1920, h: 1080 };
case '1:1': return { w: 1080, h: 1080 };
case '9:16':
default: return { w: 1080, h: 1920 };
}
}
function drawFrame(ctx, video, ds, W, H) {
// Background — black so letterboxing is clean.
ctx.fillStyle = '#000';
ctx.fillRect(0, 0, W, H);
// Draw video with cover-fit + optional horizontal mirror.
// Beauty filter (skin smoothness, brightness, warmth, glow) is applied
// via ctx.filter so the recorded video matches what the user sees in
// the live preview. Falls back gracefully on browsers that don't support
// ctx.filter — the live still has the CSS filter applied.
//
// Face slim: when ds.slim > 0 AND we have a recent face bbox from
// MediaPipe (ds.face), delegate to drawFaceSlim which handles the
// full pipeline including the cover-fit draw + face-region warp.
if (video instanceof HTMLVideoElement && video.readyState >= 2) {
if (ds.slim > 0 && ds.face && typeof window.drawFaceSlim === 'function') {
window.drawFaceSlim(ctx, video, ds.face, W, H, ds.slim, ds.mirror, ds.filter);
} else {
const vw = video.videoWidth || 1280;
const vh = video.videoHeight || 720;
const scale = Math.max(W / vw, H / vh);
const dw = vw * scale, dh = vh * scale;
const dx = (W - dw) / 2, dy = (H - dh) / 2;
ctx.save();
if (ds.filter && ds.filter !== 'none') {
ctx.filter = ds.filter;
}
if (ds.mirror) {
ctx.translate(W, 0);
ctx.scale(-1, 1);
ctx.drawImage(video, W - dx - dw, dy, dw, dh);
} else {
ctx.drawImage(video, dx, dy, dw, dh);
}
ctx.restore();
}
}
// Caption / teleprompter burn-in.
if (ds.burnInCaption && ds.slideText) {
drawCaptionStrip(ctx, ds, W, H);
}
}
// drawCaptionStrip — bottom 24% gradient + word-level highlight matching
// the smart-follow progress. Mirrors the on-screen teleprompter card so
// what's exported looks like what the user saw.
function drawCaptionStrip(ctx, ds, W, H) {
const text = ds.slideText;
if (!text) return;
const stripH = Math.round(H * 0.22);
const stripY = H - stripH;
// Smoke-glass background.
const grad = ctx.createLinearGradient(0, stripY, 0, H);
grad.addColorStop(0, 'rgba(0,0,0,0.0)');
grad.addColorStop(0.4, 'rgba(0,0,0,0.55)');
grad.addColorStop(1, 'rgba(0,0,0,0.85)');
ctx.fillStyle = grad;
ctx.fillRect(0, stripY, W, stripH);
// Layout the text — fits within ~92% width, wrapping at word boundaries.
const fontSize = Math.round(W * 0.045);
const padX = Math.round(W * 0.06);
const innerW = W - padX * 2;
ctx.font = `600 ${fontSize}px -apple-system, "Inter", system-ui, sans-serif`;
ctx.textBaseline = 'middle';
ctx.textAlign = 'left';
const lines = wrapText(ctx, text, innerW);
const lineH = Math.round(fontSize * 1.25);
const totalH = lines.length * lineH;
const startY = stripY + (stripH - totalH) / 2 + lineH / 2;
// Word-level highlight: count tokens up to ds.progress, color spoken
// ones in the accent color.
const allWords = text.match(/\S+/g) || [];
const spokenCount = Math.round((ds.progress || 0) * allWords.length);
let wordIdx = 0;
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const words = line.match(/\S+/g) || [];
let x = padX;
for (const w of words) {
const isSpoken = wordIdx < spokenCount;
ctx.fillStyle = isSpoken ? '#E87B4E' : 'rgba(255,255,255,0.92)';
ctx.shadowColor = 'rgba(0,0,0,0.6)';
ctx.shadowBlur = 6;
ctx.fillText(w, x, startY + i * lineH);
x += ctx.measureText(w + ' ').width;
wordIdx++;
}
}
ctx.shadowBlur = 0;
}
function wrapText(ctx, text, maxWidth) {
const words = text.split(/\s+/);
const lines = [];
let line = '';
for (const w of words) {
const test = line ? line + ' ' + w : w;
if (ctx.measureText(test).width <= maxWidth) {
line = test;
} else {
if (line) lines.push(line);
line = w;
}
}
if (line) lines.push(line);
// Hard-cap to 3 lines so it never eats the video.
if (lines.length > 3) {
lines.length = 3;
lines[2] = lines[2].replace(/\s+\S*$/, '') + '…';
}
return lines;
}
function pickRecorderMime() {
if (typeof MediaRecorder === 'undefined') return '';
const candidates = [
'video/webm;codecs=vp9,opus',
'video/webm;codecs=vp8,opus',
'video/webm;codecs=h264,opus',
'video/webm',
'video/mp4;codecs=h264,aac',
'video/mp4',
];
for (const c of candidates) {
try { if (MediaRecorder.isTypeSupported(c)) return c; } catch {}
}
return '';
}
// ─────────────────────────────────────────────────────────────
// Smart teleprompter: SpeechRecognition + sliding-window aligner
// ─────────────────────────────────────────────────────────────
function useSpeechFollow({ enabled, scriptText, lang, onAdvance, onResult }) {
const hasSR = typeof window !== 'undefined' && (
'SpeechRecognition' in window || 'webkitSpeechRecognition' in window
);
// We treat the feature as supported if either the browser has SR OR the
// server has a configured /api/transcribe (we probe lazily on first enable).
const [serverFallback, setServerFallback] = React.useState(false);
const supported = hasSR || serverFallback;
const [progress, setProgress] = React.useState(0);
const [listening, setListening] = React.useState(false);
const [error, setError] = React.useState(null);
const [provider, setProvider] = React.useState(hasSR ? 'browser' : 'pending');
const recRef = React.useRef(null);
const alignerRef = React.useRef(null);
const advancedRef = React.useRef(false);
const accumulatedTextRef = React.useRef('');
const fallbackRecRef = React.useRef(null);
// Rebuild aligner whenever the active script or language changes.
React.useEffect(() => {
alignerRef.current = makeWordAligner(scriptText, lang);
setProgress(0);
advancedRef.current = false;
accumulatedTextRef.current = '';
}, [scriptText, lang]);
// Probe /api/transcribe once when the user enables smart-follow on a
// browser without SpeechRecognition (Firefox).
React.useEffect(() => {
if (!enabled || hasSR) return;
let cancelled = false;
(async () => {
try {
// HEAD-style probe: empty POST returns a quick 400 if route exists,
// 404/503 otherwise. We treat any 2xx/4xx (non-503) as "route exists".
const r = await fetch('/api/transcribe', { method: 'POST', body: new Uint8Array() });
if (cancelled) return;
if (r.status === 503) {
setError('Smart follow needs Chrome/Edge/Safari, or a configured /api/transcribe.');
setProvider('unavailable');
} else {
setServerFallback(true);
setProvider('server');
}
} catch (e) {
if (!cancelled) {
setError('Smart follow needs Chrome/Edge/Safari, or a configured /api/transcribe.');
setProvider('unavailable');
}
}
})();
return () => { cancelled = true; };
}, [enabled, hasSR]);
// Browser SpeechRecognition path (Chrome / Edge / Safari).
React.useEffect(() => {
if (!enabled || !hasSR) return;
const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
let stopped = false;
let rec;
try {
rec = new SR();
} catch (e) {
setError(e.message);
return;
}
rec.continuous = true;
rec.interimResults = true;
rec.lang = mapBcp47(lang);
rec.onstart = () => setListening(true);
rec.onresult = (e) => {
let full = '';
for (let i = 0; i < e.results.length; i++) full += e.results[i][0].transcript + ' ';
feedAligner(full);
};
rec.onerror = (e) => {
if (e.error && e.error !== 'no-speech' && e.error !== 'aborted') {
setError(e.error);
}
};
rec.onend = () => {
setListening(false);
if (!stopped) { try { rec.start(); } catch {} }
};
try { rec.start(); } catch (e) { setError(e.message); }
recRef.current = rec;
return () => {
stopped = true;
try { rec.onend = null; rec.abort(); } catch {}
recRef.current = null;
setListening(false);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [enabled, hasSR, lang]);
// Server-fallback path (Firefox or any browser w/o SpeechRecognition).
// Captures the mic in 5s windows and POSTs each chunk to /api/transcribe,
// accumulating returned text. Higher latency than browser SR but works.
React.useEffect(() => {
if (!enabled || hasSR || !serverFallback) return;
let stopped = false;
let stream;
let recorder;
(async () => {
try {
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
if (stopped) { stream.getTracks().forEach(t => t.stop()); return; }
const mime = pickRecorderMime();
recorder = new MediaRecorder(stream, mime ? { mimeType: mime } : undefined);
const localChunks = [];
recorder.ondataavailable = async (e) => {
if (!e.data || e.data.size === 0) return;
localChunks.push(e.data);
};
// Every time the recorder emits a timeslice, we ship the accumulated
// chunks. Each clip is short (~5s) so latency stays usable.
let inflight = false;
const flush = async () => {
if (inflight || localChunks.length === 0) return;
inflight = true;
const blob = new Blob(localChunks.splice(0, localChunks.length), { type: recorder.mimeType });
try {
const r = await fetch('/api/transcribe' + (lang ? `?lang=${encodeURIComponent(mapBcp47(lang).split('-')[0])}` : ''), {
method: 'POST',
body: await blob.arrayBuffer(),
headers: { 'Content-Type': blob.type || 'audio/webm' },
});
if (r.ok) {
const data = await r.json();
if (data.text) {
accumulatedTextRef.current = (accumulatedTextRef.current + ' ' + data.text).trim();
feedAligner(accumulatedTextRef.current);
}
}
} catch (err) {
// Network blip — keep recording, try next chunk.
} finally {
inflight = false;
}
};
recorder.onstart = () => setListening(true);
recorder.onstop = () => setListening(false);
recorder.start(5000); // 5s timeslices
// Poll for new chunks
const id = setInterval(flush, 5500);
fallbackRecRef.current = { recorder, stream, intervalId: id };
} catch (e) {
setError(e.message || String(e));
}
})();
return () => {
stopped = true;
const ref = fallbackRecRef.current;
if (ref) {
clearInterval(ref.intervalId);
try { if (ref.recorder.state !== 'inactive') ref.recorder.stop(); } catch {}
ref.stream.getTracks().forEach(t => t.stop());
fallbackRecRef.current = null;
}
setListening(false);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [enabled, hasSR, serverFallback, lang]);
function feedAligner(full) {
const aligner = alignerRef.current;
const p = aligner ? aligner.feed(full) : 0;
setProgress(p);
onResult && onResult({ text: full, progress: p });
if (p >= 0.85 && !advancedRef.current) {
advancedRef.current = true;
onAdvance && onAdvance();
}
}
return { progress, listening, error, supported, provider };
}
// makeWordAligner — given a script and language, returns a stateful matcher
// that consumes growing transcripts and reports the script-position cursor
// (as a 0..1 progress fraction). The cursor is monotonic — once a word is
// "spoken" it stays counted, so a stutter in the recognizer doesn't rewind
// the teleprompter.
function makeWordAligner(scriptText, lang) {
const bcp = mapBcp47(lang);
const scriptTokens = tokenize(scriptText, bcp).map(normalizeToken).filter(Boolean);
let cursor = 0;
return {
feed(transcript) {
const tt = tokenize(transcript, bcp).map(normalizeToken).filter(Boolean);
if (!tt.length || !scriptTokens.length) {
return cursor / Math.max(1, scriptTokens.length);
}
// Match the trailing N transcript tokens against the next K script tokens.
// Tail length 1..4, window 12. Pick the longest exact-suffix match found.
const TAIL = Math.min(4, tt.length);
const LOOKAHEAD = 12;
const start = cursor;
const end = Math.min(scriptTokens.length, start + LOOKAHEAD);
let best = cursor;
for (let len = TAIL; len >= 1; len--) {
const tail = tt.slice(-len);
for (let i = start; i + len <= end; i++) {
let ok = true;
for (let j = 0; j < len; j++) {
if (scriptTokens[i + j] !== tail[j]) { ok = false; break; }
}
if (ok && i + len > best) best = i + len;
}
if (best > cursor) break; // longest match wins; bail early
}
cursor = best;
return cursor / scriptTokens.length;
},
reset() { cursor = 0; },
};
}
function tokenize(text, bcp) {
if (!text) return [];
try {
if (typeof Intl !== 'undefined' && Intl.Segmenter) {
const seg = new Intl.Segmenter(bcp, { granularity: 'word' });
return [...seg.segment(text)].filter(s => s.isWordLike).map(s => s.segment);
}
} catch {}
return text.split(/\s+/).filter(Boolean);
}
function normalizeToken(s) {
if (!s) return '';
try { s = s.normalize('NFKC'); } catch {}
return s.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
}
function mapBcp47(lang) {
if (!lang) return 'en-US';
if (lang === 'ja') return 'ja-JP';
if (lang === 'zh') return 'zh-CN';
if (lang === 'en') return 'en-US';
return lang === 'auto' ? 'en-US' : lang;
}
function humanizeMediaError(err, source) {
const name = err?.name || '';
if (name === 'NotAllowedError' || name === 'PermissionDeniedError') {
return source === 'screen'
? 'Screen sharing was cancelled. Click the screen icon again to retry.'
: 'Camera & mic permission denied. Enable them in the browser site settings, then reload.';
}
if (name === 'NotFoundError' || name === 'DevicesNotFoundError') {
return 'No camera or microphone found on this device.';
}
if (name === 'NotReadableError') {
return 'Camera is in use by another app. Close it and try again.';
}
return (err?.message || 'Could not access ' + source + '.');
}
window.ScreenRecord = ScreenRecord;
window.makeWordAligner = makeWordAligner;
window.tokenize = tokenize;
window.normalizeToken = normalizeToken;
window.startCompositor = startCompositor;
window.aspectToSize = aspectToSize;
window.wrapText = wrapText;
// Shared with studio-record.jsx so the desktop record view can reuse
// the same media + speech pipeline without duplicating the code.
window.useSpeechFollow = useSpeechFollow;
window.getMediaStream = getMediaStream;
window.pickRecorderMime = pickRecorderMime;
window.humanizeMediaError = humanizeMediaError;
window.composeBeautyFilter = composeBeautyFilter;
window.BEAUTY_PRESETS = BEAUTY_PRESETS;
// ─────────────────────────────────────────────────────────────
// Beauty filter — CSS/Canvas filter string from slider values.
// Same string is applied to the live element and to
// ctx.filter in the compositor so the export matches the preview.
//
// `smoothness` (0..100) — slight blur acts as skin smoothing
// `brightness` (0..100, 50=neutral) — brightness 0.85..1.20
// `warmth` (0..100, 50=neutral) — sepia 0..0.18 + saturate 0.9..1.15
// `glow` (0..100) — contrast 1..1.12 + saturate 1..1.18
// ─────────────────────────────────────────────────────────────
function composeBeautyFilter({ smoothness = 0, brightness = 50, warmth = 50, glow = 0 } = {}) {
const parts = [];
// Brightness: 50 → 1.0, 0 → 0.85, 100 → 1.20
if (brightness !== 50) {
const b = brightness < 50
? 0.85 + (brightness / 50) * 0.15
: 1.0 + ((brightness - 50) / 50) * 0.20;
parts.push(`brightness(${b.toFixed(3)})`);
}
// Saturate / sepia for warmth
if (warmth !== 50) {
if (warmth > 50) {
const sep = ((warmth - 50) / 50) * 0.18;
const sat = 1.0 + ((warmth - 50) / 50) * 0.15;
parts.push(`sepia(${sep.toFixed(3)})`);
parts.push(`saturate(${sat.toFixed(3)})`);
} else {
// Cooler — desaturate slightly + tiny hue rotate toward blue
const sat = 0.85 + (warmth / 50) * 0.15;
const hue = -(50 - warmth) / 5; // up to -10deg
parts.push(`saturate(${sat.toFixed(3)})`);
parts.push(`hue-rotate(${hue.toFixed(1)}deg)`);
}
}
// Glow: contrast + saturate bump (different feel than warmth)
if (glow > 0) {
const c = 1.0 + (glow / 100) * 0.12;
const s = 1.0 + (glow / 100) * 0.18;
parts.push(`contrast(${c.toFixed(3)})`);
parts.push(`saturate(${s.toFixed(3)})`);
}
// Skin smoothness — small blur. Capped at 1.8px to keep edges legible.
if (smoothness > 0) {
const blur = (smoothness / 100) * 1.8;
parts.push(`blur(${blur.toFixed(2)}px)`);
}
return parts.length ? parts.join(' ') : 'none';
}
// Named preset palettes — single source of truth for the UI chips and
// the default applied when the user toggles beauty on.
const BEAUTY_PRESETS = {
off: { smoothness: 0, brightness: 50, warmth: 50, glow: 0 },
natural: { smoothness: 12, brightness: 54, warmth: 54, glow: 8 },
soft: { smoothness: 28, brightness: 58, warmth: 56, glow: 12 },
bright: { smoothness: 18, brightness: 66, warmth: 50, glow: 20 },
warm: { smoothness: 22, brightness: 56, warmth: 68, glow: 14 },
cool: { smoothness: 18, brightness: 56, warmth: 36, glow: 12 },
cinematic: { smoothness: 14, brightness: 48, warmth: 58, glow: 28 },
};