322 lines
12 KiB
TypeScript
322 lines
12 KiB
TypeScript
import React from "react";
|
|
import ReactDOM from "react-dom/client";
|
|
import { Check, Clock, Play, RefreshCw, Send, ShieldCheck, X } from "lucide-react";
|
|
import "./styles.css";
|
|
|
|
type TranscriptChunk = { speaker: string; timestamp: string; text: string };
|
|
type PendingAction = {
|
|
id: number;
|
|
tool: string;
|
|
intent: string;
|
|
reasoning: string;
|
|
confidence: number;
|
|
payload: Record<string, unknown>;
|
|
extracted_entities: Record<string, unknown>;
|
|
validation: { valid: boolean; errors: string[] };
|
|
policy: { allowed: boolean; reasons: string[] };
|
|
status: string;
|
|
execution_result?: unknown;
|
|
created_at: string;
|
|
};
|
|
type Dashboard = {
|
|
transcript: TranscriptChunk[];
|
|
context: Record<string, unknown>;
|
|
memories: Array<Record<string, unknown>>;
|
|
pending_actions: PendingAction[];
|
|
tasks: Array<Record<string, unknown>>;
|
|
notes: Array<Record<string, unknown>>;
|
|
reservations: Array<Record<string, unknown>>;
|
|
events: Array<Record<string, unknown>>;
|
|
};
|
|
type TranscriptionJob = {
|
|
job_id: string;
|
|
status: "uploading" | "queued" | "processing" | "completed" | "error";
|
|
transcription_id?: string | null;
|
|
segments: Array<{
|
|
id: string;
|
|
speaker: string;
|
|
original_text: string;
|
|
edited_text: string;
|
|
start_ms: number;
|
|
end_ms: number;
|
|
}>;
|
|
generated_action_ids: number[];
|
|
error?: string | null;
|
|
};
|
|
|
|
const API = import.meta.env.VITE_API_URL ?? "";
|
|
const WS_URL = import.meta.env.VITE_WS_URL ?? `ws://${window.location.host}/ws/transcript`;
|
|
const sampleTranscript = [
|
|
{ speaker: "Bride", text: "Add a note that the customer prefers white flowers." },
|
|
{ speaker: "Planner", text: "Create a task for Maria to confirm the photographer." },
|
|
{ speaker: "Groom", text: "We should reserve the venue for July 15." },
|
|
{ speaker: "Bride", text: "I'll call the florist tomorrow." }
|
|
];
|
|
|
|
function JsonBlock({ value }: { value: unknown }) {
|
|
return <pre>{JSON.stringify(value, null, 2)}</pre>;
|
|
}
|
|
|
|
function App() {
|
|
const [dashboard, setDashboard] = React.useState<Dashboard | null>(null);
|
|
const [speaker, setSpeaker] = React.useState("Bride");
|
|
const [text, setText] = React.useState("Create a task for Maria to confirm the photographer.");
|
|
const [busy, setBusy] = React.useState(false);
|
|
const [isRecording, setIsRecording] = React.useState(false);
|
|
const [recordingError, setRecordingError] = React.useState<string | null>(null);
|
|
const [recordingSeconds, setRecordingSeconds] = React.useState(0);
|
|
const [transcriptionJob, setTranscriptionJob] = React.useState<TranscriptionJob | null>(null);
|
|
const mediaRecorderRef = React.useRef<MediaRecorder | null>(null);
|
|
const mediaStreamRef = React.useRef<MediaStream | null>(null);
|
|
const audioChunksRef = React.useRef<Blob[]>([]);
|
|
const recordingStartedAtRef = React.useRef<number | null>(null);
|
|
|
|
const refresh = React.useCallback(async () => {
|
|
const response = await fetch(`${API}/dashboard`);
|
|
setDashboard(await response.json());
|
|
}, []);
|
|
|
|
React.useEffect(() => {
|
|
refresh();
|
|
const interval = window.setInterval(refresh, 2500);
|
|
return () => window.clearInterval(interval);
|
|
}, [refresh]);
|
|
|
|
React.useEffect(() => {
|
|
if (!isRecording) {
|
|
return;
|
|
}
|
|
const timer = window.setInterval(() => {
|
|
const startedAt = recordingStartedAtRef.current;
|
|
setRecordingSeconds(startedAt ? Math.floor((Date.now() - startedAt) / 1000) : 0);
|
|
}, 500);
|
|
return () => window.clearInterval(timer);
|
|
}, [isRecording]);
|
|
|
|
React.useEffect(() => {
|
|
const jobId = transcriptionJob?.job_id;
|
|
if (!jobId || transcriptionJob.status === "completed" || transcriptionJob.status === "error") {
|
|
return;
|
|
}
|
|
const timer = window.setInterval(async () => {
|
|
const response = await fetch(`${API}/transcriptions/${jobId}`);
|
|
if (!response.ok) {
|
|
return;
|
|
}
|
|
const job = (await response.json()) as TranscriptionJob;
|
|
setTranscriptionJob(job);
|
|
if (job.status === "completed") {
|
|
await refresh();
|
|
}
|
|
}, 2000);
|
|
return () => window.clearInterval(timer);
|
|
}, [transcriptionJob, refresh]);
|
|
|
|
const sendChunk = React.useCallback(async (chunk = { speaker, text }) => {
|
|
setBusy(true);
|
|
await new Promise<void>((resolve, reject) => {
|
|
const ws = new WebSocket(WS_URL);
|
|
ws.onopen = () => ws.send(JSON.stringify({ ...chunk, timestamp: new Date().toISOString() }));
|
|
ws.onmessage = () => {
|
|
ws.close();
|
|
resolve();
|
|
};
|
|
ws.onerror = () => reject(new Error("WebSocket failed"));
|
|
}).finally(() => setBusy(false));
|
|
await refresh();
|
|
}, [speaker, text, refresh]);
|
|
|
|
const replay = async () => {
|
|
for (const chunk of sampleTranscript) {
|
|
await sendChunk(chunk);
|
|
await new Promise((resolve) => window.setTimeout(resolve, 450));
|
|
}
|
|
};
|
|
|
|
const pickMimeType = () => {
|
|
if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) return "audio/webm;codecs=opus";
|
|
if (MediaRecorder.isTypeSupported("audio/webm")) return "audio/webm";
|
|
if (MediaRecorder.isTypeSupported("audio/mp4")) return "audio/mp4";
|
|
return "audio/webm";
|
|
};
|
|
|
|
const startMic = async () => {
|
|
setRecordingError(null);
|
|
setTranscriptionJob(null);
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
|
|
});
|
|
const recorder = new MediaRecorder(stream, { mimeType: pickMimeType() });
|
|
audioChunksRef.current = [];
|
|
mediaRecorderRef.current = recorder;
|
|
mediaStreamRef.current = stream;
|
|
recordingStartedAtRef.current = Date.now();
|
|
recorder.ondataavailable = (event) => {
|
|
if (event.data.size > 0) audioChunksRef.current.push(event.data);
|
|
};
|
|
recorder.start(250);
|
|
setRecordingSeconds(0);
|
|
setIsRecording(true);
|
|
};
|
|
|
|
const stopMic = async () => {
|
|
const recorder = mediaRecorderRef.current;
|
|
if (!recorder) return;
|
|
const blob = await new Promise<Blob | null>((resolve) => {
|
|
recorder.onstop = () => {
|
|
const chunks = audioChunksRef.current;
|
|
resolve(chunks.length > 0 ? new Blob(chunks, { type: recorder.mimeType }) : null);
|
|
};
|
|
try {
|
|
recorder.requestData();
|
|
} catch {
|
|
// Ignore browsers that have no buffered data yet.
|
|
}
|
|
recorder.stop();
|
|
});
|
|
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
|
|
mediaRecorderRef.current = null;
|
|
mediaStreamRef.current = null;
|
|
setIsRecording(false);
|
|
recordingStartedAtRef.current = null;
|
|
if (!blob) {
|
|
setRecordingError("No microphone audio was captured.");
|
|
return;
|
|
}
|
|
const form = new FormData();
|
|
const extension = blob.type.includes("mp4") ? "m4a" : "webm";
|
|
form.append("audio", blob, `meeting-recording.${extension}`);
|
|
const response = await fetch(`${API}/transcriptions`, { method: "POST", body: form });
|
|
if (!response.ok) {
|
|
const body = await response.json().catch(() => null);
|
|
setRecordingError(body?.detail ?? "Audio upload failed.");
|
|
return;
|
|
}
|
|
setTranscriptionJob((await response.json()) as TranscriptionJob);
|
|
};
|
|
|
|
const toggleMic = async () => {
|
|
try {
|
|
if (isRecording) {
|
|
await stopMic();
|
|
} else {
|
|
await startMic();
|
|
}
|
|
} catch (error) {
|
|
setRecordingError(error instanceof Error ? error.message : "Microphone access failed.");
|
|
setIsRecording(false);
|
|
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
|
|
}
|
|
};
|
|
|
|
const generateCall = async (id: number) => {
|
|
await fetch(`${API}/pending-actions/${id}/approve`, { method: "POST" });
|
|
await refresh();
|
|
};
|
|
|
|
const reject = async (id: number) => {
|
|
await fetch(`${API}/pending-actions/${id}/reject`, { method: "POST" });
|
|
await refresh();
|
|
};
|
|
|
|
const latest = dashboard?.pending_actions[0];
|
|
|
|
return (
|
|
<main className="shell">
|
|
<header className="topbar">
|
|
<div>
|
|
<h1>AI Orchestrator</h1>
|
|
<p>Wedding planning meeting control room</p>
|
|
</div>
|
|
<button title="Refresh dashboard" onClick={refresh}><RefreshCw size={18} /> Refresh</button>
|
|
</header>
|
|
|
|
<section className="grid">
|
|
<aside className="panel transcript">
|
|
<div className="panel-title"><Clock size={18} /> Live Transcript</div>
|
|
<div className="mic-box">
|
|
<button onClick={toggleMic} className={isRecording ? "recording" : ""}>
|
|
{isRecording ? <X size={17} /> : <Play size={17} />}
|
|
{isRecording ? "Stop Mic" : "Start Mic"}
|
|
</button>
|
|
<span>{isRecording ? `${recordingSeconds}s` : transcriptionJob ? transcriptionJob.status : "Soniox mic transcription"}</span>
|
|
{recordingError && <p className="error">{recordingError}</p>}
|
|
{transcriptionJob?.status === "completed" && (
|
|
<p className="success">Generated {transcriptionJob.generated_action_ids.length} pending action(s).</p>
|
|
)}
|
|
</div>
|
|
<div className="composer">
|
|
<input value={speaker} onChange={(event) => setSpeaker(event.target.value)} aria-label="Speaker" />
|
|
<textarea value={text} onChange={(event) => setText(event.target.value)} aria-label="Transcript text" />
|
|
<div className="actions-row">
|
|
<button onClick={() => sendChunk()} disabled={busy}><Send size={17} /> Send</button>
|
|
<button onClick={replay} disabled={busy}><Play size={17} /> Replay</button>
|
|
</div>
|
|
</div>
|
|
<div className="list">
|
|
{(dashboard?.transcript ?? []).map((chunk, index) => (
|
|
<article className="line" key={`${chunk.timestamp}-${index}`}>
|
|
<strong>{chunk.speaker}</strong>
|
|
<span>{new Date(chunk.timestamp).toLocaleTimeString()}</span>
|
|
<p>{chunk.text}</p>
|
|
</article>
|
|
))}
|
|
</div>
|
|
</aside>
|
|
|
|
<section className="panel reasoning">
|
|
<div className="panel-title"><ShieldCheck size={18} /> Planner, Policy, Memory</div>
|
|
{latest ? (
|
|
<div className="decision">
|
|
<div className="metric-row">
|
|
<span className="tag">{latest.intent}</span>
|
|
<span className="confidence">{Math.round(latest.confidence * 100)}%</span>
|
|
<span className={`status ${latest.status}`}>{latest.status.replace("_", " ")}</span>
|
|
</div>
|
|
<p className="reason">{latest.reasoning}</p>
|
|
<div className="columns">
|
|
<div><h2>Payload</h2><JsonBlock value={latest.payload} /></div>
|
|
<div><h2>Validation</h2><JsonBlock value={latest.validation} /></div>
|
|
</div>
|
|
<div className="columns">
|
|
<div><h2>Policy</h2><JsonBlock value={latest.policy} /></div>
|
|
<div><h2>Retrieved Memories</h2><JsonBlock value={dashboard?.memories ?? []} /></div>
|
|
</div>
|
|
</div>
|
|
) : (
|
|
<div className="empty">Send or replay transcript chunks to create reviewed AI actions.</div>
|
|
)}
|
|
</section>
|
|
|
|
<aside className="panel pending">
|
|
<div className="panel-title"><Check size={18} /> Pending Review</div>
|
|
<div className="list">
|
|
{(dashboard?.pending_actions ?? []).map((action) => (
|
|
<article className="action-card" key={action.id}>
|
|
<div className="card-head">
|
|
<strong>{action.tool}</strong>
|
|
<span className={`status ${action.status}`}>{action.status.replace("_", " ")}</span>
|
|
</div>
|
|
<p>{action.reasoning}</p>
|
|
<JsonBlock value={action.payload} />
|
|
<div className="actions-row">
|
|
<button onClick={() => generateCall(action.id)} disabled={action.status !== "pending_review"}><Check size={16} /> Generate Call</button>
|
|
<button onClick={() => reject(action.id)} disabled={action.status !== "pending_review"}><X size={16} /> Reject</button>
|
|
</div>
|
|
</article>
|
|
))}
|
|
</div>
|
|
</aside>
|
|
</section>
|
|
|
|
<section className="bottom">
|
|
<div className="panel"><div className="panel-title">Current Context</div><JsonBlock value={dashboard?.context ?? {}} /></div>
|
|
<div className="panel"><div className="panel-title">Execution Logs</div><JsonBlock value={dashboard?.events ?? []} /></div>
|
|
<div className="panel"><div className="panel-title">REST State</div><JsonBlock value={{ tasks: dashboard?.tasks, notes: dashboard?.notes, reservations: dashboard?.reservations }} /></div>
|
|
</section>
|
|
</main>
|
|
);
|
|
}
|
|
|
|
ReactDOM.createRoot(document.getElementById("root")!).render(<App />);
|