Implement AI orchestration wedding demo
This commit is contained in:
pelpanagiotis
321
frontend/src/main.tsx
Normal file
321
frontend/src/main.tsx
Normal file
@@ -0,0 +1,321 @@
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { Check, Clock, Play, RefreshCw, Send, ShieldCheck, X } from "lucide-react";
|
||||
import "./styles.css";
|
||||
|
||||
type TranscriptChunk = { speaker: string; timestamp: string; text: string };
|
||||
type PendingAction = {
|
||||
id: number;
|
||||
tool: string;
|
||||
intent: string;
|
||||
reasoning: string;
|
||||
confidence: number;
|
||||
payload: Record<string, unknown>;
|
||||
extracted_entities: Record<string, unknown>;
|
||||
validation: { valid: boolean; errors: string[] };
|
||||
policy: { allowed: boolean; reasons: string[] };
|
||||
status: string;
|
||||
execution_result?: unknown;
|
||||
created_at: string;
|
||||
};
|
||||
type Dashboard = {
|
||||
transcript: TranscriptChunk[];
|
||||
context: Record<string, unknown>;
|
||||
memories: Array<Record<string, unknown>>;
|
||||
pending_actions: PendingAction[];
|
||||
tasks: Array<Record<string, unknown>>;
|
||||
notes: Array<Record<string, unknown>>;
|
||||
reservations: Array<Record<string, unknown>>;
|
||||
events: Array<Record<string, unknown>>;
|
||||
};
|
||||
type TranscriptionJob = {
|
||||
job_id: string;
|
||||
status: "uploading" | "queued" | "processing" | "completed" | "error";
|
||||
transcription_id?: string | null;
|
||||
segments: Array<{
|
||||
id: string;
|
||||
speaker: string;
|
||||
original_text: string;
|
||||
edited_text: string;
|
||||
start_ms: number;
|
||||
end_ms: number;
|
||||
}>;
|
||||
generated_action_ids: number[];
|
||||
error?: string | null;
|
||||
};
|
||||
|
||||
const API = import.meta.env.VITE_API_URL ?? "";
|
||||
const WS_URL = import.meta.env.VITE_WS_URL ?? `ws://${window.location.host}/ws/transcript`;
|
||||
const sampleTranscript = [
|
||||
{ speaker: "Bride", text: "Add a note that the customer prefers white flowers." },
|
||||
{ speaker: "Planner", text: "Create a task for Maria to confirm the photographer." },
|
||||
{ speaker: "Groom", text: "We should reserve the venue for July 15." },
|
||||
{ speaker: "Bride", text: "I'll call the florist tomorrow." }
|
||||
];
|
||||
|
||||
function JsonBlock({ value }: { value: unknown }) {
|
||||
return <pre>{JSON.stringify(value, null, 2)}</pre>;
|
||||
}
|
||||
|
||||
function App() {
|
||||
const [dashboard, setDashboard] = React.useState<Dashboard | null>(null);
|
||||
const [speaker, setSpeaker] = React.useState("Bride");
|
||||
const [text, setText] = React.useState("Create a task for Maria to confirm the photographer.");
|
||||
const [busy, setBusy] = React.useState(false);
|
||||
const [isRecording, setIsRecording] = React.useState(false);
|
||||
const [recordingError, setRecordingError] = React.useState<string | null>(null);
|
||||
const [recordingSeconds, setRecordingSeconds] = React.useState(0);
|
||||
const [transcriptionJob, setTranscriptionJob] = React.useState<TranscriptionJob | null>(null);
|
||||
const mediaRecorderRef = React.useRef<MediaRecorder | null>(null);
|
||||
const mediaStreamRef = React.useRef<MediaStream | null>(null);
|
||||
const audioChunksRef = React.useRef<Blob[]>([]);
|
||||
const recordingStartedAtRef = React.useRef<number | null>(null);
|
||||
|
||||
const refresh = React.useCallback(async () => {
|
||||
const response = await fetch(`${API}/dashboard`);
|
||||
setDashboard(await response.json());
|
||||
}, []);
|
||||
|
||||
React.useEffect(() => {
|
||||
refresh();
|
||||
const interval = window.setInterval(refresh, 2500);
|
||||
return () => window.clearInterval(interval);
|
||||
}, [refresh]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!isRecording) {
|
||||
return;
|
||||
}
|
||||
const timer = window.setInterval(() => {
|
||||
const startedAt = recordingStartedAtRef.current;
|
||||
setRecordingSeconds(startedAt ? Math.floor((Date.now() - startedAt) / 1000) : 0);
|
||||
}, 500);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [isRecording]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const jobId = transcriptionJob?.job_id;
|
||||
if (!jobId || transcriptionJob.status === "completed" || transcriptionJob.status === "error") {
|
||||
return;
|
||||
}
|
||||
const timer = window.setInterval(async () => {
|
||||
const response = await fetch(`${API}/transcriptions/${jobId}`);
|
||||
if (!response.ok) {
|
||||
return;
|
||||
}
|
||||
const job = (await response.json()) as TranscriptionJob;
|
||||
setTranscriptionJob(job);
|
||||
if (job.status === "completed") {
|
||||
await refresh();
|
||||
}
|
||||
}, 2000);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [transcriptionJob, refresh]);
|
||||
|
||||
const sendChunk = React.useCallback(async (chunk = { speaker, text }) => {
|
||||
setBusy(true);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ws = new WebSocket(WS_URL);
|
||||
ws.onopen = () => ws.send(JSON.stringify({ ...chunk, timestamp: new Date().toISOString() }));
|
||||
ws.onmessage = () => {
|
||||
ws.close();
|
||||
resolve();
|
||||
};
|
||||
ws.onerror = () => reject(new Error("WebSocket failed"));
|
||||
}).finally(() => setBusy(false));
|
||||
await refresh();
|
||||
}, [speaker, text, refresh]);
|
||||
|
||||
const replay = async () => {
|
||||
for (const chunk of sampleTranscript) {
|
||||
await sendChunk(chunk);
|
||||
await new Promise((resolve) => window.setTimeout(resolve, 450));
|
||||
}
|
||||
};
|
||||
|
||||
const pickMimeType = () => {
|
||||
if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) return "audio/webm;codecs=opus";
|
||||
if (MediaRecorder.isTypeSupported("audio/webm")) return "audio/webm";
|
||||
if (MediaRecorder.isTypeSupported("audio/mp4")) return "audio/mp4";
|
||||
return "audio/webm";
|
||||
};
|
||||
|
||||
const startMic = async () => {
|
||||
setRecordingError(null);
|
||||
setTranscriptionJob(null);
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
|
||||
});
|
||||
const recorder = new MediaRecorder(stream, { mimeType: pickMimeType() });
|
||||
audioChunksRef.current = [];
|
||||
mediaRecorderRef.current = recorder;
|
||||
mediaStreamRef.current = stream;
|
||||
recordingStartedAtRef.current = Date.now();
|
||||
recorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) audioChunksRef.current.push(event.data);
|
||||
};
|
||||
recorder.start(250);
|
||||
setRecordingSeconds(0);
|
||||
setIsRecording(true);
|
||||
};
|
||||
|
||||
const stopMic = async () => {
|
||||
const recorder = mediaRecorderRef.current;
|
||||
if (!recorder) return;
|
||||
const blob = await new Promise<Blob | null>((resolve) => {
|
||||
recorder.onstop = () => {
|
||||
const chunks = audioChunksRef.current;
|
||||
resolve(chunks.length > 0 ? new Blob(chunks, { type: recorder.mimeType }) : null);
|
||||
};
|
||||
try {
|
||||
recorder.requestData();
|
||||
} catch {
|
||||
// Ignore browsers that have no buffered data yet.
|
||||
}
|
||||
recorder.stop();
|
||||
});
|
||||
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
mediaRecorderRef.current = null;
|
||||
mediaStreamRef.current = null;
|
||||
setIsRecording(false);
|
||||
recordingStartedAtRef.current = null;
|
||||
if (!blob) {
|
||||
setRecordingError("No microphone audio was captured.");
|
||||
return;
|
||||
}
|
||||
const form = new FormData();
|
||||
const extension = blob.type.includes("mp4") ? "m4a" : "webm";
|
||||
form.append("audio", blob, `meeting-recording.${extension}`);
|
||||
const response = await fetch(`${API}/transcriptions`, { method: "POST", body: form });
|
||||
if (!response.ok) {
|
||||
const body = await response.json().catch(() => null);
|
||||
setRecordingError(body?.detail ?? "Audio upload failed.");
|
||||
return;
|
||||
}
|
||||
setTranscriptionJob((await response.json()) as TranscriptionJob);
|
||||
};
|
||||
|
||||
const toggleMic = async () => {
|
||||
try {
|
||||
if (isRecording) {
|
||||
await stopMic();
|
||||
} else {
|
||||
await startMic();
|
||||
}
|
||||
} catch (error) {
|
||||
setRecordingError(error instanceof Error ? error.message : "Microphone access failed.");
|
||||
setIsRecording(false);
|
||||
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
};
|
||||
|
||||
const generateCall = async (id: number) => {
|
||||
await fetch(`${API}/pending-actions/${id}/approve`, { method: "POST" });
|
||||
await refresh();
|
||||
};
|
||||
|
||||
const reject = async (id: number) => {
|
||||
await fetch(`${API}/pending-actions/${id}/reject`, { method: "POST" });
|
||||
await refresh();
|
||||
};
|
||||
|
||||
const latest = dashboard?.pending_actions[0];
|
||||
|
||||
return (
|
||||
<main className="shell">
|
||||
<header className="topbar">
|
||||
<div>
|
||||
<h1>AI Orchestrator</h1>
|
||||
<p>Wedding planning meeting control room</p>
|
||||
</div>
|
||||
<button title="Refresh dashboard" onClick={refresh}><RefreshCw size={18} /> Refresh</button>
|
||||
</header>
|
||||
|
||||
<section className="grid">
|
||||
<aside className="panel transcript">
|
||||
<div className="panel-title"><Clock size={18} /> Live Transcript</div>
|
||||
<div className="mic-box">
|
||||
<button onClick={toggleMic} className={isRecording ? "recording" : ""}>
|
||||
{isRecording ? <X size={17} /> : <Play size={17} />}
|
||||
{isRecording ? "Stop Mic" : "Start Mic"}
|
||||
</button>
|
||||
<span>{isRecording ? `${recordingSeconds}s` : transcriptionJob ? transcriptionJob.status : "Soniox mic transcription"}</span>
|
||||
{recordingError && <p className="error">{recordingError}</p>}
|
||||
{transcriptionJob?.status === "completed" && (
|
||||
<p className="success">Generated {transcriptionJob.generated_action_ids.length} pending action(s).</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="composer">
|
||||
<input value={speaker} onChange={(event) => setSpeaker(event.target.value)} aria-label="Speaker" />
|
||||
<textarea value={text} onChange={(event) => setText(event.target.value)} aria-label="Transcript text" />
|
||||
<div className="actions-row">
|
||||
<button onClick={() => sendChunk()} disabled={busy}><Send size={17} /> Send</button>
|
||||
<button onClick={replay} disabled={busy}><Play size={17} /> Replay</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="list">
|
||||
{(dashboard?.transcript ?? []).map((chunk, index) => (
|
||||
<article className="line" key={`${chunk.timestamp}-${index}`}>
|
||||
<strong>{chunk.speaker}</strong>
|
||||
<span>{new Date(chunk.timestamp).toLocaleTimeString()}</span>
|
||||
<p>{chunk.text}</p>
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<section className="panel reasoning">
|
||||
<div className="panel-title"><ShieldCheck size={18} /> Planner, Policy, Memory</div>
|
||||
{latest ? (
|
||||
<div className="decision">
|
||||
<div className="metric-row">
|
||||
<span className="tag">{latest.intent}</span>
|
||||
<span className="confidence">{Math.round(latest.confidence * 100)}%</span>
|
||||
<span className={`status ${latest.status}`}>{latest.status.replace("_", " ")}</span>
|
||||
</div>
|
||||
<p className="reason">{latest.reasoning}</p>
|
||||
<div className="columns">
|
||||
<div><h2>Payload</h2><JsonBlock value={latest.payload} /></div>
|
||||
<div><h2>Validation</h2><JsonBlock value={latest.validation} /></div>
|
||||
</div>
|
||||
<div className="columns">
|
||||
<div><h2>Policy</h2><JsonBlock value={latest.policy} /></div>
|
||||
<div><h2>Retrieved Memories</h2><JsonBlock value={dashboard?.memories ?? []} /></div>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="empty">Send or replay transcript chunks to create reviewed AI actions.</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<aside className="panel pending">
|
||||
<div className="panel-title"><Check size={18} /> Pending Review</div>
|
||||
<div className="list">
|
||||
{(dashboard?.pending_actions ?? []).map((action) => (
|
||||
<article className="action-card" key={action.id}>
|
||||
<div className="card-head">
|
||||
<strong>{action.tool}</strong>
|
||||
<span className={`status ${action.status}`}>{action.status.replace("_", " ")}</span>
|
||||
</div>
|
||||
<p>{action.reasoning}</p>
|
||||
<JsonBlock value={action.payload} />
|
||||
<div className="actions-row">
|
||||
<button onClick={() => generateCall(action.id)} disabled={action.status !== "pending_review"}><Check size={16} /> Generate Call</button>
|
||||
<button onClick={() => reject(action.id)} disabled={action.status !== "pending_review"}><X size={16} /> Reject</button>
|
||||
</div>
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
</aside>
|
||||
</section>
|
||||
|
||||
<section className="bottom">
|
||||
<div className="panel"><div className="panel-title">Current Context</div><JsonBlock value={dashboard?.context ?? {}} /></div>
|
||||
<div className="panel"><div className="panel-title">Execution Logs</div><JsonBlock value={dashboard?.events ?? []} /></div>
|
||||
<div className="panel"><div className="panel-title">REST State</div><JsonBlock value={{ tasks: dashboard?.tasks, notes: dashboard?.notes, reservations: dashboard?.reservations }} /></div>
|
||||
</section>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(<App />);
|
||||
Reference in New Issue
Block a user