Implement AI orchestration wedding demo

This commit is contained in:
pelpanagiotis
2026-06-27 13:50:11 +03:00
commit 55d4fe95b4
67 changed files with 4736 additions and 0 deletions

15
.env.example Normal file
View File

@@ -0,0 +1,15 @@
ANTHROPIC_API_KEY=
ANTHROPIC_MODEL=claude-sonnet-4-6
WED_API_BASE_URL=http://host.docker.internal:8000
WED_API_OPENAPI_URL=
WED_API_BEARER_TOKEN=
WED_DEFAULT_PARTNER_ID=
WED_GUEST_FIRST_NAME=Demo
WED_GUEST_LAST_NAME=Customer
WED_GUEST_EMAIL=demo.customer@example.com
WED_GUEST_PHONE=
SONIOX_API_KEY=
SONIOX_MODEL=stt-async-preview
DATABASE_URL=postgresql+psycopg://orchestrator:orchestrator@postgres:5432/orchestrator
REDIS_URL=redis://redis:6379/0
QDRANT_URL=http://qdrant:6333

12
.gitignore vendored Normal file
View File

@@ -0,0 +1,12 @@
.env
__pycache__/
*.pyc
.pytest_cache/
.venv/
*.egg-info/
node_modules/
dist/
backend/orchestrator.db
backend/test.db
orchestrator.db
.DS_Store

21
README.md Normal file
View File

@@ -0,0 +1,21 @@
# AI Orchestrator Wedding Planning Demo
Production-quality demo of an event-driven AI orchestrator between a live transcription stream and an application's REST API. It is intentionally not a chatbot: transcript chunks become validated pending actions, and human approval generates exact API calls without sending them.
```bash
docker compose up --build
```
Open http://localhost:5173 for the dashboard and http://localhost:9000/docs for the API.
The planner runs in hybrid mode. If `ANTHROPIC_API_KEY` is present, it uses Claude with structured tool output. Without a key, it uses deterministic intent extraction so the demo and tests remain offline-friendly.
Approved actions generate calls for the real Wed backend API without executing them. Configure:
```bash
export WED_API_BASE_URL="http://host.docker.internal:8000"
export WED_DEFAULT_PARTNER_ID="partner_uuid_for_guest_reservation_requests"
export SONIOX_API_KEY="your_soniox_api_key"
```
Use **Start Mic** in the dashboard to record microphone audio. When you stop recording, the backend transcribes it with Soniox and feeds the completed transcript segments into the AI orchestrator.

11
backend/README.md Normal file
View File

@@ -0,0 +1,11 @@
# Backend
FastAPI service containing the REST demo backend, WebSocket streaming gateway, generated tool registry, planner, policy engine, validation layer, executor, memory adapters, and event publisher.
Run locally:
```bash
uvicorn app.main:app --reload
```
The planner uses Anthropic Claude when `ANTHROPIC_API_KEY` is configured and otherwise falls back to deterministic intent extraction. The default model is `claude-sonnet-4-6`.

36
backend/alembic.ini Normal file
View File

@@ -0,0 +1,36 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = sqlite:///./orchestrator.db
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s

35
backend/alembic/env.py Normal file
View File

@@ -0,0 +1,35 @@
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
from app.config import get_settings
from app.infrastructure.database import Base
config = context.config
config.set_main_option("sqlalchemy.url", get_settings().database_url)
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
context.configure(url=get_settings().database_url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
connectable = engine_from_config(config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@@ -0,0 +1,70 @@
"""initial schema
Revision ID: 0001_initial
Revises:
Create Date: 2026-06-26 00:00:00.000000
"""
from alembic import op
import sqlalchemy as sa
revision = "0001_initial"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"tasks",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("title", sa.String(length=240), nullable=False),
sa.Column("assignee", sa.String(length=120), nullable=True),
sa.Column("due_date", sa.DateTime(), nullable=True),
sa.Column("status", sa.String(length=40), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_table(
"notes",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("content", sa.String(length=2000), nullable=False),
sa.Column("category", sa.String(length=40), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_table(
"reservations",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("vendor", sa.String(length=180), nullable=False),
sa.Column("reservation_date", sa.DateTime(), nullable=False),
sa.Column("status", sa.String(length=40), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_table(
"pending_actions",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("meeting_id", sa.String(length=120), nullable=False),
sa.Column("tool", sa.String(length=120), nullable=False),
sa.Column("intent", sa.String(length=120), nullable=False),
sa.Column("reasoning", sa.String(length=1000), nullable=False),
sa.Column("confidence", sa.Float(), nullable=False),
sa.Column("payload", sa.JSON(), nullable=False),
sa.Column("extracted_entities", sa.JSON(), nullable=False),
sa.Column("validation", sa.JSON(), nullable=False),
sa.Column("policy", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=40), nullable=False),
sa.Column("execution_result", sa.JSON(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_index("ix_pending_actions_meeting_id", "pending_actions", ["meeting_id"])
def downgrade() -> None:
op.drop_index("ix_pending_actions_meeting_id", table_name="pending_actions")
op.drop_table("pending_actions")
op.drop_table("reservations")
op.drop_table("notes")
op.drop_table("tasks")

1
backend/app/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Wedding planning AI orchestrator backend."""

3
backend/app/ai/README.md Normal file
View File

@@ -0,0 +1,3 @@
# AI
OpenAPI tool generation and registry code. REST operations are converted into planner-visible tool definitions so tools are not hand-authored.

View File

@@ -0,0 +1 @@
"""AI planner and tool generation modules."""

View File

@@ -0,0 +1,71 @@
import re
from typing import Any
from app.domain.schemas import ToolDefinition
def operation_name(method: str, path: str) -> str:
clean = path.strip("/").replace("{", "").replace("}", "")
parts = [part for part in re.split(r"[/_-]+", clean) if part and part != "id"]
resource = "_".join(parts)
verb = {"POST": "create", "PATCH": "update", "PUT": "replace", "GET": "get", "DELETE": "delete"}[method.upper()]
return f"{verb}_{resource}".rstrip("_")
def path_params(path: str) -> list[str]:
return re.findall(r"{([^}]+)}", path)
class OpenAPIToolGenerator:
"""Generates callable tool metadata from OpenAPI operations."""
def __init__(self, spec: dict[str, Any]) -> None:
self.spec = spec
def generate(self) -> list[ToolDefinition]:
tools: list[ToolDefinition] = []
for path, methods in self.spec.get("paths", {}).items():
for method, operation in methods.items():
upper = method.upper()
if upper not in {"POST", "PATCH", "PUT", "DELETE", "GET"}:
continue
request_schema = self._request_schema(operation)
params = path_params(path)
if not request_schema and upper in {"POST", "PATCH", "PUT"}:
continue
schema = request_schema or {"type": "object", "properties": {}}
if params:
schema = {
**schema,
"properties": {
**schema.get("properties", {}),
**{param: {"type": "string"} for param in params},
},
"required": list(dict.fromkeys([*schema.get("required", []), *params])),
}
tools.append(
ToolDefinition(
name=operation.get("operationId") or operation_name(upper, path),
description=operation.get("summary") or operation.get("description") or f"{upper} {path}",
method=upper,
path=path,
schema=schema,
required=schema.get("required", []),
path_params=params,
)
)
return tools
def _request_schema(self, operation: dict[str, Any]) -> dict[str, Any] | None:
body = operation.get("requestBody", {}).get("content", {}).get("application/json", {})
schema = body.get("schema")
if not schema:
return None
return self._resolve(schema)
def _resolve(self, schema: dict[str, Any]) -> dict[str, Any]:
ref = schema.get("$ref")
if not ref:
return schema
name = ref.split("/")[-1]
return self.spec.get("components", {}).get("schemas", {}).get(name, {})

View File

@@ -0,0 +1,15 @@
from app.domain.schemas import ToolDefinition
class ToolRegistry:
def __init__(self, tools: list[ToolDefinition]) -> None:
self._tools = {tool.name: tool for tool in tools}
def list(self) -> list[ToolDefinition]:
return list(self._tools.values())
def get(self, name: str) -> ToolDefinition | None:
return self._tools.get(name)
def names(self) -> set[str]:
return set(self._tools)

View File

@@ -0,0 +1,168 @@
"""Wed backend OpenAPI subset used to generate orchestration tools.
The source backend lives at `/Volumes/Corsair/Wed/wed-backend` and exposes these
FastAPI routes:
- `POST /tasks/`
- `PATCH /tasks/{task_id}`
- `POST /notes/`
- `PATCH /notes/{note_id}`
- `POST /reservations/guest`
- `PATCH /reservations/{reservation_id}`
"""
from typing import Any
def wed_openapi_subset() -> dict[str, Any]:
"""Return a minimal OpenAPI document matching the real Wed backend payloads."""
return {
"openapi": "3.1.0",
"info": {"title": "Wedding Plan API", "version": "1.0"},
"paths": {
"/tasks/": {
"post": {
"operationId": "create_task",
"summary": "Create task in Wed backend",
"requestBody": {
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/TaskCreate"}}}
},
}
},
"/tasks/{task_id}": {
"patch": {
"operationId": "update_task",
"summary": "Update task in Wed backend",
"requestBody": {
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/UpdateTask"}}}
},
}
},
"/notes/": {
"post": {
"operationId": "create_note",
"summary": "Create note in Wed backend",
"requestBody": {
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/NoteCreateSchema"}}}
},
}
},
"/notes/{note_id}": {
"patch": {
"operationId": "update_note",
"summary": "Update note in Wed backend",
"requestBody": {
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/NoteUpdateSchema"}}}
},
}
},
"/reservations/guest": {
"post": {
"operationId": "create_guest_reservation",
"summary": "Create guest reservation request in Wed backend",
"requestBody": {
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/ReservationPendingCreateGuestSchema"}
}
}
},
}
},
"/reservations/{reservation_id}": {
"patch": {
"operationId": "update_reservation",
"summary": "Update reservation in Wed backend",
"requestBody": {
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/ReservationsUpdateSchema"}}
}
},
}
},
},
"components": {
"schemas": {
"TaskCreate": {
"type": "object",
"required": ["title"],
"additionalProperties": False,
"properties": {
"title": {"type": "string"},
"description": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None},
"notes": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None},
"is_completed": {"type": "boolean", "default": False},
"due_date": {
"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}],
"default": None,
},
},
},
"UpdateTask": {
"type": "object",
"additionalProperties": False,
"properties": {
"title": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"description": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"notes": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"is_completed": {"anyOf": [{"type": "boolean"}, {"type": "null"}]},
"due_date": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]},
},
},
"NoteCreateSchema": {
"type": "object",
"required": ["content"],
"additionalProperties": False,
"properties": {
"content": {"type": "string"},
"reservation_id": {
"anyOf": [{"type": "string", "format": "uuid"}, {"type": "null"}],
"default": None,
},
},
},
"NoteUpdateSchema": {
"type": "object",
"additionalProperties": False,
"properties": {
"content": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"reservation_id": {"anyOf": [{"type": "string", "format": "uuid"}, {"type": "null"}]},
},
},
"ReservationPendingCreateGuestSchema": {
"type": "object",
"required": ["partner_id", "guest_first_name", "guest_last_name", "guest_email"],
"additionalProperties": False,
"properties": {
"partner_id": {"type": "string", "format": "uuid"},
"guest_first_name": {"type": "string"},
"guest_last_name": {"type": "string"},
"guest_email": {"type": "string", "format": "email"},
"guest_phone": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None},
"event_date": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]},
"details": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"budget_per_reservation": {"anyOf": [{"type": "number"}, {"type": "null"}]},
"interested_dates": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"guest_count": {"anyOf": [{"type": "integer"}, {"type": "null"}]},
"event_type": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"other_comments": {"anyOf": [{"type": "string"}, {"type": "null"}]},
},
},
"ReservationsUpdateSchema": {
"type": "object",
"additionalProperties": False,
"properties": {
"status": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"event_date": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]},
"details": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"budget_per_reservation": {"anyOf": [{"type": "number"}, {"type": "null"}]},
"interested_dates": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"guest_count": {"anyOf": [{"type": "integer"}, {"type": "null"}]},
"event_type": {"anyOf": [{"type": "string"}, {"type": "null"}]},
"other_comments": {"anyOf": [{"type": "string"}, {"type": "null"}]},
},
},
}
},
}

View File

@@ -0,0 +1,3 @@
# API
FastAPI routes for transcript ingestion, pending review workflow, generated tools, dashboard state, and the demo REST backend.

View File

@@ -0,0 +1 @@
"""FastAPI routers."""

View File

@@ -0,0 +1,73 @@
from functools import lru_cache
import httpx
from fastapi import Request
from app.ai.tool_generator import OpenAPIToolGenerator
from app.ai.tool_registry import ToolRegistry
from app.ai.wed_api_spec import wed_openapi_subset
from app.application.orchestrator import Orchestrator
from app.config import get_settings
from app.infrastructure.events import EventPublisher
from app.memory.context_builder import ContextBuilder
from app.memory.memory_manager import MemoryManager
from app.planner.llm_planner import HybridPlanner
@lru_cache
def context_builder() -> ContextBuilder:
return ContextBuilder()
@lru_cache
def memory_manager() -> MemoryManager:
manager = MemoryManager()
manager.seed_demo()
return manager
@lru_cache
def event_publisher() -> EventPublisher:
return EventPublisher()
@lru_cache
def wed_api_spec() -> dict:
settings = get_settings()
if settings.wed_api_openapi_url:
try:
response = httpx.get(settings.wed_api_openapi_url, timeout=5)
response.raise_for_status()
return response.json()
except Exception:
return wed_openapi_subset()
return wed_openapi_subset()
def tool_registry(request: Request) -> ToolRegistry:
spec = wed_api_spec()
tools = OpenAPIToolGenerator(spec).generate()
filtered = [
tool
for tool in tools
if tool.name
in {
"create_task",
"update_task",
"create_note",
"update_note",
"create_guest_reservation",
"update_reservation",
}
]
return ToolRegistry(filtered)
def orchestrator(request: Request) -> Orchestrator:
return Orchestrator(
context=context_builder(),
memory=memory_manager(),
planner=HybridPlanner(),
registry=tool_registry(request),
publisher=event_publisher(),
)

203
backend/app/api/routes.py Normal file
View File

@@ -0,0 +1,203 @@
from datetime import datetime
from typing import Annotated
from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Request, UploadFile, WebSocket, WebSocketDisconnect
from sqlalchemy.orm import Session
from app.api.dependencies import context_builder, event_publisher, memory_manager, orchestrator, tool_registry
from app.domain.schemas import (
ActionStatus,
DashboardRead,
EventType,
NoteCreate,
NoteRead,
PendingActionRead,
PendingActionUpdate,
ReservationCreate,
ReservationPatch,
ReservationRead,
TaskCreate,
TaskPatch,
TaskRead,
TranscriptChunk,
TranscriptionJobRead,
)
from app.executor.action_executor import ActionExecutor
from app.infrastructure.database import Note, PendingAction, Reservation, Task, get_session
from app.infrastructure.soniox import SonioxTranscriptionService, jobs
router = APIRouter()
DbSession = Annotated[Session, Depends(get_session)]
@router.post("/tasks", response_model=TaskRead, operation_id="create_task", summary="Create task")
def create_task(payload: TaskCreate, session: DbSession) -> Task:
task = Task(**payload.model_dump())
session.add(task)
session.commit()
session.refresh(task)
return task
@router.patch("/tasks/{task_id}", response_model=TaskRead, operation_id="update_task", summary="Update task")
def update_task(task_id: int, payload: TaskPatch, session: DbSession) -> Task:
task = session.get(Task, task_id)
if not task:
raise HTTPException(status_code=404, detail="Task not found")
for key, value in payload.model_dump(exclude_unset=True).items():
setattr(task, key, value)
session.commit()
session.refresh(task)
return task
@router.post("/notes", response_model=NoteRead, operation_id="create_note", summary="Create note")
def create_note(payload: NoteCreate, session: DbSession) -> Note:
note = Note(**payload.model_dump())
session.add(note)
session.commit()
session.refresh(note)
return note
@router.post("/reservations", response_model=ReservationRead, operation_id="create_reservation", summary="Create reservation")
def create_reservation(payload: ReservationCreate, session: DbSession) -> Reservation:
reservation = Reservation(**payload.model_dump())
session.add(reservation)
session.commit()
session.refresh(reservation)
return reservation
@router.patch(
"/reservations/{reservation_id}",
response_model=ReservationRead,
operation_id="update_reservation",
summary="Update reservation",
)
def update_reservation(reservation_id: int, payload: ReservationPatch, session: DbSession) -> Reservation:
reservation = session.get(Reservation, reservation_id)
if not reservation:
raise HTTPException(status_code=404, detail="Reservation not found")
for key, value in payload.model_dump(exclude_unset=True).items():
setattr(reservation, key, value)
session.commit()
session.refresh(reservation)
return reservation
@router.get("/pending-actions", response_model=list[PendingActionRead])
def list_pending_actions(session: DbSession) -> list[PendingAction]:
return session.query(PendingAction).order_by(PendingAction.created_at.desc()).all()
@router.patch("/pending-actions/{action_id}", response_model=PendingActionRead)
def edit_pending_action(action_id: int, payload: PendingActionUpdate, session: DbSession) -> PendingAction:
action = session.get(PendingAction, action_id)
if not action:
raise HTTPException(status_code=404, detail="Pending action not found")
if payload.payload is not None:
action.payload = payload.payload
session.commit()
session.refresh(action)
return action
@router.post("/pending-actions/{action_id}/approve", response_model=PendingActionRead)
async def approve_action(action_id: int, session: DbSession, request: Request) -> PendingAction:
action = session.get(PendingAction, action_id)
if not action:
raise HTTPException(status_code=404, detail="Pending action not found")
action.status = ActionStatus.APPROVED
session.commit()
session.refresh(action)
generated = await ActionExecutor(tool_registry(request)).execute(session, action)
event_publisher().publish(
EventType.API_CALL_GENERATED,
f"approve-{action_id}",
{"action_id": action_id, "status": generated.status, "api_call": generated.execution_result},
)
return generated
@router.post("/pending-actions/{action_id}/reject", response_model=PendingActionRead)
def reject_action(action_id: int, session: DbSession) -> PendingAction:
action = session.get(PendingAction, action_id)
if not action:
raise HTTPException(status_code=404, detail="Pending action not found")
action.status = ActionStatus.REJECTED
session.commit()
session.refresh(action)
event_publisher().publish(EventType.ACTION_REJECTED, f"reject-{action_id}", {"action_id": action_id})
return action
@router.get("/dashboard", response_model=DashboardRead)
def dashboard(session: DbSession):
return DashboardRead(
transcript=context_builder().transcript(),
context=context_builder().snapshot(
{
"tasks": session.query(Task).count(),
"notes": session.query(Note).count(),
"reservations": session.query(Reservation).count(),
}
),
memories=memory_manager().retrieve("wedding venue flowers tasks", limit=6),
pending_actions=session.query(PendingAction).order_by(PendingAction.created_at.desc()).all(),
tasks=session.query(Task).order_by(Task.created_at.desc()).all(),
notes=session.query(Note).order_by(Note.created_at.desc()).all(),
reservations=session.query(Reservation).order_by(Reservation.created_at.desc()).all(),
events=event_publisher().recent(),
)
@router.get("/tools")
def tools(request: Request):
return [tool.model_dump() for tool in tool_registry(request).list()]
@router.post("/transcriptions", response_model=TranscriptionJobRead, status_code=202)
async def create_transcription_job(
request: Request,
background_tasks: BackgroundTasks,
audio: UploadFile = File(...),
) -> TranscriptionJobRead:
payload = await audio.read()
if not payload:
raise HTTPException(status_code=400, detail="No audio file provided.")
job = jobs.create()
background_tasks.add_task(
SonioxTranscriptionService().process_upload,
job.job_id,
payload,
audio.filename or "recording.webm",
request,
)
return job
@router.get("/transcriptions/{job_id}", response_model=TranscriptionJobRead)
def get_transcription_job(job_id: str) -> TranscriptionJobRead:
job = jobs.get(job_id)
if not job:
raise HTTPException(status_code=404, detail="Transcription job not found.")
return job
@router.websocket("/ws/transcript")
async def transcript_socket(websocket: WebSocket):
await websocket.accept()
db = next(get_session())
try:
while True:
raw = await websocket.receive_json()
if "timestamp" not in raw:
raw["timestamp"] = datetime.utcnow().isoformat()
chunk = TranscriptChunk.model_validate(raw)
action = await orchestrator(websocket).ingest(db, chunk) # type: ignore[arg-type]
await websocket.send_json({"ok": True, "pending_action_id": action.id if action else None})
except WebSocketDisconnect:
return
finally:
db.close()

View File

@@ -0,0 +1,3 @@
# Application
Application services coordinate transcript events, context, memory, planning, validation, policy checks, persistence, and event publication.

View File

@@ -0,0 +1 @@
"""Application services coordinating domain and infrastructure."""

View File

@@ -0,0 +1,72 @@
from uuid import uuid4
from sqlalchemy.orm import Session
from app.ai.tool_registry import ToolRegistry
from app.domain.schemas import EventType, PendingActionCreate, TranscriptChunk
from app.infrastructure.database import PendingAction
from app.infrastructure.events import EventPublisher
from app.memory.context_builder import ContextBuilder
from app.memory.memory_manager import MemoryManager
from app.planner.llm_planner import HybridPlanner
from app.validation.action_validator import ActionValidator
from app.validation.policy_engine import PolicyEngine
class Orchestrator:
"""Turns transcript events into pending, validated review actions."""
def __init__(
self,
context: ContextBuilder,
memory: MemoryManager,
planner: HybridPlanner,
registry: ToolRegistry,
publisher: EventPublisher,
) -> None:
self.context = context
self.memory = memory
self.planner = planner
self.registry = registry
self.publisher = publisher
async def ingest(self, session: Session, chunk: TranscriptChunk) -> PendingAction | None:
request_id = str(uuid4())
self.context.add_chunk(chunk)
self.publisher.publish(EventType.CONVERSATION_EVENT, request_id, chunk.model_dump(mode="json"))
memories = self.memory.retrieve(chunk.text)
decision = await self.planner.plan(chunk, self.context.snapshot(), memories, self.registry.list())
self.context.add_decision(decision.model_dump())
validation = ActionValidator(self.registry).validate(decision)
existing = [row.payload for row in session.query(PendingAction).filter(PendingAction.status == "pending_review").all()]
policy = PolicyEngine(existing).evaluate(decision)
if not validation.valid or not policy.allowed:
self.publisher.publish(
EventType.ACTION_REJECTED,
request_id,
{"decision": decision.model_dump(), "validation": validation.model_dump(), "policy": policy.model_dump()},
)
return None
action_in = PendingActionCreate(
meeting_id="demo-meeting",
tool=decision.tool or "",
intent=decision.intent,
reasoning=decision.reasoning,
confidence=decision.confidence,
payload=decision.arguments,
extracted_entities=decision.extracted_entities,
validation=validation,
policy=policy,
)
row = action_in.model_dump(mode="json")
row["validation"] = validation.model_dump()
row["policy"] = policy.model_dump()
action = PendingAction(**row, status="pending_review")
session.add(action)
session.commit()
session.refresh(action)
self.memory.upsert("pending_action", f"{action.intent}: {action.payload}", {"action_id": action.id})
self.publisher.publish(EventType.ACTION_CREATED, request_id, {"action_id": action.id, "decision": decision.model_dump()})
self.publisher.publish(EventType.MEMORY_UPDATED, request_id, {"action_id": action.id, "type": "pending_action"})
return action

38
backend/app/config.py Normal file
View File

@@ -0,0 +1,38 @@
from functools import lru_cache
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""Runtime configuration shared by API, workers, and simulator."""
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
app_name: str = "AI Orchestrator Wedding Demo"
environment: str = "local"
database_url: str = "sqlite:///./orchestrator.db"
redis_url: str = "redis://redis:6379/0"
qdrant_url: str = "http://qdrant:6333"
anthropic_api_key: str | None = None
anthropic_model: str = "claude-sonnet-4-6"
wed_api_base_url: str = "http://localhost:8000"
wed_api_openapi_url: str | None = None
wed_api_bearer_token: str | None = None
wed_default_partner_id: str | None = None
wed_guest_first_name: str = "Demo"
wed_guest_last_name: str = "Customer"
wed_guest_email: str = "demo.customer@example.com"
wed_guest_phone: str | None = None
soniox_api_key: str | None = None
soniox_model: str = "stt-async-preview"
soniox_poll_interval_seconds: float = 3.0
soniox_poll_attempts: int = 240
planner_confidence_threshold: float = 0.72
backend_base_url: str = "http://localhost:9000"
cors_origins: list[str] = Field(default_factory=lambda: ["http://localhost:5173", "http://127.0.0.1:5173"])
@lru_cache
def get_settings() -> Settings:
return Settings()

View File

@@ -0,0 +1,3 @@
# Domain
Pydantic models and lifecycle enums shared across planner, validation, executor, API, and frontend payloads.

View File

@@ -0,0 +1 @@
"""Domain models and business concepts."""

View File

@@ -0,0 +1,178 @@
from datetime import datetime
from enum import StrEnum
from typing import Any, Literal
from uuid import UUID, uuid4
from pydantic import BaseModel, ConfigDict, Field
class ActionStatus(StrEnum):
CREATED = "created"
PENDING_REVIEW = "pending_review"
APPROVED = "approved"
REJECTED = "rejected"
EXECUTED = "executed"
EXECUTION_FAILED = "execution_failed"
class EventType(StrEnum):
CONVERSATION_EVENT = "ConversationEvent"
ACTION_CREATED = "ActionCreated"
ACTION_REJECTED = "ActionRejected"
ACTION_EXECUTED = "ActionExecuted"
API_CALL_GENERATED = "ApiCallGenerated"
MEMORY_UPDATED = "MemoryUpdated"
CONVERSATION_SUMMARIZED = "ConversationSummarized"
class TranscriptChunk(BaseModel):
speaker: str
timestamp: datetime
text: str = Field(min_length=1)
class ConversationEvent(BaseModel):
id: UUID = Field(default_factory=uuid4)
request_id: str
meeting_id: str = "demo-meeting"
chunk: TranscriptChunk
class ToolDefinition(BaseModel):
name: str
description: str
method: Literal["GET", "POST", "PATCH", "PUT", "DELETE"]
path: str
schema: dict[str, Any]
required: list[str] = Field(default_factory=list)
path_params: list[str] = Field(default_factory=list)
class PlannerDecision(BaseModel):
intent: str
confidence: float = Field(ge=0, le=1)
reasoning: str
tool: str | None = None
arguments: dict[str, Any] = Field(default_factory=dict)
extracted_entities: dict[str, Any] = Field(default_factory=dict)
class ValidationResult(BaseModel):
valid: bool
errors: list[str] = Field(default_factory=list)
class PolicyResult(BaseModel):
allowed: bool
reasons: list[str] = Field(default_factory=list)
class PendingActionCreate(BaseModel):
meeting_id: str
tool: str
intent: str
reasoning: str
confidence: float
payload: dict[str, Any]
extracted_entities: dict[str, Any] = Field(default_factory=dict)
validation: ValidationResult
policy: PolicyResult
class PendingActionRead(PendingActionCreate):
model_config = ConfigDict(from_attributes=True)
id: int
status: ActionStatus
created_at: datetime
updated_at: datetime
execution_result: dict[str, Any] | None = None
class PendingActionUpdate(BaseModel):
payload: dict[str, Any] | None = None
class TaskCreate(BaseModel):
title: str
assignee: str | None = None
due_date: datetime | None = None
status: Literal["pending", "in_progress", "done"] = "pending"
class TaskPatch(BaseModel):
title: str | None = None
assignee: str | None = None
due_date: datetime | None = None
status: Literal["pending", "in_progress", "done"] | None = None
class TaskRead(TaskCreate):
model_config = ConfigDict(from_attributes=True)
id: int
created_at: datetime
class NoteCreate(BaseModel):
content: str
category: Literal["general", "preference", "vendor", "budget"] = "general"
class NoteRead(NoteCreate):
model_config = ConfigDict(from_attributes=True)
id: int
created_at: datetime
class ReservationCreate(BaseModel):
vendor: str
reservation_date: datetime
status: Literal["pending", "reserved", "cancelled"] = "pending"
class ReservationPatch(BaseModel):
vendor: str | None = None
reservation_date: datetime | None = None
status: Literal["pending", "reserved", "cancelled"] | None = None
class ReservationRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
vendor: str
reservation_date: datetime
status: str
created_at: datetime
class DashboardRead(BaseModel):
transcript: list[TranscriptChunk]
context: dict[str, Any]
memories: list[dict[str, Any]]
pending_actions: list[PendingActionRead]
tasks: list[TaskRead]
notes: list[NoteRead]
reservations: list[ReservationRead]
events: list[dict[str, Any]]
class TranscriptSegment(BaseModel):
id: str
speaker: str
original_text: str
edited_text: str
start_ms: int
end_ms: int
class TranscriptionJobRead(BaseModel):
job_id: str
status: Literal["uploading", "queued", "processing", "completed", "error"]
transcription_id: str | None = None
segments: list[TranscriptSegment] = Field(default_factory=list)
generated_action_ids: list[int] = Field(default_factory=list)
error: str | None = None
created_at: datetime

View File

@@ -0,0 +1,3 @@
# Executor
Generates exact HTTP call descriptions for approved pending actions. It resolves generated tools, path parameters, headers, URL, JSON body, and cURL, but never sends the request to the Wed backend.

View File

@@ -0,0 +1 @@
"""Action execution services."""

View File

@@ -0,0 +1,57 @@
from typing import Any
from sqlalchemy.orm import Session
from app.ai.tool_registry import ToolRegistry
from app.config import get_settings
from app.domain.schemas import ActionStatus
from app.infrastructure.database import PendingAction
class ActionExecutor:
"""Generates approved API calls without sending them to the Wed backend."""
def __init__(self, registry: ToolRegistry) -> None:
self.registry = registry
self.settings = get_settings()
self.base_url = self.settings.wed_api_base_url.rstrip("/")
async def execute(self, session: Session, action: PendingAction) -> PendingAction:
if action.status != ActionStatus.APPROVED:
raise ValueError("Only approved actions can be converted into API calls.")
tool = self.registry.get(action.tool)
if not tool:
raise ValueError(f"Unknown tool: {action.tool}")
path = tool.path
body = dict(action.payload)
for param in tool.path_params:
value = body.pop(param, None)
if value is None:
raise ValueError(f"Missing path parameter: {param}")
path = path.replace(f"{{{param}}}", str(value))
headers = {}
if self.settings.wed_api_bearer_token:
headers["Authorization"] = "Bearer <configured WED_API_BEARER_TOKEN>"
action.execution_result = {
"generated_only": True,
"method": tool.method,
"url": f"{self.base_url}{path}",
"path": path,
"headers": headers,
"json": body,
"curl": self._curl(tool.method, f"{self.base_url}{path}", headers, body),
}
session.add(action)
session.commit()
session.refresh(action)
return action
def _curl(self, method: str, url: str, headers: dict[str, str], body: dict[str, Any]) -> str:
import json
import shlex
parts = ["curl", "-X", method, url, "-H", "Content-Type: application/json"]
for key, value in headers.items():
parts.extend(["-H", f"{key}: {value}"])
parts.extend(["--data", json.dumps(body, default=str)])
return " ".join(shlex.quote(part) for part in parts)

View File

@@ -0,0 +1,3 @@
# Infrastructure
SQLAlchemy persistence, Redis Stream event publication, and runtime adapters. Local fallbacks keep tests and offline demos usable.

View File

@@ -0,0 +1 @@
"""Infrastructure adapters for persistence, queues, and external services."""

View File

@@ -0,0 +1,80 @@
from collections.abc import Generator
from datetime import datetime
from sqlalchemy import JSON, DateTime, Float, String, create_engine
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
from app.config import get_settings
class Base(DeclarativeBase):
pass
def json_type():
return JSON()
class TimestampMixin:
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class Task(Base, TimestampMixin):
__tablename__ = "tasks"
id: Mapped[int] = mapped_column(primary_key=True)
title: Mapped[str] = mapped_column(String(240))
assignee: Mapped[str | None] = mapped_column(String(120), nullable=True)
due_date: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
status: Mapped[str] = mapped_column(String(40), default="pending")
class Note(Base, TimestampMixin):
__tablename__ = "notes"
id: Mapped[int] = mapped_column(primary_key=True)
content: Mapped[str] = mapped_column(String(2000))
category: Mapped[str] = mapped_column(String(40), default="general")
class Reservation(Base, TimestampMixin):
__tablename__ = "reservations"
id: Mapped[int] = mapped_column(primary_key=True)
vendor: Mapped[str] = mapped_column(String(180))
reservation_date: Mapped[datetime] = mapped_column(DateTime)
status: Mapped[str] = mapped_column(String(40), default="pending")
class PendingAction(Base, TimestampMixin):
__tablename__ = "pending_actions"
id: Mapped[int] = mapped_column(primary_key=True)
meeting_id: Mapped[str] = mapped_column(String(120), index=True)
tool: Mapped[str] = mapped_column(String(120))
intent: Mapped[str] = mapped_column(String(120))
reasoning: Mapped[str] = mapped_column(String(1000))
confidence: Mapped[float] = mapped_column(Float)
payload: Mapped[dict] = mapped_column(json_type())
extracted_entities: Mapped[dict] = mapped_column(json_type(), default=dict)
validation: Mapped[dict] = mapped_column(json_type())
policy: Mapped[dict] = mapped_column(json_type())
status: Mapped[str] = mapped_column(String(40), default="pending_review")
execution_result: Mapped[dict | None] = mapped_column(json_type(), nullable=True)
engine = create_engine(get_settings().database_url, pool_pre_ping=True)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
def init_db() -> None:
Base.metadata.create_all(bind=engine)
def get_session() -> Generator[Session, None, None]:
session = SessionLocal()
try:
yield session
finally:
session.close()

View File

@@ -0,0 +1,44 @@
import json
import logging
from datetime import datetime
from typing import Any
import redis
from app.config import get_settings
from app.domain.schemas import EventType
logger = logging.getLogger(__name__)
class EventPublisher:
"""Publishes orchestration events to Redis Streams with in-memory fallback."""
def __init__(self, redis_url: str | None = None) -> None:
self.stream = "orchestrator.events"
self._fallback: list[dict[str, Any]] = []
self._redis = None
try:
self._redis = redis.Redis.from_url(redis_url or get_settings().redis_url, decode_responses=True)
self._redis.ping()
except Exception as exc: # pragma: no cover - depends on local infra
self._redis = None
logger.warning("Redis unavailable, using in-memory event log: %s", exc)
def publish(self, event_type: EventType, request_id: str, payload: dict[str, Any]) -> dict[str, Any]:
event = {
"type": event_type.value,
"request_id": request_id,
"timestamp": datetime.utcnow().isoformat(),
"payload": payload,
}
if self._redis:
self._redis.xadd(self.stream, {"event": json.dumps(event, default=str)}, maxlen=500, approximate=True)
self._fallback.append(event)
return event
def recent(self, limit: int = 50) -> list[dict[str, Any]]:
if self._redis:
rows = self._redis.xrevrange(self.stream, count=limit)
return [json.loads(fields["event"]) for _, fields in reversed(rows)]
return self._fallback[-limit:]

View File

@@ -0,0 +1,214 @@
import asyncio
import json
from datetime import datetime, timezone
from typing import Any
from uuid import uuid4
import httpx
from sqlalchemy.orm import Session
from app.api.dependencies import orchestrator
from app.config import get_settings
from app.domain.schemas import TranscriptChunk, TranscriptSegment, TranscriptionJobRead
from app.infrastructure.database import SessionLocal
BASE_URL = "https://api.soniox.com/v1"
class SonioxError(RuntimeError):
pass
def _message_from_response(response: httpx.Response, fallback: str) -> str:
try:
body = response.json()
return str(body.get("message") or body.get("error_message") or fallback)
except Exception:
return fallback
def _token_from_json(raw: dict[str, Any]) -> dict[str, Any]:
return {
"text": raw.get("text") or "",
"start_ms": int(raw.get("start_ms") or 0),
"end_ms": int(raw.get("end_ms") or 0),
"confidence": float(raw.get("confidence") or 0),
"speaker": str(raw.get("speaker") or "unknown"),
}
def _ends_sentence(text: str) -> bool:
return text.rstrip().endswith((".", "?", "!"))
def group_by_speaker(tokens: list[dict[str, Any]]) -> list[TranscriptSegment]:
segments: list[TranscriptSegment] = []
active_speaker: str | None = None
buffer = ""
start_ms = 0
end_ms = 0
def flush() -> None:
nonlocal buffer
text = buffer.strip()
if active_speaker is None or not text:
buffer = ""
return
segments.append(
TranscriptSegment(
id=str(uuid4()),
speaker=active_speaker,
original_text=text,
edited_text=text,
start_ms=start_ms,
end_ms=end_ms,
)
)
buffer = ""
for token in tokens:
speaker = token["speaker"] or "unknown"
if active_speaker is not None and speaker != active_speaker:
flush()
if active_speaker != speaker:
active_speaker = speaker
start_ms = token["start_ms"]
buffer += token["text"]
end_ms = token["end_ms"]
if _ends_sentence(token["text"]):
flush()
active_speaker = None
flush()
return segments
class TranscriptionJobs:
"""In-memory Soniox job store for the demo runtime."""
def __init__(self) -> None:
self._jobs: dict[str, TranscriptionJobRead] = {}
def create(self) -> TranscriptionJobRead:
job = TranscriptionJobRead(
job_id=str(uuid4()),
status="uploading",
created_at=datetime.now(timezone.utc),
)
self._jobs[job.job_id] = job
return job
def get(self, job_id: str) -> TranscriptionJobRead | None:
return self._jobs.get(job_id)
def update(self, job_id: str, **changes: Any) -> TranscriptionJobRead:
job = self._jobs[job_id].model_copy(update=changes)
self._jobs[job_id] = job
return job
jobs = TranscriptionJobs()
class SonioxTranscriptionService:
def __init__(self) -> None:
self.settings = get_settings()
async def process_upload(self, job_id: str, audio: bytes, filename: str, request_like: Any) -> None:
try:
if not self.settings.soniox_api_key:
raise SonioxError("SONIOX_API_KEY is not configured.")
file_id = await self._upload_file(audio, filename)
jobs.update(job_id, status="queued")
transcription_id = await self._create_transcription(file_id)
jobs.update(job_id, status="processing", transcription_id=transcription_id)
await self._wait_until_complete(transcription_id, job_id)
segments = await self._fetch_segments(transcription_id)
action_ids = await self._ingest_segments(segments, request_like)
jobs.update(job_id, status="completed", segments=segments, generated_action_ids=action_ids)
except Exception as exc:
jobs.update(job_id, status="error", error=str(exc))
async def _upload_file(self, audio: bytes, filename: str) -> str:
async with httpx.AsyncClient(timeout=60) as client:
response = await client.post(
f"{BASE_URL}/files",
headers={"Authorization": f"Bearer {self.settings.soniox_api_key}"},
files={"file": (filename, audio, "application/octet-stream")},
)
if not response.is_success:
raise SonioxError(_message_from_response(response, "Soniox file upload failed."))
data = response.json()
file_id = str(data.get("id") or data.get("file_id") or "")
if not file_id:
raise SonioxError("Soniox did not return a file id.")
return file_id
async def _create_transcription(self, file_id: str) -> str:
async with httpx.AsyncClient(timeout=30) as client:
response = await client.post(
f"{BASE_URL}/transcriptions",
headers={
"Authorization": f"Bearer {self.settings.soniox_api_key}",
"Content-Type": "application/json",
},
content=json.dumps(
{
"model": self.settings.soniox_model,
"file_id": file_id,
"enable_speaker_diarization": True,
}
),
)
if response.status_code != 201:
raise SonioxError(_message_from_response(response, "Soniox transcription creation failed."))
return str(response.json()["id"])
async def _wait_until_complete(self, transcription_id: str, job_id: str) -> None:
async with httpx.AsyncClient(timeout=30) as client:
for _ in range(self.settings.soniox_poll_attempts):
response = await client.get(
f"{BASE_URL}/transcriptions/{transcription_id}",
headers={"Authorization": f"Bearer {self.settings.soniox_api_key}"},
)
if response.status_code != 200:
raise SonioxError(_message_from_response(response, "Unable to read Soniox status."))
data = response.json()
status = data.get("status") or "processing"
if status == "completed":
return
if status == "error":
raise SonioxError(str(data.get("error_message") or "Soniox transcription failed."))
jobs.update(job_id, status="processing")
await asyncio.sleep(self.settings.soniox_poll_interval_seconds)
raise SonioxError("Timed out waiting for Soniox transcription.")
async def _fetch_segments(self, transcription_id: str) -> list[TranscriptSegment]:
async with httpx.AsyncClient(timeout=30) as client:
response = await client.get(
f"{BASE_URL}/transcriptions/{transcription_id}/transcript",
headers={"Authorization": f"Bearer {self.settings.soniox_api_key}"},
)
if response.status_code != 200:
raise SonioxError(_message_from_response(response, "Unable to download Soniox transcript."))
tokens = [_token_from_json(item) for item in response.json().get("tokens", []) if isinstance(item, dict)]
return group_by_speaker(tokens)
async def _ingest_segments(self, segments: list[TranscriptSegment], request_like: Any) -> list[int]:
session: Session = SessionLocal()
action_ids: list[int] = []
try:
for segment in segments:
action = await orchestrator(request_like).ingest(
session,
TranscriptChunk(
speaker=segment.speaker,
timestamp=datetime.now(timezone.utc),
text=segment.edited_text,
),
)
if action:
action_ids.append(action.id)
finally:
session.close()
return action_ids

53
backend/app/main.py Normal file
View File

@@ -0,0 +1,53 @@
import logging
import time
from uuid import uuid4
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from pythonjsonlogger import jsonlogger
from app.api.routes import router
from app.config import get_settings
from app.infrastructure.database import init_db
def configure_logging() -> None:
handler = logging.StreamHandler()
handler.setFormatter(jsonlogger.JsonFormatter("%(asctime)s %(levelname)s %(name)s %(message)s %(request_id)s %(latency_ms)s"))
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
def create_app() -> FastAPI:
configure_logging()
settings = get_settings()
app = FastAPI(title=settings.app_name, version="0.1.0")
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(router)
@app.on_event("startup")
def startup() -> None:
init_db()
@app.middleware("http")
async def request_logging(request: Request, call_next):
request_id = request.headers.get("x-request-id", str(uuid4()))
start = time.perf_counter()
response = await call_next(request)
latency_ms = round((time.perf_counter() - start) * 1000, 2)
logging.getLogger("app.request").info(
"request complete",
extra={"request_id": request_id, "latency_ms": latency_ms, "path": request.url.path, "status_code": response.status_code},
)
response.headers["x-request-id"] = request_id
return response
return app
app = create_app()

View File

@@ -0,0 +1,3 @@
# Memory
Context builder and semantic memory manager. Redis and Qdrant are represented behind replaceable service boundaries with deterministic local behavior for tests.

View File

@@ -0,0 +1 @@
"""Short- and long-term memory services."""

View File

@@ -0,0 +1,33 @@
from collections import deque
from typing import Any
from app.domain.schemas import TranscriptChunk
class ContextBuilder:
"""Maintains a Redis-compatible meeting context; uses memory for offline tests."""
def __init__(self, window_size: int = 12) -> None:
self.window_size = window_size
self._transcript: deque[TranscriptChunk] = deque(maxlen=window_size)
self._participants: set[str] = set()
self._recent_decisions: deque[dict[str, Any]] = deque(maxlen=10)
def add_chunk(self, chunk: TranscriptChunk) -> None:
self._transcript.append(chunk)
self._participants.add(chunk.speaker)
def add_decision(self, decision: dict[str, Any]) -> None:
self._recent_decisions.append(decision)
def snapshot(self, app_state: dict[str, Any] | None = None) -> dict[str, Any]:
return {
"meeting": {"id": "demo-meeting", "title": "Wedding Planning Review"},
"participants": sorted(self._participants),
"transcript_window": [chunk.model_dump(mode="json") for chunk in self._transcript],
"recent_ai_decisions": list(self._recent_decisions),
"application_state": app_state or {},
}
def transcript(self) -> list[TranscriptChunk]:
return list(self._transcript)

View File

@@ -0,0 +1,47 @@
from hashlib import sha256
from math import sqrt
from typing import Any
def embed_text(text: str, size: int = 32) -> list[float]:
"""Deterministic local embedding used for offline demo and tests."""
buckets = [0.0] * size
for word in text.lower().split():
digest = sha256(word.encode()).digest()
buckets[digest[0] % size] += 1.0
norm = sqrt(sum(v * v for v in buckets)) or 1.0
return [v / norm for v in buckets]
class MemoryManager:
"""Stores semantic memories behind a Qdrant-like interface with local fallback."""
def __init__(self) -> None:
self._items: list[dict[str, Any]] = []
def upsert(self, memory_type: str, text: str, metadata: dict[str, Any] | None = None) -> dict[str, Any]:
item = {
"id": sha256(f"{memory_type}:{text}".encode()).hexdigest()[:16],
"type": memory_type,
"text": text,
"metadata": metadata or {},
"embedding": embed_text(text),
}
self._items = [existing for existing in self._items if existing["id"] != item["id"]]
self._items.append(item)
return item
def retrieve(self, query: str, limit: int = 5) -> list[dict[str, Any]]:
query_vec = embed_text(query)
def score(item: dict[str, Any]) -> float:
return sum(a * b for a, b in zip(query_vec, item["embedding"], strict=True))
ranked = sorted(self._items, key=score, reverse=True)
return [{k: v for k, v in item.items() if k != "embedding"} | {"score": score(item)} for item in ranked[:limit]]
def seed_demo(self) -> None:
self.upsert("customer_preference", "Customer prefers white flowers and a classic minimal arrangement.", {"customer": "Demo"})
self.upsert("vendor", "Aegean Blooms is the preferred florist for white flower arrangements.", {"vendor": "Aegean Blooms"})
self.upsert("meeting", "Previous meeting agreed to compare venues before reserving July dates.", {"meeting_id": "prior-1"})

View File

@@ -0,0 +1,3 @@
# Planner
Hybrid planner implementation. Uses Anthropic Claude tool output when `ANTHROPIC_API_KEY` is configured and deterministic extraction otherwise. The default model is `claude-sonnet-4-6`.

View File

@@ -0,0 +1 @@
"""Planner implementations."""

View File

@@ -0,0 +1,166 @@
import re
import json
from datetime import datetime, timedelta
from typing import Any
from app.config import get_settings
from app.domain.schemas import PlannerDecision, ToolDefinition, TranscriptChunk
class HybridPlanner:
"""Claude planner with deterministic fallback for offline demos."""
def __init__(self) -> None:
self.settings = get_settings()
async def plan(
self,
chunk: TranscriptChunk,
context: dict[str, Any],
memories: list[dict[str, Any]],
tools: list[ToolDefinition],
) -> PlannerDecision:
if self.settings.anthropic_api_key:
try:
return await self._plan_with_claude(chunk, context, memories, tools)
except Exception:
return self._deterministic_plan(chunk)
return self._deterministic_plan(chunk)
async def _plan_with_claude(
self,
chunk: TranscriptChunk,
context: dict[str, Any],
memories: list[dict[str, Any]],
tools: list[ToolDefinition],
) -> PlannerDecision:
from anthropic import AsyncAnthropic
client = AsyncAnthropic(api_key=self.settings.anthropic_api_key)
response = await client.messages.create(
model=self.settings.anthropic_model,
max_tokens=1200,
system=(
"You are an event-driven meeting planner. Produce exactly one structured action plan "
"or no_action. Never execute tools. Do not invent missing required fields."
),
messages=[
{
"role": "user",
"content": json.dumps(
{
"transcript_chunk": chunk.model_dump(mode="json"),
"context": context,
"memories": memories,
"available_tools": [tool.model_dump() for tool in tools],
"business_rules": [
"All actions must start as pending review.",
"Only select tools from available_tools.",
"Do not include parameters that are not in the tool schema.",
],
},
default=str,
),
}
],
tools=[
{
"name": "emit_plan",
"description": "Emit the planner decision for the current transcript chunk.",
"input_schema": PlannerDecision.model_json_schema(),
}
],
tool_choice={"type": "tool", "name": "emit_plan"},
)
for block in response.content:
if getattr(block, "type", None) == "tool_use" and getattr(block, "name", None) == "emit_plan":
return PlannerDecision.model_validate(block.input)
raise ValueError("Claude did not emit the required planner tool output.")
def _deterministic_plan(self, chunk: TranscriptChunk) -> PlannerDecision:
text = chunk.text.strip()
lowered = text.lower()
if "task" in lowered or lowered.startswith("i'll ") or "i will " in lowered:
assignee = self._extract_assignee(text) or chunk.speaker
title = self._task_title(text)
args: dict[str, Any] = {
"title": title,
"description": f"Follow-up captured from meeting transcript. Suggested owner: {assignee}.",
"notes": text,
"is_completed": False,
}
if "tomorrow" in lowered:
args["due_date"] = (chunk.timestamp + timedelta(days=1)).isoformat()
return PlannerDecision(
intent="create_task",
confidence=0.94,
reasoning="The speaker described an explicit follow-up task.",
tool="create_task",
arguments=args,
extracted_entities={"assignee": assignee, "title": title},
)
if "note" in lowered or "prefers" in lowered or "preference" in lowered:
category = "preference" if "prefer" in lowered else "general"
return PlannerDecision(
intent="create_note",
confidence=0.91,
reasoning="The utterance captures information that should be preserved as a note.",
tool="create_note",
arguments={"content": text},
extracted_entities={"category": category},
)
if "reserve" in lowered or "reservation" in lowered or "venue" in lowered:
if not self.settings.wed_default_partner_id:
return PlannerDecision(
intent="no_action",
confidence=0.65,
reasoning="A reservation request was detected, but WED_DEFAULT_PARTNER_ID is required to target the Wed backend.",
extracted_entities={"missing_configuration": "WED_DEFAULT_PARTNER_ID"},
)
date = self._extract_date(text, chunk.timestamp)
event_type = "Wedding"
match = re.search(r"reserve (?:the )?([a-zA-Z ]+?)(?: for| on|$)", text, re.I)
if match:
event_type = match.group(1).strip().title()
return PlannerDecision(
intent="create_guest_reservation",
confidence=0.89,
reasoning="The utterance requests a reservation that requires review before execution.",
tool="create_guest_reservation",
arguments={
"partner_id": self.settings.wed_default_partner_id,
"guest_first_name": self.settings.wed_guest_first_name,
"guest_last_name": self.settings.wed_guest_last_name,
"guest_email": self.settings.wed_guest_email,
"guest_phone": self.settings.wed_guest_phone,
"event_date": date.isoformat(),
"details": text,
"event_type": event_type,
"interested_dates": date.date().isoformat(),
},
extracted_entities={"event_type": event_type, "event_date": date.isoformat()},
)
return PlannerDecision(intent="no_action", confidence=0.55, reasoning="No actionable backend intent was detected.")
def _extract_assignee(self, text: str) -> str | None:
for pattern in [r"for ([A-Z][a-z]+) to", r"([A-Z][a-z]+) to"]:
match = re.search(pattern, text)
if match:
return match.group(1)
return None
def _task_title(self, text: str) -> str:
cleaned = re.sub(r"create a task for [A-Z][a-z]+ to ", "", text, flags=re.I)
cleaned = re.sub(r"^i(?:'ll| will) ", "", cleaned, flags=re.I)
cleaned = cleaned.rstrip(".")
return cleaned[:1].upper() + cleaned[1:]
def _extract_date(self, text: str, fallback: datetime) -> datetime:
lowered = text.lower()
if "tomorrow" in lowered:
return fallback + timedelta(days=1)
match = re.search(r"(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{1,2})", lowered)
if match:
month = datetime.strptime(match.group(1), "%B").month
return datetime(fallback.year, month, int(match.group(2)), 12, 0)
return fallback + timedelta(days=7)

41
backend/app/seed.py Normal file
View File

@@ -0,0 +1,41 @@
from datetime import datetime, timedelta
from app.infrastructure.database import Note, Reservation, SessionLocal, Task, init_db
from app.memory.memory_manager import MemoryManager
def seed() -> None:
init_db()
session = SessionLocal()
try:
if session.query(Task).count() == 0:
session.add_all(
[
Task(title="Review venue shortlist", assignee="Maria", status="pending"),
Task(title="Confirm catering tasting", assignee="Nikos", status="in_progress"),
]
)
if session.query(Note).count() == 0:
session.add_all(
[
Note(content="Customer prefers white flowers.", category="preference"),
Note(content="Budget conversation should stay under 20,000 EUR.", category="budget"),
]
)
if session.query(Reservation).count() == 0:
session.add(
Reservation(
vendor="Aegean Garden Venue",
reservation_date=datetime.utcnow() + timedelta(days=45),
status="pending",
)
)
session.commit()
memory = MemoryManager()
memory.seed_demo()
finally:
session.close()
if __name__ == "__main__":
seed()

25
backend/app/simulator.py Normal file
View File

@@ -0,0 +1,25 @@
import asyncio
import json
from datetime import datetime
import websockets
TRANSCRIPT = [
{"speaker": "Bride", "text": "Add a note that the customer prefers white flowers."},
{"speaker": "Planner", "text": "Create a task for Maria to confirm the photographer."},
{"speaker": "Groom", "text": "We should reserve the venue for July 15."},
{"speaker": "Bride", "text": "I'll call the florist tomorrow."},
]
async def main() -> None:
async with websockets.connect("ws://localhost:9000/ws/transcript") as socket:
for item in TRANSCRIPT:
await socket.send(json.dumps({**item, "timestamp": datetime.utcnow().isoformat()}))
print(await socket.recv())
await asyncio.sleep(1)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -0,0 +1,3 @@
# Validation
Schema validation and policy enforcement for planner decisions before any action enters pending review.

View File

@@ -0,0 +1 @@
"""Validation and policy services."""

View File

@@ -0,0 +1,35 @@
from jsonschema import Draft202012Validator, FormatChecker
from pydantic import ValidationError
from app.domain.schemas import PlannerDecision, ValidationResult
from app.ai.tool_registry import ToolRegistry
class ActionValidator:
def __init__(self, registry: ToolRegistry) -> None:
self.registry = registry
def validate(self, decision: PlannerDecision) -> ValidationResult:
if decision.intent == "no_action":
return ValidationResult(valid=False, errors=["No actionable intent detected."])
if not decision.tool:
return ValidationResult(valid=False, errors=["Planner did not select a tool."])
tool = self.registry.get(decision.tool)
if not tool:
return ValidationResult(valid=False, errors=[f"Unknown tool: {decision.tool}"])
try:
Draft202012Validator(tool.schema, format_checker=FormatChecker()).validate(decision.arguments)
except Exception as exc:
return ValidationResult(valid=False, errors=[str(exc)])
extra = set(decision.arguments) - set(tool.schema.get("properties", {}))
if extra:
return ValidationResult(valid=False, errors=[f"Hallucinated parameters: {sorted(extra)}"])
try:
self._validate_dates(decision.arguments)
except ValidationError as exc:
return ValidationResult(valid=False, errors=[str(exc)])
return ValidationResult(valid=True)
def _validate_dates(self, args: dict) -> None:
# JSON Schema catches shape; FastAPI/Pydantic validates API models at execution time.
return None

View File

@@ -0,0 +1,22 @@
from app.config import get_settings
from app.domain.schemas import PlannerDecision, PolicyResult
class PolicyEngine:
def __init__(self, existing_payloads: list[dict] | None = None) -> None:
self.threshold = get_settings().planner_confidence_threshold
self.existing_payloads = existing_payloads or []
def evaluate(self, decision: PlannerDecision) -> PolicyResult:
reasons: list[str] = []
if decision.confidence < self.threshold:
reasons.append(f"Confidence {decision.confidence:.2f} is below threshold {self.threshold:.2f}.")
if decision.arguments in self.existing_payloads:
reasons.append("Duplicate pending action payload.")
if decision.tool == "create_guest_reservation" and not decision.arguments.get("event_date"):
reasons.append("Reservation requires a date.")
if decision.tool == "create_guest_reservation" and not decision.arguments.get("partner_id"):
reasons.append("Reservation requires a Wed partner_id.")
if decision.tool == "create_task" and not decision.arguments.get("title"):
reasons.append("Task requires a title.")
return PolicyResult(allowed=not reasons, reasons=reasons)

42
backend/pyproject.toml Normal file
View File

@@ -0,0 +1,42 @@
[build-system]
requires = ["setuptools>=72.0"]
build-backend = "setuptools.build_meta"
[project]
name = "ai-orchestrator-wedding-demo"
version = "0.1.0"
description = "Event-driven AI orchestrator demo for wedding planning meetings"
requires-python = ">=3.12"
dependencies = [
"alembic>=1.13.2",
"fastapi>=0.115.0",
"httpx>=0.27.0",
"jsonschema>=4.23.0",
"anthropic>=0.45.0",
"psycopg[binary]>=3.2.1",
"pydantic>=2.8.2",
"pydantic-settings>=2.4.0",
"python-multipart>=0.0.12",
"python-json-logger>=2.0.7",
"qdrant-client>=1.11.1",
"redis>=5.0.8",
"sqlalchemy>=2.0.32",
"uvicorn[standard]>=0.30.6",
"websockets>=13.0.1",
]
[project.optional-dependencies]
dev = [
"pytest>=8.3.2",
"pytest-asyncio>=0.24.0",
"respx>=0.21.1",
]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
pythonpath = ["."]
[tool.setuptools.packages.find]
where = ["."]
include = ["app*"]

View File

@@ -0,0 +1,29 @@
from fastapi.testclient import TestClient
from app.infrastructure.database import PendingAction, SessionLocal, init_db
from app.main import app
def test_websocket_ingestion_creates_pending_action():
init_db()
session = SessionLocal()
session.query(PendingAction).delete()
session.commit()
session.close()
with TestClient(app) as client:
with client.websocket_connect("/ws/transcript") as socket:
socket.send_json(
{
"speaker": "Planner",
"timestamp": "2026-06-26T12:00:00Z",
"text": "Create a task for Maria to confirm the photographer.",
}
)
response = socket.receive_json()
assert response["ok"] is True
assert response["pending_action_id"] is not None
dashboard = client.get("/dashboard").json()
assert any(action["tool"] == "create_task" for action in dashboard["pending_actions"])

View File

@@ -0,0 +1,71 @@
import pytest
from app.ai.tool_registry import ToolRegistry
from app.domain.schemas import ActionStatus
from app.domain.schemas import ToolDefinition
from app.executor.action_executor import ActionExecutor
from app.infrastructure.database import PendingAction
from app.memory.memory_manager import MemoryManager
def test_memory_retrieves_semantic_items():
memory = MemoryManager()
memory.upsert("preference", "Customer prefers white flowers")
results = memory.retrieve("white flower preference")
assert results[0]["type"] == "preference"
@pytest.mark.asyncio
async def test_executor_rejects_unapproved_action():
registry = ToolRegistry(
[
ToolDefinition(
name="create_task",
description="Create task",
method="POST",
path="/tasks",
schema={"type": "object", "properties": {"title": {"type": "string"}}},
)
]
)
action = PendingAction(tool="create_task", status="pending_review", payload={"title": "x"})
with pytest.raises(ValueError, match="Only approved"):
await ActionExecutor(registry).execute(None, action) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_executor_generates_call_without_http_request():
registry = ToolRegistry(
[
ToolDefinition(
name="update_reservation",
description="Update reservation",
method="PATCH",
path="/reservations/{reservation_id}",
path_params=["reservation_id"],
schema={"type": "object", "properties": {"reservation_id": {"type": "string"}, "status": {"type": "string"}}},
)
]
)
action = PendingAction(tool="update_reservation", status=ActionStatus.APPROVED, payload={"reservation_id": "abc", "status": "ACCEPTED"})
class FakeSession:
def add(self, _action):
return None
def commit(self):
return None
def refresh(self, _action):
return None
result = await ActionExecutor(registry).execute(FakeSession(), action) # type: ignore[arg-type]
assert result.status == ActionStatus.APPROVED
assert result.execution_result["generated_only"] is True
assert result.execution_result["method"] == "PATCH"
assert result.execution_result["path"] == "/reservations/abc"
assert result.execution_result["json"] == {"status": "ACCEPTED"}

View File

@@ -0,0 +1,59 @@
from datetime import datetime
import pytest
from app.ai.tool_registry import ToolRegistry
from app.domain.schemas import PlannerDecision, ToolDefinition, TranscriptChunk
from app.planner.llm_planner import HybridPlanner
from app.validation.action_validator import ActionValidator
from app.validation.policy_engine import PolicyEngine
@pytest.mark.asyncio
async def test_fallback_planner_extracts_task():
planner = HybridPlanner()
chunk = TranscriptChunk(speaker="Bride", timestamp=datetime(2026, 6, 26, 12, 0), text="Create a task for Maria to confirm the photographer.")
decision = await planner.plan(chunk, {}, [], [])
assert decision.tool == "create_task"
assert decision.extracted_entities["assignee"] == "Maria"
assert decision.arguments["is_completed"] is False
assert "Suggested owner: Maria" in decision.arguments["description"]
assert "photographer" in decision.arguments["title"].lower()
def test_validator_rejects_hallucinated_parameters():
registry = ToolRegistry(
[
ToolDefinition(
name="create_task",
description="Create task",
method="POST",
path="/tasks",
schema={"type": "object", "required": ["title"], "properties": {"title": {"type": "string"}}},
required=["title"],
)
]
)
decision = PlannerDecision(
intent="create_task",
confidence=0.9,
reasoning="test",
tool="create_task",
arguments={"title": "Confirm photographer", "unexpected": True},
)
result = ActionValidator(registry).validate(decision)
assert not result.valid
assert "Hallucinated" in result.errors[0]
def test_policy_rejects_low_confidence():
decision = PlannerDecision(intent="create_note", confidence=0.2, reasoning="weak", tool="create_note", arguments={"content": "x"})
result = PolicyEngine().evaluate(decision)
assert not result.allowed
assert "below threshold" in result.reasons[0]

View File

@@ -0,0 +1,17 @@
from app.infrastructure.soniox import group_by_speaker
def test_group_by_speaker_flushes_on_sentence_and_speaker_change():
segments = group_by_speaker(
[
{"speaker": "Bride", "text": "Create ", "start_ms": 0, "end_ms": 100},
{"speaker": "Bride", "text": "a task.", "start_ms": 100, "end_ms": 500},
{"speaker": "Planner", "text": "Add ", "start_ms": 600, "end_ms": 700},
{"speaker": "Planner", "text": "a note.", "start_ms": 700, "end_ms": 1000},
]
)
assert len(segments) == 2
assert segments[0].speaker == "Bride"
assert segments[0].edited_text == "Create a task."
assert segments[1].speaker == "Planner"

View File

@@ -0,0 +1,63 @@
from app.ai.tool_generator import OpenAPIToolGenerator
def test_generates_tools_from_openapi_spec():
spec = {
"paths": {
"/tasks": {
"post": {
"operationId": "create_task",
"summary": "Create task",
"requestBody": {
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/TaskCreate"}
}
}
},
}
}
},
"components": {
"schemas": {
"TaskCreate": {
"type": "object",
"required": ["title"],
"properties": {"title": {"type": "string"}},
}
}
},
}
tools = OpenAPIToolGenerator(spec).generate()
assert tools[0].name == "create_task"
assert tools[0].required == ["title"]
assert tools[0].path == "/tasks"
def test_generates_path_params_for_patch_tools():
spec = {
"paths": {
"/reservations/{reservation_id}": {
"patch": {
"operationId": "update_reservation",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {"status": {"type": "string"}},
}
}
}
},
}
}
}
}
tools = OpenAPIToolGenerator(spec).generate()
assert tools[0].path_params == ["reservation_id"]
assert "reservation_id" in tools[0].schema["required"]

79
docker-compose.yml Normal file
View File

@@ -0,0 +1,79 @@
services:
postgres:
image: postgres:16-alpine
environment:
POSTGRES_DB: orchestrator
POSTGRES_USER: orchestrator
POSTGRES_PASSWORD: orchestrator
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U orchestrator -d orchestrator"]
interval: 5s
timeout: 5s
retries: 10
redis:
image: redis:7-alpine
ports:
- "6379:6379"
qdrant:
image: qdrant/qdrant:v1.11.5
ports:
- "6333:6333"
volumes:
- qdrant_data:/qdrant/storage
backend:
build:
context: ./backend
dockerfile: ../docker/backend.Dockerfile
environment:
DATABASE_URL: postgresql+psycopg://orchestrator:orchestrator@postgres:5432/orchestrator
REDIS_URL: redis://redis:6379/0
QDRANT_URL: http://qdrant:6333
BACKEND_BASE_URL: http://backend:8000
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
ANTHROPIC_MODEL: ${ANTHROPIC_MODEL:-claude-sonnet-4-6}
WED_API_BASE_URL: ${WED_API_BASE_URL:-http://host.docker.internal:8000}
WED_API_OPENAPI_URL: ${WED_API_OPENAPI_URL:-}
WED_API_BEARER_TOKEN: ${WED_API_BEARER_TOKEN:-}
WED_DEFAULT_PARTNER_ID: ${WED_DEFAULT_PARTNER_ID:-}
WED_GUEST_FIRST_NAME: ${WED_GUEST_FIRST_NAME:-Demo}
WED_GUEST_LAST_NAME: ${WED_GUEST_LAST_NAME:-Customer}
WED_GUEST_EMAIL: ${WED_GUEST_EMAIL:-demo.customer@example.com}
WED_GUEST_PHONE: ${WED_GUEST_PHONE:-}
SONIOX_API_KEY: ${SONIOX_API_KEY:-}
SONIOX_MODEL: ${SONIOX_MODEL:-stt-async-preview}
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_started
qdrant:
condition: service_started
ports:
- "9000:8000"
command: >
sh -c "alembic upgrade head &&
python -m app.seed &&
uvicorn app.main:app --host 0.0.0.0 --port 8000"
frontend:
build:
context: ./frontend
dockerfile: ../docker/frontend.Dockerfile
environment:
VITE_BACKEND_URL: http://backend:8000
depends_on:
- backend
ports:
- "5173:5173"
command: npm run dev
volumes:
postgres_data:
qdrant_data:

10
docker/backend.Dockerfile Normal file
View File

@@ -0,0 +1,10 @@
FROM python:3.12-slim
WORKDIR /app
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
COPY . .
RUN pip install --no-cache-dir -e ".[dev]"
EXPOSE 8000

View File

@@ -0,0 +1,8 @@
FROM node:22-alpine
WORKDIR /app
COPY package.json ./
RUN npm install
COPY . .
EXPOSE 5173

37
docs/api.md Normal file
View File

@@ -0,0 +1,37 @@
# API
Primary orchestrator/demo endpoints:
- `POST /tasks`
- `PATCH /tasks/{task_id}`
- `POST /notes`
- `POST /reservations`
- `PATCH /reservations/{reservation_id}`
- `GET /dashboard`
- `GET /tools`
- `POST /transcriptions`
- `GET /transcriptions/{job_id}`
- `GET /pending-actions`
- `PATCH /pending-actions/{action_id}`
- `POST /pending-actions/{action_id}/approve` generates the API call; it does not send it
- `POST /pending-actions/{action_id}/reject`
- `WS /ws/transcript`
Generated Wed backend tools:
- `create_task` -> `POST /tasks/`
- `update_task` -> `PATCH /tasks/{task_id}`
- `create_note` -> `POST /notes/`
- `update_note` -> `PATCH /notes/{note_id}`
- `create_guest_reservation` -> `POST /reservations/guest`
- `update_reservation` -> `PATCH /reservations/{reservation_id}`
Transcript chunks:
```json
{
"speaker": "Bride",
"timestamp": "2026-06-26T12:00:00Z",
"text": "Create a task for Maria to confirm the photographer."
}
```

54
docs/architecture.md Normal file
View File

@@ -0,0 +1,54 @@
# Architecture
```mermaid
flowchart LR
Client[React Dashboard] -->|WebSocket transcript chunks| Gateway[Streaming Gateway]
Gateway --> Orchestrator[AI Orchestrator]
Orchestrator --> Context[Context Builder]
Orchestrator --> Memory[Memory Manager]
Orchestrator --> Planner[Hybrid LLM Planner]
Orchestrator --> Registry[Generated Tool Registry]
Orchestrator --> Policy[Policy Engine]
Orchestrator --> Validation[Validation Layer]
Orchestrator --> Publisher[Event Publisher]
Client -->|approve/reject/edit| Review[Pending Review API]
Review --> Generator[API Call Generator]
Generator -->|method + URL + headers + JSON + cURL| GeneratedCall[Generated Wed API Call]
GeneratedCall -. not sent .-> WedAPI[Wed Backend API]
Executor -. dashboard demo state .-> REST[REST Demo Backend]
REST --> Postgres[(PostgreSQL)]
WedAPI --> WedPostgres[(Wed PostgreSQL)]
Context --> Redis[(Redis)]
Publisher --> Streams[(Redis Streams)]
Memory --> Qdrant[(Qdrant)]
```
## Sequence
```mermaid
sequenceDiagram
participant UI as React Dashboard
participant WS as Streaming Gateway
participant O as Orchestrator
participant P as Planner
participant V as Validation
participant PE as Policy Engine
participant DB as PostgreSQL
participant EX as Executor
UI->>WS: transcript chunk
WS->>O: ConversationEvent
O->>P: transcript + context + memories + tools
P-->>O: structured plan
O->>V: validate schema and parameters
O->>PE: evaluate confidence, duplicates, rules
O->>DB: create PendingAction
UI->>DB: approve/edit/reject pending action
UI->>EX: approve / generate call
EX-->>DB: store generated API call
EX-->>UI: generated call state
```
## Design Notes
The planner never executes tools. It produces structured decisions against the Wed backend tool contract, the policy and validation layers gate those decisions, and approval only generates the exact Wed REST call. The app does not send that call to the Wed backend. The deterministic planner keeps demos and tests reliable when no Claude key is configured.

67
docs/setup.md Normal file
View File

@@ -0,0 +1,67 @@
# Setup
## Docker
```bash
docker compose up --build
```
Open:
- Frontend: http://localhost:5173
- Backend docs: http://localhost:9000/docs
The backend runs Alembic migrations and sample data seeding before startup.
## Local Backend
```bash
cd backend
pip install -e ".[dev]"
uvicorn app.main:app --reload
```
## Local Frontend
```bash
cd frontend
npm install
npm run dev
```
Set `ANTHROPIC_API_KEY` to use the Claude planner. Leave it empty to use the deterministic planner.
```bash
export ANTHROPIC_API_KEY="your_claude_key_here"
export ANTHROPIC_MODEL="claude-3-5-sonnet-latest"
docker compose up --build
```
## Real Wed Backend API
The orchestrator generates tools from the Wed backend contract found in:
```text
/Volumes/Corsair/Wed/wed-backend
```
Approved actions generate a Wed API call description, including method, URL, headers, JSON body, and cURL. The orchestrator does not send the request.
```bash
export WED_API_BASE_URL="http://host.docker.internal:8000"
export WED_DEFAULT_PARTNER_ID="partner_uuid_for_reservation_requests"
export SONIOX_API_KEY="your_soniox_api_key"
docker compose up -d --build
```
`WED_DEFAULT_PARTNER_ID` is only required for transcript phrases that create guest reservation requests.
## Microphone Transcription
The dashboard records microphone audio locally with the browser `MediaRecorder` API. Stopping the recording uploads the audio to:
```text
POST /transcriptions
```
The backend sends the audio to Soniox using `SONIOX_API_KEY`, polls until transcription completes, groups speaker-labeled segments, and ingests each segment through the orchestrator. The resulting pending actions still only generate API calls; they do not execute them.

10
frontend/README.md Normal file
View File

@@ -0,0 +1,10 @@
# Frontend
Vite React dashboard for the AI Orchestrator demo. It shows transcript ingestion, AI reasoning, validation, policy decisions, pending review actions, execution logs, retrieved memories, and REST backend state.
Run locally:
```bash
npm install
npm run dev
```

12
frontend/index.html Normal file
View File

@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>AI Orchestrator Wedding Demo</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

1655
frontend/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

24
frontend/package.json Normal file
View File

@@ -0,0 +1,24 @@
{
"name": "ai-orchestrator-wedding-demo-frontend",
"version": "0.1.0",
"private": true,
"type": "module",
"scripts": {
"dev": "vite --host 0.0.0.0",
"build": "tsc && vite build",
"preview": "vite preview --host 0.0.0.0"
},
"dependencies": {
"@vitejs/plugin-react": "^4.3.1",
"lucide-react": "^0.468.0",
"typescript": "^5.6.2",
"vite": "^5.4.8",
"react": "^18.3.1",
"react-dom": "^18.3.1"
},
"devDependencies": {
"@types/node": "^22.7.4",
"@types/react": "^18.3.9",
"@types/react-dom": "^18.3.0"
}
}

321
frontend/src/main.tsx Normal file
View File

@@ -0,0 +1,321 @@
import React from "react";
import ReactDOM from "react-dom/client";
import { Check, Clock, Play, RefreshCw, Send, ShieldCheck, X } from "lucide-react";
import "./styles.css";
type TranscriptChunk = { speaker: string; timestamp: string; text: string };
type PendingAction = {
id: number;
tool: string;
intent: string;
reasoning: string;
confidence: number;
payload: Record<string, unknown>;
extracted_entities: Record<string, unknown>;
validation: { valid: boolean; errors: string[] };
policy: { allowed: boolean; reasons: string[] };
status: string;
execution_result?: unknown;
created_at: string;
};
type Dashboard = {
transcript: TranscriptChunk[];
context: Record<string, unknown>;
memories: Array<Record<string, unknown>>;
pending_actions: PendingAction[];
tasks: Array<Record<string, unknown>>;
notes: Array<Record<string, unknown>>;
reservations: Array<Record<string, unknown>>;
events: Array<Record<string, unknown>>;
};
type TranscriptionJob = {
job_id: string;
status: "uploading" | "queued" | "processing" | "completed" | "error";
transcription_id?: string | null;
segments: Array<{
id: string;
speaker: string;
original_text: string;
edited_text: string;
start_ms: number;
end_ms: number;
}>;
generated_action_ids: number[];
error?: string | null;
};
const API = import.meta.env.VITE_API_URL ?? "";
const WS_URL = import.meta.env.VITE_WS_URL ?? `ws://${window.location.host}/ws/transcript`;
const sampleTranscript = [
{ speaker: "Bride", text: "Add a note that the customer prefers white flowers." },
{ speaker: "Planner", text: "Create a task for Maria to confirm the photographer." },
{ speaker: "Groom", text: "We should reserve the venue for July 15." },
{ speaker: "Bride", text: "I'll call the florist tomorrow." }
];
function JsonBlock({ value }: { value: unknown }) {
return <pre>{JSON.stringify(value, null, 2)}</pre>;
}
function App() {
const [dashboard, setDashboard] = React.useState<Dashboard | null>(null);
const [speaker, setSpeaker] = React.useState("Bride");
const [text, setText] = React.useState("Create a task for Maria to confirm the photographer.");
const [busy, setBusy] = React.useState(false);
const [isRecording, setIsRecording] = React.useState(false);
const [recordingError, setRecordingError] = React.useState<string | null>(null);
const [recordingSeconds, setRecordingSeconds] = React.useState(0);
const [transcriptionJob, setTranscriptionJob] = React.useState<TranscriptionJob | null>(null);
const mediaRecorderRef = React.useRef<MediaRecorder | null>(null);
const mediaStreamRef = React.useRef<MediaStream | null>(null);
const audioChunksRef = React.useRef<Blob[]>([]);
const recordingStartedAtRef = React.useRef<number | null>(null);
const refresh = React.useCallback(async () => {
const response = await fetch(`${API}/dashboard`);
setDashboard(await response.json());
}, []);
React.useEffect(() => {
refresh();
const interval = window.setInterval(refresh, 2500);
return () => window.clearInterval(interval);
}, [refresh]);
React.useEffect(() => {
if (!isRecording) {
return;
}
const timer = window.setInterval(() => {
const startedAt = recordingStartedAtRef.current;
setRecordingSeconds(startedAt ? Math.floor((Date.now() - startedAt) / 1000) : 0);
}, 500);
return () => window.clearInterval(timer);
}, [isRecording]);
React.useEffect(() => {
const jobId = transcriptionJob?.job_id;
if (!jobId || transcriptionJob.status === "completed" || transcriptionJob.status === "error") {
return;
}
const timer = window.setInterval(async () => {
const response = await fetch(`${API}/transcriptions/${jobId}`);
if (!response.ok) {
return;
}
const job = (await response.json()) as TranscriptionJob;
setTranscriptionJob(job);
if (job.status === "completed") {
await refresh();
}
}, 2000);
return () => window.clearInterval(timer);
}, [transcriptionJob, refresh]);
const sendChunk = React.useCallback(async (chunk = { speaker, text }) => {
setBusy(true);
await new Promise<void>((resolve, reject) => {
const ws = new WebSocket(WS_URL);
ws.onopen = () => ws.send(JSON.stringify({ ...chunk, timestamp: new Date().toISOString() }));
ws.onmessage = () => {
ws.close();
resolve();
};
ws.onerror = () => reject(new Error("WebSocket failed"));
}).finally(() => setBusy(false));
await refresh();
}, [speaker, text, refresh]);
const replay = async () => {
for (const chunk of sampleTranscript) {
await sendChunk(chunk);
await new Promise((resolve) => window.setTimeout(resolve, 450));
}
};
const pickMimeType = () => {
if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) return "audio/webm;codecs=opus";
if (MediaRecorder.isTypeSupported("audio/webm")) return "audio/webm";
if (MediaRecorder.isTypeSupported("audio/mp4")) return "audio/mp4";
return "audio/webm";
};
const startMic = async () => {
setRecordingError(null);
setTranscriptionJob(null);
const stream = await navigator.mediaDevices.getUserMedia({
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
});
const recorder = new MediaRecorder(stream, { mimeType: pickMimeType() });
audioChunksRef.current = [];
mediaRecorderRef.current = recorder;
mediaStreamRef.current = stream;
recordingStartedAtRef.current = Date.now();
recorder.ondataavailable = (event) => {
if (event.data.size > 0) audioChunksRef.current.push(event.data);
};
recorder.start(250);
setRecordingSeconds(0);
setIsRecording(true);
};
const stopMic = async () => {
const recorder = mediaRecorderRef.current;
if (!recorder) return;
const blob = await new Promise<Blob | null>((resolve) => {
recorder.onstop = () => {
const chunks = audioChunksRef.current;
resolve(chunks.length > 0 ? new Blob(chunks, { type: recorder.mimeType }) : null);
};
try {
recorder.requestData();
} catch {
// Ignore browsers that have no buffered data yet.
}
recorder.stop();
});
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
mediaRecorderRef.current = null;
mediaStreamRef.current = null;
setIsRecording(false);
recordingStartedAtRef.current = null;
if (!blob) {
setRecordingError("No microphone audio was captured.");
return;
}
const form = new FormData();
const extension = blob.type.includes("mp4") ? "m4a" : "webm";
form.append("audio", blob, `meeting-recording.${extension}`);
const response = await fetch(`${API}/transcriptions`, { method: "POST", body: form });
if (!response.ok) {
const body = await response.json().catch(() => null);
setRecordingError(body?.detail ?? "Audio upload failed.");
return;
}
setTranscriptionJob((await response.json()) as TranscriptionJob);
};
const toggleMic = async () => {
try {
if (isRecording) {
await stopMic();
} else {
await startMic();
}
} catch (error) {
setRecordingError(error instanceof Error ? error.message : "Microphone access failed.");
setIsRecording(false);
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
}
};
const generateCall = async (id: number) => {
await fetch(`${API}/pending-actions/${id}/approve`, { method: "POST" });
await refresh();
};
const reject = async (id: number) => {
await fetch(`${API}/pending-actions/${id}/reject`, { method: "POST" });
await refresh();
};
const latest = dashboard?.pending_actions[0];
return (
<main className="shell">
<header className="topbar">
<div>
<h1>AI Orchestrator</h1>
<p>Wedding planning meeting control room</p>
</div>
<button title="Refresh dashboard" onClick={refresh}><RefreshCw size={18} /> Refresh</button>
</header>
<section className="grid">
<aside className="panel transcript">
<div className="panel-title"><Clock size={18} /> Live Transcript</div>
<div className="mic-box">
<button onClick={toggleMic} className={isRecording ? "recording" : ""}>
{isRecording ? <X size={17} /> : <Play size={17} />}
{isRecording ? "Stop Mic" : "Start Mic"}
</button>
<span>{isRecording ? `${recordingSeconds}s` : transcriptionJob ? transcriptionJob.status : "Soniox mic transcription"}</span>
{recordingError && <p className="error">{recordingError}</p>}
{transcriptionJob?.status === "completed" && (
<p className="success">Generated {transcriptionJob.generated_action_ids.length} pending action(s).</p>
)}
</div>
<div className="composer">
<input value={speaker} onChange={(event) => setSpeaker(event.target.value)} aria-label="Speaker" />
<textarea value={text} onChange={(event) => setText(event.target.value)} aria-label="Transcript text" />
<div className="actions-row">
<button onClick={() => sendChunk()} disabled={busy}><Send size={17} /> Send</button>
<button onClick={replay} disabled={busy}><Play size={17} /> Replay</button>
</div>
</div>
<div className="list">
{(dashboard?.transcript ?? []).map((chunk, index) => (
<article className="line" key={`${chunk.timestamp}-${index}`}>
<strong>{chunk.speaker}</strong>
<span>{new Date(chunk.timestamp).toLocaleTimeString()}</span>
<p>{chunk.text}</p>
</article>
))}
</div>
</aside>
<section className="panel reasoning">
<div className="panel-title"><ShieldCheck size={18} /> Planner, Policy, Memory</div>
{latest ? (
<div className="decision">
<div className="metric-row">
<span className="tag">{latest.intent}</span>
<span className="confidence">{Math.round(latest.confidence * 100)}%</span>
<span className={`status ${latest.status}`}>{latest.status.replace("_", " ")}</span>
</div>
<p className="reason">{latest.reasoning}</p>
<div className="columns">
<div><h2>Payload</h2><JsonBlock value={latest.payload} /></div>
<div><h2>Validation</h2><JsonBlock value={latest.validation} /></div>
</div>
<div className="columns">
<div><h2>Policy</h2><JsonBlock value={latest.policy} /></div>
<div><h2>Retrieved Memories</h2><JsonBlock value={dashboard?.memories ?? []} /></div>
</div>
</div>
) : (
<div className="empty">Send or replay transcript chunks to create reviewed AI actions.</div>
)}
</section>
<aside className="panel pending">
<div className="panel-title"><Check size={18} /> Pending Review</div>
<div className="list">
{(dashboard?.pending_actions ?? []).map((action) => (
<article className="action-card" key={action.id}>
<div className="card-head">
<strong>{action.tool}</strong>
<span className={`status ${action.status}`}>{action.status.replace("_", " ")}</span>
</div>
<p>{action.reasoning}</p>
<JsonBlock value={action.payload} />
<div className="actions-row">
<button onClick={() => generateCall(action.id)} disabled={action.status !== "pending_review"}><Check size={16} /> Generate Call</button>
<button onClick={() => reject(action.id)} disabled={action.status !== "pending_review"}><X size={16} /> Reject</button>
</div>
</article>
))}
</div>
</aside>
</section>
<section className="bottom">
<div className="panel"><div className="panel-title">Current Context</div><JsonBlock value={dashboard?.context ?? {}} /></div>
<div className="panel"><div className="panel-title">Execution Logs</div><JsonBlock value={dashboard?.events ?? []} /></div>
<div className="panel"><div className="panel-title">REST State</div><JsonBlock value={{ tasks: dashboard?.tasks, notes: dashboard?.notes, reservations: dashboard?.reservations }} /></div>
</section>
</main>
);
}
ReactDOM.createRoot(document.getElementById("root")!).render(<App />);

262
frontend/src/styles.css Normal file
View File

@@ -0,0 +1,262 @@
:root {
color: #192126;
background: #f4f6f4;
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
}
* { box-sizing: border-box; }
body { margin: 0; }
button, input, textarea { font: inherit; }
.shell {
min-height: 100vh;
padding: 18px;
}
.topbar {
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
margin-bottom: 16px;
}
h1, h2, p { margin: 0; }
h1 { font-size: 28px; letter-spacing: 0; }
h2 { font-size: 13px; margin-bottom: 8px; color: #526064; }
.topbar p { color: #667277; margin-top: 3px; }
button {
min-height: 38px;
border: 1px solid #b9c5bd;
border-radius: 7px;
background: #ffffff;
color: #1f2d31;
display: inline-flex;
align-items: center;
justify-content: center;
gap: 8px;
padding: 8px 12px;
cursor: pointer;
}
button:hover:not(:disabled) { background: #edf4ef; }
button:disabled { opacity: 0.45; cursor: not-allowed; }
.grid {
display: grid;
grid-template-columns: minmax(260px, 0.85fr) minmax(360px, 1.5fr) minmax(280px, 0.95fr);
gap: 14px;
align-items: stretch;
min-width: 0;
}
.panel {
background: #ffffff;
border: 1px solid #d7dfda;
border-radius: 8px;
padding: 14px;
min-width: 0;
box-shadow: 0 1px 2px rgba(31, 45, 49, 0.05);
}
.grid > *,
.bottom > *,
.columns > *,
.decision > * {
min-width: 0;
}
.panel-title {
display: flex;
align-items: center;
gap: 8px;
font-weight: 700;
margin-bottom: 12px;
}
.composer {
display: grid;
gap: 8px;
margin-bottom: 12px;
}
.mic-box {
display: grid;
gap: 7px;
margin-bottom: 12px;
padding: 10px;
border: 1px solid #dfe8e2;
border-radius: 8px;
background: #f8faf8;
}
.mic-box span {
color: #5d6a6e;
font-size: 13px;
}
button.recording {
background: #f8e7e5;
border-color: #dfb5ae;
color: #7c2c23;
}
.error {
color: #8d2d22;
font-size: 13px;
line-height: 1.35;
}
.success {
color: #225235;
font-size: 13px;
line-height: 1.35;
}
input, textarea {
width: 100%;
border: 1px solid #c6d1cb;
border-radius: 7px;
padding: 10px;
background: #fbfcfb;
color: #182327;
}
textarea {
min-height: 92px;
resize: vertical;
}
.actions-row {
display: flex;
gap: 8px;
flex-wrap: wrap;
}
.list {
display: grid;
gap: 10px;
max-height: 62vh;
overflow: auto;
padding-right: 2px;
}
.line, .action-card {
border: 1px solid #e0e6e1;
border-radius: 8px;
padding: 10px;
background: #fbfcfb;
}
.line {
display: grid;
gap: 5px;
}
.line span {
color: #718086;
font-size: 12px;
}
.reasoning {
min-height: 520px;
overflow: hidden;
}
.decision {
display: grid;
gap: 14px;
overflow: hidden;
}
.metric-row, .card-head {
display: flex;
align-items: center;
justify-content: space-between;
gap: 8px;
flex-wrap: wrap;
}
.tag, .confidence, .status {
border-radius: 999px;
padding: 5px 9px;
font-size: 12px;
font-weight: 700;
}
.tag { background: #e8f1ed; color: #25443b; }
.confidence { background: #e9eef4; color: #26405d; }
.status { background: #f1eee5; color: #5d4d27; }
.executed { background: #e6f3ea; color: #225235; }
.execution_failed, .rejected { background: #f8e7e5; color: #7c2c23; }
.pending_review { background: #fff4d8; color: #704b00; }
.reason {
line-height: 1.45;
color: #2f3b40;
}
.columns {
display: grid;
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
gap: 12px;
overflow: hidden;
}
pre {
margin: 0;
overflow: auto;
max-width: 100%;
max-height: 250px;
border-radius: 7px;
border: 1px solid #e2e8e4;
background: #f7f9f8;
padding: 10px;
color: #203034;
font-size: 12px;
line-height: 1.45;
white-space: pre;
}
.action-card {
display: grid;
gap: 10px;
}
.action-card p {
color: #48575c;
line-height: 1.4;
}
.empty {
min-height: 380px;
display: grid;
place-items: center;
color: #607074;
text-align: center;
}
.bottom {
display: grid;
grid-template-columns: 1fr 1fr 1fr;
gap: 14px;
margin-top: 14px;
min-width: 0;
}
@media (max-width: 1050px) {
.grid, .bottom {
grid-template-columns: 1fr;
}
.list {
max-height: 420px;
}
}
@media (max-width: 620px) {
.shell { padding: 12px; }
.topbar { align-items: flex-start; flex-direction: column; }
.columns { grid-template-columns: 1fr; }
h1 { font-size: 24px; }
}

1
frontend/src/vite-env.d.ts vendored Normal file
View File

@@ -0,0 +1 @@
/// <reference types="vite/client" />

21
frontend/tsconfig.json Normal file
View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ES2020",
"useDefineForClassFields": true,
"lib": ["DOM", "DOM.Iterable", "ES2020"],
"allowJs": false,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true,
"forceConsistentCasingInFileNames": true,
"module": "ESNext",
"moduleResolution": "Node",
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx"
},
"include": ["src"],
"references": []
}

22
frontend/vite.config.ts Normal file
View File

@@ -0,0 +1,22 @@
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
const backend = process.env.VITE_BACKEND_URL ?? "http://localhost:9000";
const backendWs = backend.replace(/^http/, "ws");
export default defineConfig({
plugins: [react()],
server: {
port: 5173,
proxy: {
"/dashboard": backend,
"/pending-actions": backend,
"/transcriptions": backend,
"/tools": backend,
"/ws": {
target: backendWs,
ws: true
}
}
}
});