Recipes
Agent side panel
An agent side panel over your app: chat with page context, dictation, and a hold-to-talk voice mode whose turns land back in the chat. Style the panel and orb in the Studio, bring your own composer, connect useChat.
Orders
128Use it in your app
- 1Style the panel and orbCopy the panel props from the Side Panel Studio and the orb props from the Voice Orb Studio.
<AgentSidePanel session={session} open={open} onOpenChange={setOpen} panel={{ variant: "floating", surface: "glass", width: 420, motion: "spring" }} orb={{ variant: "liquid", palette: "ember", glow: 0.25 }} /> - 2Place it over your appIt sits fixed against the viewport edge; pass contained in panel to keep it inside a positioned container. Tell people what the agent sees with context.
<AgentSidePanel session={session} open={open} onOpenChange={setOpen} context="Orders · 128 rows" /> - 3Bring your composerThe default composer has attachments, dictation and voice. Paste one from the Prompt Input Studio instead; it gets status, sendMessage and stop.
<AgentSidePanel session={session} open={open} composer={({ status, sendMessage, stop }) => ( <PromptInput status={status} onSubmit={({ text, files }) => sendMessage({ text, files })}> … </PromptInput> )} /> - 4Connect your sessionchat has the shape of the AI SDK's useChat, with messages flattened to text. Add voice to enable voice mode, or leave it out.
const { messages, status, sendMessage, stop } = useChat() const session = { chat: { messages: toPanelMessages(messages), status, sendMessage, stop }, }
Install
pnpm dlx shadcn@latest add @jds/agent-side-panel
Installs the recipe and every component it uses into your project, ready to run.
Connect your backend
The UI only reads this session. Build a hook that returns it from your realtime provider, then swap it for the simulated one.
Session interface
/**
* Everything <AgentSidePanel> reads. `chat` has the shape of the AI SDK's useChat
* (messages, status, sendMessage, stop), with messages flattened to text. `voice` drives
* hold-to-talk; leave it out to hide voice mode. `useSimulatedPanel` is a scripted stand-in.
*/
type PanelSession = {
chat: {
messages: PanelMessage[]
status: ChatStatus
sendMessage: (message: { text: string; files?: File[] }) => void
stop: () => void
}
voice?: {
state: VoiceState
/** Loudness 0 to 1 of whoever is talking. */
level: number
transcript: TranscriptSegment[]
/** Hold to talk: start listening. */
start: () => void
/** Release: stop listening and let the agent answer. */
release: () => void
/** Leave voice mode. Implementations add the spoken turns to the chat. */
end: () => void
}
}Source
components/agent-side-panel/agent-side-panel.tsx
"use client"
import * as React from "react"
import {
AgentPanel,
AgentPanelBody,
AgentPanelFooter,
AgentPanelHeader,
AgentPanelVoiceButton,
AgentPanelVoiceExit,
type PanelMode,
} from "@/components/ai/agent-panel"
import { Conversation, ConversationContent, ConversationScrollButton } from "@/components/ai/conversation"
import { Message, MessageContent } from "@/components/ai/message"
import {
PromptInput,
PromptInputAttachButton,
PromptInputFrame,
PromptInputHeader,
PromptInputSubmit,
PromptInputTextarea,
PromptInputToolbar,
PromptInputTools,
} from "@/components/ai/prompt-input"
import { PromptInputMic } from "@/components/ai/prompt-input-mic"
import { Response } from "@/components/ai/response"
import { TypingIndicator } from "@/components/ai/shimmer"
import { Kbd } from "@/components/ui/kbd"
import { LiveTranscript } from "@/components/voice/live-transcript"
import { PushToTalk } from "@/components/voice/push-to-talk"
import { VoiceOrb, type VoiceState } from "@/components/voice/voice-orb"
import { FileIcon, VoiceIcon } from "@/lib/icons"
import type { PanelSession } from "./session"
/** Panel styling, exactly as the Side Panel Studio generates it. */
type PanelStyle = Pick<
React.ComponentProps<typeof AgentPanel>,
"side" | "variant" | "surface" | "width" | "inset" | "motion" | "contained"
>
/** Orb styling for voice mode, exactly as the Voice Orb Studio generates it. */
type PanelOrb = Omit<React.ComponentProps<typeof VoiceOrb>, "state" | "level">
/** What a custom composer gets: the same names the Prompt Input Studio's code uses. */
type ComposerApi = PanelSession["chat"]
const stateLabel: Record<VoiceState, string> = {
idle: "Hold to talk",
connecting: "Connecting…",
listening: "Listening…",
thinking: "Thinking…",
speaking: "Speaking…",
error: "Couldn't connect",
}
/**
* An agent side panel over your app: chat, dictation and a hold-to-talk voice mode whose
* turns land back in the chat. Style it with `panel` and `orb` from the Studio, swap the
* composer with `composer`, and pass any `PanelSession`; `useSimulatedPanel` plays a sample.
*/
function AgentSidePanel({
session,
open,
onOpenChange,
panel,
orb,
title = "Agent",
context,
placeholder = "Ask about this page…",
composer,
}: {
session: PanelSession
open: boolean
onOpenChange?: (open: boolean) => void
/** Paste the props from the Side Panel Studio. */
panel?: PanelStyle
/** Paste the props from the Voice Orb Studio. Without it, the orb follows the nearest VoiceOrbProvider. */
orb?: PanelOrb
title?: string
/** What the agent is looking at, shown above the composer, e.g. "Orders · 128 rows". */
context?: string
placeholder?: string
/** Replace the default composer, e.g. with code from the Prompt Input Studio. */
composer?: (api: ComposerApi) => React.ReactNode
}) {
const { chat, voice } = session
const [mode, setMode] = React.useState<PanelMode>("chat")
// One way out of voice mode, whether from the exit button or the header switch.
const changeMode = (next: PanelMode) => {
if (mode === "voice" && next === "chat") voice?.end()
setMode(next)
}
const input = (
<PromptInput status={chat.status} onSubmit={({ text, files }) => chat.sendMessage({ text, files })}>
<PromptInputTextarea placeholder={placeholder} />
<PromptInputToolbar>
<PromptInputTools>
<PromptInputAttachButton />
</PromptInputTools>
<div className="flex min-w-0 items-center gap-1">
<PromptInputMic />
{voice && <AgentPanelVoiceButton />}
<PromptInputSubmit onStop={chat.stop} />
</div>
</PromptInputToolbar>
</PromptInput>
)
return (
<AgentPanel {...panel} open={open} onOpenChange={onOpenChange} mode={mode} onModeChange={changeMode}>
<AgentPanelHeader title={title} />
<AgentPanelBody
chat={
<Conversation>
<ConversationContent className="gap-5 px-4 py-4">
{chat.messages.length === 0 && (
<div className="flex flex-col items-center gap-2 py-16 text-center">
<p className="text-sm font-medium">Ask about this page</p>
<p className="text-sm text-muted-foreground">
{voice ? "Type, dictate with the mic, or switch to voice." : "Type or dictate with the mic."}
</p>
</div>
)}
{chat.messages.map((m) => (
<Message key={m.id} from={m.from}>
<MessageContent>
{m.from === "assistant" ? (
m.text ? (
<Response isAnimating={chat.status === "streaming"}>{m.text}</Response>
) : (
<TypingIndicator />
)
) : (
m.text
)}
{m.via === "voice" && (
<span className="flex items-center gap-1 text-xs text-muted-foreground">
<VoiceIcon className="size-3" />
Voice
</span>
)}
</MessageContent>
</Message>
))}
{chat.status === "submitted" && (
<Message from="assistant">
<MessageContent>
<TypingIndicator />
</MessageContent>
</Message>
)}
</ConversationContent>
<ConversationScrollButton />
</Conversation>
}
voice={
voice && (
<div className="flex min-h-0 flex-1 flex-col items-center gap-4 px-6 py-6">
<VoiceOrb size={160} {...orb} state={voice.state} level={voice.level} />
<p className="flex h-5 items-center gap-1.5 text-sm text-muted-foreground">
{stateLabel[voice.state]}
{voice.state === "idle" && <Kbd>Space</Kbd>}
</p>
<LiveTranscript segments={voice.transcript} className="min-h-0 w-full flex-1 overflow-y-auto" />
<div className="flex shrink-0 flex-col items-center gap-3">
<PushToTalk onPressStart={voice.start} onPressEnd={voice.release} level={voice.level} />
<AgentPanelVoiceExit />
</div>
</div>
)
}
/>
<AgentPanelFooter>
{composer ? (
composer(chat)
) : context ? (
<PromptInputFrame>
<PromptInputHeader icon={<FileIcon />}>{context}</PromptInputHeader>
{input}
</PromptInputFrame>
) : (
input
)}
</AgentPanelFooter>
</AgentPanel>
)
}
export { AgentSidePanel, type ComposerApi, type PanelOrb, type PanelStyle }components/agent-side-panel/session.ts
"use client"
import * as React from "react"
import type { ChatStatus } from "@/components/ai/prompt-input"
import type { TranscriptSegment } from "@/components/voice/live-transcript"
import type { VoiceState } from "@/components/voice/voice-orb"
import { useSimulatedSpectrum } from "@/hooks/use-simulated-spectrum"
type PanelMessage = { id: string; from: "user" | "assistant"; text: string; via?: "voice" }
/**
* Everything <AgentSidePanel> reads. `chat` has the shape of the AI SDK's useChat
* (messages, status, sendMessage, stop), with messages flattened to text. `voice` drives
* hold-to-talk; leave it out to hide voice mode. `useSimulatedPanel` is a scripted stand-in.
*/
type PanelSession = {
chat: {
messages: PanelMessage[]
status: ChatStatus
sendMessage: (message: { text: string; files?: File[] }) => void
stop: () => void
}
voice?: {
state: VoiceState
/** Loudness 0 to 1 of whoever is talking. */
level: number
transcript: TranscriptSegment[]
/** Hold to talk: start listening. */
start: () => void
/** Release: stop listening and let the agent answer. */
release: () => void
/** Leave voice mode. Implementations add the spoken turns to the chat. */
end: () => void
}
}
const REPLY = `Three orders are overdue by more than a week:
- **#4817** · Maria Rossi · 9 days
- **#4802** · Ken Adams · 12 days
- **#4795** · Lea Martin · 15 days
All three shipped with the same carrier. Want me to open a claim and email the customers?`
const VOICE_USER = "Open a claim for all three and email them."
const VOICE_AGENT = "Done. Claims are open and the customers have an email with the tracking update."
const wait = (ms: number) => new Promise((r) => setTimeout(r, ms))
/** A scripted session: every message gets the same sample reply, and voice plays one exchange. */
function useSimulatedPanel(): PanelSession {
const [messages, setMessages] = React.useState<PanelMessage[]>([])
const [status, setStatus] = React.useState<ChatStatus>("ready")
const [voiceState, setVoiceState] = React.useState<VoiceState>("idle")
const [transcript, setTranscript] = React.useState<TranscriptSegment[]>([])
const run = React.useRef(0)
const timers = React.useRef<ReturnType<typeof setTimeout>[]>([])
const { level } = useSimulatedSpectrum(voiceState === "listening" || voiceState === "speaking")
const clear = () => {
timers.current.forEach(clearTimeout)
timers.current = []
}
React.useEffect(() => clear, [])
/** Types `text` into transcript segment `id` a word at a time. */
const typeInto = (id: string, text: string, speed: number, done?: () => void) => {
const words = text.split(" ")
words.forEach((_, i) =>
timers.current.push(
setTimeout(() => {
setTranscript((list) =>
list.map((s) =>
s.id === id ? { ...s, text: words.slice(0, i + 1).join(" "), final: i === words.length - 1 } : s,
),
)
if (i === words.length - 1) done?.()
}, speed * (i + 1)),
),
)
}
const sendMessage = async ({ text }: { text: string }) => {
if (!text) return
const id = ++run.current
const alive = () => run.current === id
const replyId = `a-${Date.now()}`
setMessages((m) => [...m, { id: `u-${Date.now()}`, from: "user", text }])
setStatus("submitted")
await wait(700)
if (!alive()) return
setStatus("streaming")
setMessages((m) => [...m, { id: replyId, from: "assistant", text: "" }])
let out = ""
for (const w of REPLY.split(/(\s+)/)) {
if (!alive()) return
out += w
const snapshot = out
setMessages((m) => m.map((x) => (x.id === replyId ? { ...x, text: snapshot } : x)))
await wait(18 + Math.random() * 18)
}
if (alive()) setStatus("ready")
}
return {
chat: {
messages,
status,
sendMessage,
stop: () => {
run.current++
setStatus("ready")
},
},
voice: {
state: voiceState,
level,
transcript,
start: () => {
clear()
setVoiceState("listening")
const id = `vu-${Date.now()}`
setTranscript([{ id, speaker: "user", text: "", final: false }])
typeInto(id, VOICE_USER, 180)
},
release: () => {
clear()
setTranscript((list) => list.map((s) => (s.speaker === "user" ? { ...s, text: VOICE_USER, final: true } : s)))
setVoiceState("thinking")
timers.current.push(
setTimeout(() => {
setVoiceState("speaking")
const id = `va-${Date.now()}`
setTranscript((list) => [...list, { id, speaker: "agent", text: "", final: false }])
typeInto(id, VOICE_AGENT, 160, () => timers.current.push(setTimeout(() => setVoiceState("idle"), 400)))
}, 1200),
)
},
end: () => {
clear()
setVoiceState("idle")
// Spoken turns land back in the chat, marked as voice.
setMessages((m) => [
...m,
...transcript
.filter((s) => s.text)
.map((s) => ({
id: s.id,
from: s.speaker === "user" ? ("user" as const) : ("assistant" as const),
text: s.text,
via: "voice" as const,
})),
])
setTranscript([])
},
},
}
}
export { useSimulatedPanel, type PanelMessage, type PanelSession }hooks/use-simulated-spectrum.ts
"use client"
import * as React from "react"
/** Speech-like spectrum for agent playback, demos, and users who decline mic access. */
export function useSimulatedSpectrum(active: boolean, bands = 24) {
const [spectrum, setSpectrum] = React.useState<number[]>(() => Array(bands).fill(0))
React.useEffect(() => {
if (!active) return
let raf = 0
const tick = (t: number) => {
setSpectrum(
Array.from({ length: bands }, (_, i) => {
const center = 1 - Math.abs(i - bands / 2) / (bands / 2)
const wave = (Math.sin(t / 140 + i * 0.7) + Math.sin(t / 90 + i * 1.3)) / 4 + 0.5
return Math.max(0, center * wave * (0.6 + Math.random() * 0.4))
})
)
raf = requestAnimationFrame(tick)
}
raf = requestAnimationFrame(tick)
return () => {
cancelAnimationFrame(raf)
setSpectrum(Array(bands).fill(0))
}
}, [active, bands])
const level = Math.min(1, (spectrum.reduce((a, b) => a + b, 0) / bands) * 2.2)
return { spectrum, level }
}