JDS.

Search docs

Find a page or component

Recipes

Agent side panel

An agent side panel over your app: chat with page context, dictation, and a hold-to-talk voice mode whose turns land back in the chat. Style the panel and orb in the Studio, bring your own composer, connect useChat.

Orders

128

Use it in your app

  1. 1
    Style the panel and orbCopy the panel props from the Side Panel Studio and the orb props from the Voice Orb Studio.
    <AgentSidePanel
      session={session}
      open={open}
      onOpenChange={setOpen}
      panel={{ variant: "floating", surface: "glass", width: 420, motion: "spring" }}
      orb={{ variant: "liquid", palette: "ember", glow: 0.25 }}
    />
  2. 2
    Place it over your appIt sits fixed against the viewport edge; pass contained in panel to keep it inside a positioned container. Tell people what the agent sees with context.
    <AgentSidePanel session={session} open={open} onOpenChange={setOpen} context="Orders · 128 rows" />
  3. 3
    Bring your composerThe default composer has attachments, dictation and voice. Paste one from the Prompt Input Studio instead; it gets status, sendMessage and stop.
    <AgentSidePanel
      session={session}
      open={open}
      composer={({ status, sendMessage, stop }) => (
        <PromptInput status={status} onSubmit={({ text, files }) => sendMessage({ text, files })}>
          …
        </PromptInput>
      )}
    />
  4. 4
    Connect your sessionchat has the shape of the AI SDK's useChat, with messages flattened to text. Add voice to enable voice mode, or leave it out.
    const { messages, status, sendMessage, stop } = useChat()
    const session = {
      chat: { messages: toPanelMessages(messages), status, sendMessage, stop },
    }

Install

pnpm dlx shadcn@latest add @jds/agent-side-panel

Installs the recipe and every component it uses into your project, ready to run.

Connect your backend

The UI only reads this session. Build a hook that returns it from your realtime provider, then swap it for the simulated one.

Session interface
/**
 * Everything <AgentSidePanel> reads. `chat` has the shape of the AI SDK's useChat
 * (messages, status, sendMessage, stop), with messages flattened to text. `voice` drives
 * hold-to-talk; leave it out to hide voice mode. `useSimulatedPanel` is a scripted stand-in.
 */
type PanelSession = {
  chat: {
    messages: PanelMessage[]
    status: ChatStatus
    sendMessage: (message: { text: string; files?: File[] }) => void
    stop: () => void
  }
  voice?: {
    state: VoiceState
    /** Loudness 0 to 1 of whoever is talking. */
    level: number
    transcript: TranscriptSegment[]
    /** Hold to talk: start listening. */
    start: () => void
    /** Release: stop listening and let the agent answer. */
    release: () => void
    /** Leave voice mode. Implementations add the spoken turns to the chat. */
    end: () => void
  }
}

Source

components/agent-side-panel/agent-side-panel.tsx
"use client"

import * as React from "react"

import {
  AgentPanel,
  AgentPanelBody,
  AgentPanelFooter,
  AgentPanelHeader,
  AgentPanelVoiceButton,
  AgentPanelVoiceExit,
  type PanelMode,
} from "@/components/ai/agent-panel"
import { Conversation, ConversationContent, ConversationScrollButton } from "@/components/ai/conversation"
import { Message, MessageContent } from "@/components/ai/message"
import {
  PromptInput,
  PromptInputAttachButton,
  PromptInputFrame,
  PromptInputHeader,
  PromptInputSubmit,
  PromptInputTextarea,
  PromptInputToolbar,
  PromptInputTools,
} from "@/components/ai/prompt-input"
import { PromptInputMic } from "@/components/ai/prompt-input-mic"
import { Response } from "@/components/ai/response"
import { TypingIndicator } from "@/components/ai/shimmer"
import { Kbd } from "@/components/ui/kbd"
import { LiveTranscript } from "@/components/voice/live-transcript"
import { PushToTalk } from "@/components/voice/push-to-talk"
import { VoiceOrb, type VoiceState } from "@/components/voice/voice-orb"
import { FileIcon, VoiceIcon } from "@/lib/icons"

import type { PanelSession } from "./session"

/** Panel styling, exactly as the Side Panel Studio generates it. */
type PanelStyle = Pick<
  React.ComponentProps<typeof AgentPanel>,
  "side" | "variant" | "surface" | "width" | "inset" | "motion" | "contained"
>

/** Orb styling for voice mode, exactly as the Voice Orb Studio generates it. */
type PanelOrb = Omit<React.ComponentProps<typeof VoiceOrb>, "state" | "level">

/** What a custom composer gets: the same names the Prompt Input Studio's code uses. */
type ComposerApi = PanelSession["chat"]

const stateLabel: Record<VoiceState, string> = {
  idle: "Hold to talk",
  connecting: "Connecting…",
  listening: "Listening…",
  thinking: "Thinking…",
  speaking: "Speaking…",
  error: "Couldn't connect",
}

/**
 * An agent side panel over your app: chat, dictation and a hold-to-talk voice mode whose
 * turns land back in the chat. Style it with `panel` and `orb` from the Studio, swap the
 * composer with `composer`, and pass any `PanelSession`; `useSimulatedPanel` plays a sample.
 */
function AgentSidePanel({
  session,
  open,
  onOpenChange,
  panel,
  orb,
  title = "Agent",
  context,
  placeholder = "Ask about this page…",
  composer,
}: {
  session: PanelSession
  open: boolean
  onOpenChange?: (open: boolean) => void
  /** Paste the props from the Side Panel Studio. */
  panel?: PanelStyle
  /** Paste the props from the Voice Orb Studio. Without it, the orb follows the nearest VoiceOrbProvider. */
  orb?: PanelOrb
  title?: string
  /** What the agent is looking at, shown above the composer, e.g. "Orders · 128 rows". */
  context?: string
  placeholder?: string
  /** Replace the default composer, e.g. with code from the Prompt Input Studio. */
  composer?: (api: ComposerApi) => React.ReactNode
}) {
  const { chat, voice } = session
  const [mode, setMode] = React.useState<PanelMode>("chat")
  // One way out of voice mode, whether from the exit button or the header switch.
  const changeMode = (next: PanelMode) => {
    if (mode === "voice" && next === "chat") voice?.end()
    setMode(next)
  }

  const input = (
    <PromptInput status={chat.status} onSubmit={({ text, files }) => chat.sendMessage({ text, files })}>
      <PromptInputTextarea placeholder={placeholder} />
      <PromptInputToolbar>
        <PromptInputTools>
          <PromptInputAttachButton />
        </PromptInputTools>
        <div className="flex min-w-0 items-center gap-1">
          <PromptInputMic />
          {voice && <AgentPanelVoiceButton />}
          <PromptInputSubmit onStop={chat.stop} />
        </div>
      </PromptInputToolbar>
    </PromptInput>
  )

  return (
    <AgentPanel {...panel} open={open} onOpenChange={onOpenChange} mode={mode} onModeChange={changeMode}>
      <AgentPanelHeader title={title} />
      <AgentPanelBody
        chat={
          <Conversation>
            <ConversationContent className="gap-5 px-4 py-4">
              {chat.messages.length === 0 && (
                <div className="flex flex-col items-center gap-2 py-16 text-center">
                  <p className="text-sm font-medium">Ask about this page</p>
                  <p className="text-sm text-muted-foreground">
                    {voice ? "Type, dictate with the mic, or switch to voice." : "Type or dictate with the mic."}
                  </p>
                </div>
              )}
              {chat.messages.map((m) => (
                <Message key={m.id} from={m.from}>
                  <MessageContent>
                    {m.from === "assistant" ? (
                      m.text ? (
                        <Response isAnimating={chat.status === "streaming"}>{m.text}</Response>
                      ) : (
                        <TypingIndicator />
                      )
                    ) : (
                      m.text
                    )}
                    {m.via === "voice" && (
                      <span className="flex items-center gap-1 text-xs text-muted-foreground">
                        <VoiceIcon className="size-3" />
                        Voice
                      </span>
                    )}
                  </MessageContent>
                </Message>
              ))}
              {chat.status === "submitted" && (
                <Message from="assistant">
                  <MessageContent>
                    <TypingIndicator />
                  </MessageContent>
                </Message>
              )}
            </ConversationContent>
            <ConversationScrollButton />
          </Conversation>
        }
        voice={
          voice && (
            <div className="flex min-h-0 flex-1 flex-col items-center gap-4 px-6 py-6">
              <VoiceOrb size={160} {...orb} state={voice.state} level={voice.level} />
              <p className="flex h-5 items-center gap-1.5 text-sm text-muted-foreground">
                {stateLabel[voice.state]}
                {voice.state === "idle" && <Kbd>Space</Kbd>}
              </p>
              <LiveTranscript segments={voice.transcript} className="min-h-0 w-full flex-1 overflow-y-auto" />
              <div className="flex shrink-0 flex-col items-center gap-3">
                <PushToTalk onPressStart={voice.start} onPressEnd={voice.release} level={voice.level} />
                <AgentPanelVoiceExit />
              </div>
            </div>
          )
        }
      />
      <AgentPanelFooter>
        {composer ? (
          composer(chat)
        ) : context ? (
          <PromptInputFrame>
            <PromptInputHeader icon={<FileIcon />}>{context}</PromptInputHeader>
            {input}
          </PromptInputFrame>
        ) : (
          input
        )}
      </AgentPanelFooter>
    </AgentPanel>
  )
}

export { AgentSidePanel, type ComposerApi, type PanelOrb, type PanelStyle }
components/agent-side-panel/session.ts
"use client"

import * as React from "react"

import type { ChatStatus } from "@/components/ai/prompt-input"
import type { TranscriptSegment } from "@/components/voice/live-transcript"
import type { VoiceState } from "@/components/voice/voice-orb"
import { useSimulatedSpectrum } from "@/hooks/use-simulated-spectrum"

type PanelMessage = { id: string; from: "user" | "assistant"; text: string; via?: "voice" }

/**
 * Everything <AgentSidePanel> reads. `chat` has the shape of the AI SDK's useChat
 * (messages, status, sendMessage, stop), with messages flattened to text. `voice` drives
 * hold-to-talk; leave it out to hide voice mode. `useSimulatedPanel` is a scripted stand-in.
 */
type PanelSession = {
  chat: {
    messages: PanelMessage[]
    status: ChatStatus
    sendMessage: (message: { text: string; files?: File[] }) => void
    stop: () => void
  }
  voice?: {
    state: VoiceState
    /** Loudness 0 to 1 of whoever is talking. */
    level: number
    transcript: TranscriptSegment[]
    /** Hold to talk: start listening. */
    start: () => void
    /** Release: stop listening and let the agent answer. */
    release: () => void
    /** Leave voice mode. Implementations add the spoken turns to the chat. */
    end: () => void
  }
}

const REPLY = `Three orders are overdue by more than a week:

- **#4817** · Maria Rossi · 9 days
- **#4802** · Ken Adams · 12 days
- **#4795** · Lea Martin · 15 days

All three shipped with the same carrier. Want me to open a claim and email the customers?`

const VOICE_USER = "Open a claim for all three and email them."
const VOICE_AGENT = "Done. Claims are open and the customers have an email with the tracking update."

const wait = (ms: number) => new Promise((r) => setTimeout(r, ms))

/** A scripted session: every message gets the same sample reply, and voice plays one exchange. */
function useSimulatedPanel(): PanelSession {
  const [messages, setMessages] = React.useState<PanelMessage[]>([])
  const [status, setStatus] = React.useState<ChatStatus>("ready")
  const [voiceState, setVoiceState] = React.useState<VoiceState>("idle")
  const [transcript, setTranscript] = React.useState<TranscriptSegment[]>([])
  const run = React.useRef(0)
  const timers = React.useRef<ReturnType<typeof setTimeout>[]>([])
  const { level } = useSimulatedSpectrum(voiceState === "listening" || voiceState === "speaking")

  const clear = () => {
    timers.current.forEach(clearTimeout)
    timers.current = []
  }
  React.useEffect(() => clear, [])

  /** Types `text` into transcript segment `id` a word at a time. */
  const typeInto = (id: string, text: string, speed: number, done?: () => void) => {
    const words = text.split(" ")
    words.forEach((_, i) =>
      timers.current.push(
        setTimeout(() => {
          setTranscript((list) =>
            list.map((s) =>
              s.id === id ? { ...s, text: words.slice(0, i + 1).join(" "), final: i === words.length - 1 } : s,
            ),
          )
          if (i === words.length - 1) done?.()
        }, speed * (i + 1)),
      ),
    )
  }

  const sendMessage = async ({ text }: { text: string }) => {
    if (!text) return
    const id = ++run.current
    const alive = () => run.current === id
    const replyId = `a-${Date.now()}`
    setMessages((m) => [...m, { id: `u-${Date.now()}`, from: "user", text }])
    setStatus("submitted")
    await wait(700)
    if (!alive()) return
    setStatus("streaming")
    setMessages((m) => [...m, { id: replyId, from: "assistant", text: "" }])
    let out = ""
    for (const w of REPLY.split(/(\s+)/)) {
      if (!alive()) return
      out += w
      const snapshot = out
      setMessages((m) => m.map((x) => (x.id === replyId ? { ...x, text: snapshot } : x)))
      await wait(18 + Math.random() * 18)
    }
    if (alive()) setStatus("ready")
  }

  return {
    chat: {
      messages,
      status,
      sendMessage,
      stop: () => {
        run.current++
        setStatus("ready")
      },
    },
    voice: {
      state: voiceState,
      level,
      transcript,
      start: () => {
        clear()
        setVoiceState("listening")
        const id = `vu-${Date.now()}`
        setTranscript([{ id, speaker: "user", text: "", final: false }])
        typeInto(id, VOICE_USER, 180)
      },
      release: () => {
        clear()
        setTranscript((list) => list.map((s) => (s.speaker === "user" ? { ...s, text: VOICE_USER, final: true } : s)))
        setVoiceState("thinking")
        timers.current.push(
          setTimeout(() => {
            setVoiceState("speaking")
            const id = `va-${Date.now()}`
            setTranscript((list) => [...list, { id, speaker: "agent", text: "", final: false }])
            typeInto(id, VOICE_AGENT, 160, () => timers.current.push(setTimeout(() => setVoiceState("idle"), 400)))
          }, 1200),
        )
      },
      end: () => {
        clear()
        setVoiceState("idle")
        // Spoken turns land back in the chat, marked as voice.
        setMessages((m) => [
          ...m,
          ...transcript
            .filter((s) => s.text)
            .map((s) => ({
              id: s.id,
              from: s.speaker === "user" ? ("user" as const) : ("assistant" as const),
              text: s.text,
              via: "voice" as const,
            })),
        ])
        setTranscript([])
      },
    },
  }
}

export { useSimulatedPanel, type PanelMessage, type PanelSession }
hooks/use-simulated-spectrum.ts
"use client"

import * as React from "react"

/** Speech-like spectrum for agent playback, demos, and users who decline mic access. */
export function useSimulatedSpectrum(active: boolean, bands = 24) {
  const [spectrum, setSpectrum] = React.useState<number[]>(() => Array(bands).fill(0))

  React.useEffect(() => {
    if (!active) return
    let raf = 0
    const tick = (t: number) => {
      setSpectrum(
        Array.from({ length: bands }, (_, i) => {
          const center = 1 - Math.abs(i - bands / 2) / (bands / 2)
          const wave = (Math.sin(t / 140 + i * 0.7) + Math.sin(t / 90 + i * 1.3)) / 4 + 0.5
          return Math.max(0, center * wave * (0.6 + Math.random() * 0.4))
        })
      )
      raf = requestAnimationFrame(tick)
    }
    raf = requestAnimationFrame(tick)
    return () => {
      cancelAnimationFrame(raf)
      setSpectrum(Array(bands).fill(0))
    }
  }, [active, bands])

  const level = Math.min(1, (spectrum.reduce((a, b) => a + b, 0) / bands) * 2.2)
  return { spectrum, level }
}