Prompt Caching: Don't Pay for the Same Prompt Again and Again

[ Static Prefix ] --> [ Cache Breakpoint ] --> [ Dynamic Input ]
      |                      |                        |
      |                      |                        |
   (Tools, Rules,     (Cache written once)     (Only new content
   Stable Memory)                              sent each time)

Result: Lower cost, faster responses, same quality

# 1) Build the static prefix once (cacheable)
TOOLS:
  - name: click, description: "Click a selector", input: { selector: string }
  - name: type_text, description: "Type into a selector", input: { selector: string, text: string }

SYSTEM RULES:
  - "You are an automation agent. Use tools when possible."
  - "Be concise. Use JSON outputs."

STABLE INSTRUCTIONS / MEMORY:
  - "<team conventions, safety rules, canonical few-shots, DOM policy, etc.>"

# ---- CACHE BREAKPOINT (everything above is cacheable) ----

# 2) Dynamic input (changes each step)
USER INPUT:
  - "Open the calendar and pick Aug-13."

type Tool = {
  name: string;
  description: string;
  input_schema: Record<string, unknown>;
};

type CachedPrefix = {
  tools: Tool[];
  system: string[];           // rules, behavior
  stableInstructions: string; // long, rarely-changing block
};

// Simple hash for demo only; replace with a real hash (e.g., SHA-256).
function simpleHash(s: string): string {
  let h = 0;
  for (let i = 0; i < s.length; i++) h = ((h << 5) - h + s.charCodeAt(i)) | 0;
  return String(h >>> 0);
}

function buildPrefixString(p: CachedPrefix): string {
  return JSON.stringify({
    tools: p.tools,
    system: p.system,
    stable: p.stableInstructions,
  });
}

type CacheStore = Map<string, { expiresAt: number }>;
const inMemoryCache: CacheStore = new Map();

function cacheWriteIfNeeded(prefix: CachedPrefix, ttlMs: number): { key: string; wrote: boolean } {
  const key = simpleHash(buildPrefixString(prefix));
  const entry = inMemoryCache.get(key);
  const now = Date.now();
  if (!entry || entry.expiresAt <= now) {
    // In a real implementation, call your LLM provider's "cache write" API here.
    inMemoryCache.set(key, { expiresAt: now + ttlMs });
    return { key, wrote: true };
  }
  return { key, wrote: false };
}

type LlmClient = {
  // You will adapt this to your provider. Some take a cache key / handle, some take a special flag.
  createMessage(input: {
    model: string;
    max_tokens: number;
    cache_key?: string;      // provider-specific
    use_cached_prefix?: boolean;
    dynamic_messages: Array<{ role: "user" | "assistant" | "tool"; content: string }>;
  }): Promise<{ text: string; tokens_used?: number }>;
};

async function runStep(
  client: LlmClient,
  prefix: CachedPrefix,
  dynamicUserText: string,
  ttlMs = 5 * 60_000 // 5 minutes
) {
  const { key, wrote } = cacheWriteIfNeeded(prefix, ttlMs);

  // NOTE: On providers with explicit caching, you may need to:
  // (a) send the prefix with a "cache_control" tag on first call,
  // (b) then on subsequent calls, omit the prefix and pass the cache handle/key.

  const res = await client.createMessage({
    model: "my-llm-model",
    max_tokens: 512,
    cache_key: key,            // or provider-specific field
    use_cached_prefix: true,   // your adapter toggles this into the provider’s mechanism
    dynamic_messages: [
      { role: "user", content: dynamicUserText }
    ]
  });

  // Basic logging so juniors can see savings in action:
  console.log(
    wrote ? "[cache write]" : "[cache read]",
    "cache_key=", key,
    "response_tokens=", res.tokens_used ?? "n/a"
  );

  return res.text;
}

// Example usage:
const prefix: CachedPrefix = {
  tools: [
    { name: "click", description: "Click a selector", input_schema: { selector: "string" } as any },
    { name: "type_text", description: "Type into a selector", input_schema: { selector: "string", text: "string" } as any }
  ],
  system: [
    "You are an automation agent. Prefer tools.",
    "Return JSON."
  ],
  stableInstructions: "<long rules, few-shots, DOM policy, etc.>"
};

// First call likely writes cache; later calls reuse it:
await runStep(myClient, prefix, "Open the calendar and pick Aug-13.");
await runStep(myClient, prefix, "Create a new event for tomorrow at 10am.");

POST /llm/messages
Content-Type: application/json

{
  "model": "my-llm-model",
  "max_tokens": 512,

  "tools": [
    { "name": "click", "description": "Click a selector", "input_schema": { "type": "object", "properties": { "selector": { "type": "string" } }, "required": ["selector"] } },
    { "name": "type_text", "description": "Type into a selector", "input_schema": { "type": "object", "properties": { "selector": { "type": "string" }, "text": { "type": "string" } }, "required": ["selector","text"] } }
  ],

  "system": [
    { "type": "text", "text": "You are an automation agent. Prefer tools." },
    { "type": "text", "text": "<long stable rules / few-shots / policies>", "cache_control": { "type": "ephemeral" } }
  ],

  "messages": [
    { "role": "user", "content": "Start session." }
  ]
}

POST /llm/messages
Content-Type: application/json

{
  "model": "my-llm-model",
  "max_tokens": 512,

  "use_cached_prefix": true,    // provider-specific flag or pass a cache handle returned earlier

  "messages": [
    { "role": "user", "content": "Open the calendar and pick Aug-13." }
  ]
}