Optimization runs without a config

1
# LD imports
2
from ldai_optimization import (
3
    OptimizationClient,
4
    OptimizationOptions,
5
    OptimizationJudge,
6
    OptimizationResponse,
7
    LLMCallConfig,
8
    LLMCallContext,
9
)
10
from ldai import LDAIClient
11
from ldai.tracker import TokenUsage
12
from ldclient import Context
13
14
# anthropic imports
15
from claude_agent_sdk import query, ClaudeAgentOptions
16
from claude_agent_sdk.types import ResultMessage
17
18
acceptance_statement_prompt = """\
19
The orchestrator should appropriately fetch the user preferences and \
20
route to the correct sub-agent, \
21
carrying through any relevant information from the users' query.
22
The orchestrator should not provide any answers itself, \
23
just pass to the correct sub-agent.
24
Inability to fetch user preferences or mentions of missing data should \
25
be automatic failures.
26
If preferences are not included, that should be an automatic failure.
27
If the orchestrator does not mention the sub-agent it will hand off to, \
28
that is an automatic failure."""
29
30
default_fallback_model = "claude-opus-4-5-20251101"
31
32
async def handle_agent_call(
33
    key: str,
34
    config: LLMCallConfig,
35
    context: LLMCallContext,
36
    is_evaluation: bool = False,
37
) -> OptimizationResponse:
38
    model = config.model.name if config.model else default_fallback_model
39
    final_message = None
40
    async for message in query(
41
        prompt=context.user_input or "",
42
        options=ClaudeAgentOptions(
43
            system_prompt=config.instructions or "",
44
            model=model,
45
        ),
46
    ):
47
        final_message = message
48
49
    if not isinstance(final_message, ResultMessage):
50
        raise ValueError(f"Unexpected final message type: {type(final_message)}")
51
52
    u = final_message.usage or {}
53
    input_tokens = u.get("input_tokens", 0)
54
    output_tokens = u.get("output_tokens", 0)
55
56
    return OptimizationResponse(
57
        output=final_message.result or "",
58
        usage=TokenUsage(
59
            total=input_tokens + output_tokens,
60
            input=input_tokens,
61
            output=output_tokens,
62
        ),
63
    )
64
65
options = OptimizationOptions(
66
    judges={
67
        "acceptance": OptimizationJudge(
68
            acceptance_statement=acceptance_statement_prompt,
69
            threshold=0.95,
70
        ),
71
        "accuracy": OptimizationJudge(
72
            judge_key="my-accuracy-judge",
73
            threshold=0.8,
74
        ),
75
    },
76
    context_choices=[
77
        Context.builder("user-123").set("user_id", "user-123").build(),
78
    ],
79
    max_attempts=25,
80
    model_choices=["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"],
81
    judge_model="claude-haiku-4-5",
82
    variable_choices=[
83
        {
84
            "user_id": "user-123",
85
            "trip_purpose": "business",
86
        },
87
        {
88
            "user_id": "user-125",
89
            "trip_purpose": "personal",
90
        },
91
    ],
92
    user_input_options=[
93
        "I'm going to tokyo next week, where should I stay near Shinjuku?",
94
        "where to eat in anchorage",
95
        "airbnbs near tahoe",
96
        "what are some food options in sf near the airport"
97
    ],
98
    handle_agent_call=handle_agent_call,
99
    handle_judge_call=handle_agent_call,
100
)
101
102
client = OptimizationClient(ld_ai_client)
103
result = await client.optimize_from_options("travel-agent-orchestrator", options)

$
INFO:ldai_optimization.client:[Iteration 1] -> Starting (attempt 1/10, model=claude-haiku-4-5)
$
INFO:ldai_optimization.client:[Iteration 1] -> Calling agent (model=claude-haiku-4-5)...
$
INFO:ldai_optimization.client:[Iteration 1] -> Executing evaluation...
$
INFO:ldai_optimization.client:[Iteration 1] -> Running judge 1/2 'acceptance-statement-0' (acceptance)...
$
INFO:ldai_optimization.client:[Iteration 1] -> Running judge 2/2 'ld-ai-judge-accuracy-1761841832389' (config)...
$
INFO:ldai_optimization.client:[Iteration 1] -> One or more judges failed (attempt 1/10) — generating new variation
$
INFO:ldai_optimization.client:[Iteration 1] -> Generating new variation...
$
INFO:ldai_optimization.client:[Iteration 1] -> Model updated from 'claude-haiku-4-5' to 'claude-sonnet-4-5'
$
INFO:ldai_optimization.client:[Iteration 2] -> Starting (attempt 2/10, model=claude-sonnet-4-5)
$
INFO:ldai_optimization.client:[Iteration 2] -> Calling agent (model=claude-sonnet-4-5)...
$
INFO:ldai_optimization.client:[Iteration 2] -> Executing evaluation...
$
INFO:ldai_optimization.client:[Iteration 2] -> Running judge 1/2 'acceptance-statement-0' (acceptance)...
$
INFO:ldai_optimization.client:[Iteration 2] -> Running judge 2/2 'ld-ai-judge-accuracy-1761841832389' (config)...
$
INFO:ldai_optimization.client:[Iteration 2] -> One or more judges failed (attempt 2/10) — generating new variation
$
INFO:ldai_optimization.client:[Iteration 2] -> Generating new variation...

1
{
2
  "scores": {
3
    "acceptance-statement-0": {
4
      "score": 1.0,
5
      "rationale": "The response cleanly routes to the lodging-agent...",
6
      "duration_ms": 9795.460292021744,
7
      "usage": {
8
        "total": 4050,
9
        "input": 3088,
10
        "output": 962
11
      }
12
    }
13
  },
14
  "completion_response": "Routing to **lodging-agent**.\n\nHandoff context:\n- User ID placeholder: **user-125**...",
15
  "current_parameters": {
16
    // ... model parameters used for this iteration ...
17
  },
18
  "current_variables": {
19
    "trip_purpose": "business",
20
    "user_id": "user-125"
21
  },
22
  "current_model": "claude-sonnet-4-5",
23
  "user_input": "airbnbs near tahoe",
24
  "history": [
25
    // ... optimization context history ...
26
  ],
27
  "iteration": 5,
28
  "duration_ms": 2536.8339580018073,
29
  "usage": {
30
    "total": 1234,
31
    "input": 1109,
32
    "output": 125
33
  }
34
}