What we tested
We exercise two valid payloads and ten invalid ones. A spy callback records each save invocation. Missing keys, extra keys, wrong types, empty or oversized names, and non-object roots must fail before that callback runs.
Synthetic cases
Matched expectations
These counts describe the reference code below. They are not an LLM benchmark, customer result or a fix for another project.
Practical workflow
- Write the exact input contract before connecting a real persistence layer.
- Keep the validator separate from the function that performs the save.
- Test both rejection and the absence of side effects; checking an exception alone is insufficient.
Run it yourself
Expand the example, copy it to reference.py, and run python reference.py. It uses Python 3 and the standard library, requires no account or API key, and performs no network requests.
Show the runnable Python example
# Reproducible synthetic reference experiment. Python 3; standard library only.
import json
def checked_save(value,save):
if not isinstance(value,dict) or set(value)!={'name','enabled'} or not isinstance(value['name'],str) or not 1<=len(value['name'])<=80 or type(value['enabled']) is not bool:raise ValueError('save_contract_invalid')
save(dict(value));return True
def provider_route(child,parent,request):
if not isinstance(child,dict) or not set(child)<={'provider','model'} or not isinstance(child.get('model'),str) or not child['model']:raise ValueError('route_contract_invalid')
provider=child.get('provider',parent)
if not isinstance(provider,str) or provider not in {'alpha','beta'}:raise ValueError('provider_not_registered')
return request(provider,child['model'])
def checked_arguments(raw,convert):
def unique(pairs):
value={}
for key,item in pairs:
if key in value:raise ValueError('duplicate_key')
value[key]=item
return value
value=json.loads(raw,object_pairs_hook=unique)
if not isinstance(value,dict) or set(value)!={'name','limit','enabled'} or not isinstance(value['name'],str) or not value['name'] or type(value['limit']) is not int or not 1<=value['limit']<=100 or type(value['enabled']) is not bool:raise ValueError('argument_contract_invalid')
return convert(value)
def fixtures(capability):
if capability=='early_contract_validation':
return [{'value':v,'valid':valid} for v,valid in [({'name':'Sample','enabled':True},True),({'name':'Sample','enabled':False},True),
({'name':'Sample'},False),({'enabled':True},False),({'name':42,'enabled':True},False),({'name':'Sample','enabled':'true'},False),
({'name':'Sample','enabled':1},False),({'name':'','enabled':True},False),({'name':'x'*81,'enabled':True},False),
({'name':'Sample','enabled':True,'extra':1},False),(None,False),([],False)]]
if capability=='provider_route_isolation':
return [{'child':c,'parent':p,'provider':expected,'fail':fail} for c,p,expected,fail in [
({'model':'m'},'alpha','alpha',False),({'provider':'beta','model':'m'},'alpha','beta',False),
({'provider':'alpha','model':'m'},'beta','alpha',False),({'provider':'unknown','model':'m'},'alpha',None,False),
({'provider':None,'model':'m'},'alpha',None,False),({'provider':'beta','model':'m'},'alpha','beta',True),
({'provider':'beta','model':'m','url':'untrusted'},'alpha',None,False),({'provider':'beta','model':''},'alpha',None,False)]]
if capability=='tool_argument_validation':
good={'name':'sample','limit':10,'enabled':True}
return [{'raw':raw,'valid':valid} for raw,valid in [(json.dumps(good),True),(json.dumps(dict(good,limit=100)),True),
('{"name":"sample","name":"second","limit":10,"enabled":true}',False),('{',False),('[]',False),('null',False),
(json.dumps(dict(good,limit=True)),False),(json.dumps(dict(good,limit=0)),False),(json.dumps(dict(good,limit=101)),False),
(json.dumps(dict(good,enabled='true')),False),(json.dumps(dict(good,extra=1)),False),(json.dumps({'name':'sample','enabled':True}),False)]]
raise ValueError('experiment_not_registered')
def observe(capability):
cases=fixtures(capability);observations=[]
for index,case in enumerate(cases):
calls=[];returned=False;error=None
try:
if capability=='early_contract_validation':checked_save(case['value'],lambda value:calls.append(value));returned=True
elif capability=='provider_route_isolation':
def request(provider,model):
calls.append(provider)
if case['fail']:raise RuntimeError('synthetic_provider_failure')
return 'mock_response'
provider_route(case['child'],case['parent'],request);returned=True
else:checked_arguments(case['raw'],lambda value:calls.append(dict(value,name=value['name'].upper())));returned=True
except (ValueError,TypeError,RuntimeError) as exc:error=type(exc).__name__
if capability=='provider_route_isolation':
expected=[] if case['provider'] is None else [case['provider']]
matched=calls==expected and returned==(case['provider'] is not None and not case['fail'])
else:matched=returned==case['valid'] and len(calls)==int(case['valid'])
observations.append({'case':index,'matched_expectation':matched,'mock_side_effect_count':len(calls),'error_type':error})
return observations
observations = observe('early_contract_validation')
result = {"cases": len(observations), "matched_expectation": sum(row["matched_expectation"] for row in observations), "observations": observations}
assert result["cases"] == result["matched_expectation"]
print(json.dumps(result, sort_keys=True))Case-level results
Calls counts mock saves, mock provider requests or spy conversions. For invalid inputs, success means the expected rejection and callback behavior both occurred.
| Case | Fixture | Calls | Result |
|---|---|---|---|
| 1 | {"valid": true, "value": {"enabled": true, "name": "Sample"}} | 1 | Pass |
| 2 | {"valid": true, "value": {"enabled": false, "name": "Sample"}} | 1 | Pass |
| 3 | {"valid": false, "value": {"name": "Sample"}} | 0 | Pass |
| 4 | {"valid": false, "value": {"enabled": true}} | 0 | Pass |
| 5 | {"valid": false, "value": {"enabled": true, "name": 42}} | 0 | Pass |
| 6 | {"valid": false, "value": {"enabled": "true", "name": "Sample"}} | 0 | Pass |
| 7 | {"valid": false, "value": {"enabled": 1, "name": "Sample"}} | 0 | Pass |
| 8 | {"valid": false, "value": {"enabled": true, "name": ""}} | 0 | Pass |
| 9 | {"valid": false, "value": {"enabled": true, "name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}} | 0 | Pass |
| 10 | {"valid": false, "value": {"enabled": true, "extra": 1, "name": "Sample"}} | 0 | Pass |
| 11 | {"valid": false, "value": null} | 0 | Pass |
| 12 | {"valid": false, "value": []} | 0 | Pass |
Limits and evidence
This example covers a small flat Python object. It does not validate database constraints, permissions, transactions, concurrent writes or a third-party application.
Synthetic fixtures against these new Python reference functions with spy saves, mock providers and spy conversions. No network requests, real tool execution, cited-project implementation, browser behavior, model reliability or customer outcomes are measured.
Implementation SHA-256: 646e474444315a982fa8d00ce615f2f7963b89782270766f0f74085663f78f7b
Fixture SHA-256: 991519953821940ac817d60f88db1b759c58195a76940c35a63c2d6bb07fa167
Runnable example SHA-256: 49142a56de4edb11791c318ca60ee548179b2b6423c1ef30a487d6e7066bc966