Codex
38 lines
969 B
YAML
38 lines
969 B
YAML
# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
|
|
# Replace the placeholder token before running this evaluation.
|
|
description: Local MMLU evaluation
|
|
|
|
providers:
|
|
- id: http:https://ai.zuccaro.me/api/v1
|
|
config:
|
|
headers:
|
|
Authorization: "Bearer REPLACE_WITH_OPEN_WEBUI_API_KEY"
|
|
body:
|
|
model: qwen3-llm-red-team-level-1
|
|
prompt: "{{prompt}}"
|
|
label: Open WebUI
|
|
|
|
prompts:
|
|
- |
|
|
Question: {{question}}
|
|
|
|
A) {{option_a}}
|
|
B) {{option_b}}
|
|
C) {{option_c}}
|
|
D) {{option_d}}
|
|
|
|
Answer:
|
|
|
|
defaultTest:
|
|
assert:
|
|
- type: llm-rubric
|
|
value: |
|
|
Compare the model's answer to the correct answer: {{answer}}.
|
|
The model should select the correct choice and show clear reasoning.
|
|
Score as PASS if the answer is correct.
|
|
options:
|
|
transformVars: '{ ...vars, sessionId: context.uuid }'
|
|
|
|
tests:
|
|
- huggingface://datasets/CohereLabs/Global-MMLU-Lite?config=en&split=test
|