import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod/v4"; import { Input } from "@/src/components/ui/input"; import { Button } from "@/src/components/ui/button"; import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage, } from "@/src/components/ui/form"; import { api } from "@/src/utils/api"; import { zodResolver } from "@hookform/resolvers/zod"; import { extractVariables, getIsCharOrUnderscore } from "@langfuse/shared"; import router from "next/router"; import { type EvalTemplate } from "@langfuse/shared"; import { ModelParameters } from "@/src/components/ModelParameters"; import { OutputSchema, type ModelParams, ZodModelConfig, } from "@langfuse/shared"; import { PromptVariableListPreview } from "@/src/features/prompts/components/PromptVariableListPreview"; import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture"; import { getFinalModelParams } from "@/src/utils/getFinalModelParams"; import { useModelParams } from "@/src/features/playground/page/hooks/useModelParams"; import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"; import { EvalReferencedEvaluators } from "@/src/features/evals/types"; import { CodeMirrorEditor } from "@/src/components/editor"; import { Card, CardContent } from "@/src/components/ui/card"; import { type RouterInput } from "@/src/utils/types"; import { useEvaluationModel } from "@/src/features/evals/hooks/useEvaluationModel"; import { Checkbox } from "@/src/components/ui/checkbox"; import { ManageDefaultEvalModel } from "@/src/features/evals/components/manage-default-eval-model"; import { DialogFooter, DialogBody } from "@/src/components/ui/dialog"; import { AlertCircle } from "lucide-react"; import { useValidateCustomModel } from "@/src/features/evals/hooks/useValidateCustomModel"; type PartialEvalTemplate = Omit< EvalTemplate, "id" | "version" | "createdAt" | "updatedAt" > & { id?: string }; export const EvalTemplateForm = (props: { projectId: string; useDialog: boolean; existingEvalTemplate?: PartialEvalTemplate; onFormSuccess?: (template?: EvalTemplate) => void; onBeforeSubmit?: ( template: RouterInput["evals"]["createTemplate"], ) => boolean; isEditing?: boolean; setIsEditing?: (isEditing: boolean) => void; preventRedirect?: boolean; cloneSourceId?: string | null; }) => { return (
); }; const selectedModelSchema = z.object({ provider: z.string().min(1, "Select a provider"), model: z.string().min(1, "Select a model"), modelParams: ZodModelConfig, }); const formSchema = z.object({ name: z.string().min(1, "Enter a name"), prompt: z .string() .min(1, "Enter a prompt") .refine((val) => { const variables = extractVariables(val); const matches = variables.map((variable) => { // check regex here if (variable.match(/^[A-Za-z_]+$/)) { return true; } return false; }); return !matches.includes(false); }, "Variables must only contain letters and underscores (_)"), variables: z.array( z.string().min(1, "Variables must have at least one character"), ), outputScore: z.string().min(1, "Enter a score function"), outputReasoning: z.string().min(1, "Enter a reasoning function"), referencedEvaluators: z .enum(EvalReferencedEvaluators) .optional() .default(EvalReferencedEvaluators.PERSIST), shouldUseDefaultModel: z.boolean().default(true), }); export type EvalTemplateFormPreFill = { name: string; prompt: string; vars: string[]; outputSchema: { score: string; reasoning: string; }; selectedModel?: { provider: string; model: string; modelParams: ModelParams & { maxTemperature: number; }; }; }; export const InnerEvalTemplateForm = (props: { projectId: string; useDialog: boolean; // pre-filled values from langfuse-defined template or template from db preFilledFormValues?: EvalTemplateFormPreFill; // template to be updated existingEvalTemplateId?: string; existingEvalTemplateName?: string; onFormSuccess?: (template?: EvalTemplate) => void; onBeforeSubmit?: (template: any) => boolean; isEditing?: boolean; setIsEditing?: (isEditing: boolean) => void; preventRedirect?: boolean; cloneSourceId?: string | null; }) => { const capture = usePostHogClientCapture(); const [formError, setFormError] = useState(null); // Determine if we should use default model or custom model // If existing template has no provider, it was using default model const isExistingUsingDefault = props.preFilledFormValues?.selectedModel ? false : true; const { data: defaultModel } = api.defaultLlmModel.fetchDefaultModel.useQuery( { projectId: props.projectId }, { enabled: !!props.projectId }, ); // updates the model params based on the pre-filled data // either form update or from langfuse-generated template const { modelParams, setModelParams, updateModelParamValue, setModelParamEnabled, availableModels, providerModelCombinations, availableProviders, } = useModelParams(); useEvaluationModel( props.projectId, setModelParams, props.preFilledFormValues?.selectedModel, ); const { isCustomModelValid } = useValidateCustomModel( availableProviders, props.preFilledFormValues?.selectedModel, ); // updates the form based on the pre-filled data // either form update or from langfuse-generated template const form = useForm({ resolver: zodResolver(formSchema), disabled: !props.isEditing, defaultValues: { name: props.existingEvalTemplateName ?? props.preFilledFormValues?.name ?? "", prompt: props.preFilledFormValues?.prompt ?? undefined, variables: props.preFilledFormValues?.vars ?? [], outputReasoning: props.preFilledFormValues ? OutputSchema.parse(props.preFilledFormValues?.outputSchema).reasoning : "One sentence reasoning for the score", outputScore: props.preFilledFormValues ? OutputSchema.parse(props.preFilledFormValues?.outputSchema).score : "Score between 0 and 1. Score 0 if false or negative and 1 if true or positive.", shouldUseDefaultModel: isExistingUsingDefault, }, }); const useDefaultModel = form.watch("shouldUseDefaultModel"); const extractedVariables = form.watch("prompt") ? extractVariables(form.watch("prompt")).filter(getIsCharOrUnderscore) : undefined; const utils = api.useUtils(); const createEvalTemplateMutation = api.evals.createTemplate.useMutation({ onSuccess: () => { utils.models.invalidate(); if ( form.getValues("referencedEvaluators") === EvalReferencedEvaluators.UPDATE && props.existingEvalTemplateId ) { showSuccessToast({ title: "Updated evaluators", description: "Updated referenced evaluators to use new template version.", }); } }, onError: (error) => setFormError(error.message), }); const evaluatorsByTemplateNameQuery = api.evals.jobConfigsByTemplateName.useQuery( { projectId: props.projectId, evalTemplateName: props.existingEvalTemplateName as string, }, { enabled: !!props.existingEvalTemplateName, }, ); useEffect(() => { if (evaluatorsByTemplateNameQuery.data) { form.setValue( "referencedEvaluators", Boolean(evaluatorsByTemplateNameQuery.data.evaluators.length) ? EvalReferencedEvaluators.UPDATE : EvalReferencedEvaluators.PERSIST, ); } }, [evaluatorsByTemplateNameQuery.data, form]); function onSubmit(values: z.infer) { capture( props.isEditing ? "eval_templates:update_form_submit" : "eval_templates:new_form_submit", ); const evalTemplate = { name: values.name, projectId: props.projectId, prompt: values.prompt, // Only include model details if not using default model provider: values.shouldUseDefaultModel ? undefined : modelParams.provider.value, model: values.shouldUseDefaultModel ? undefined : modelParams.model.value, modelParams: values.shouldUseDefaultModel ? undefined : getFinalModelParams(modelParams), vars: extractedVariables ?? [], outputSchema: { score: values.outputScore, reasoning: values.outputReasoning, }, referencedEvaluators: values.referencedEvaluators, sourceTemplateId: props.cloneSourceId ?? undefined, }; // Only validate model if not using default if (!values.shouldUseDefaultModel) { const parsedModel = selectedModelSchema.safeParse({ provider: evalTemplate.provider, model: evalTemplate.model, modelParams: evalTemplate.modelParams, }); if (!parsedModel.success) { setFormError( `${parsedModel.error.issues[0].path}: ${parsedModel.error.issues[0].message}`, ); return; } } else { if (!defaultModel) { setFormError( "No default evaluation model set. Set up default evaluation model or use a custom model", ); return; } } // Check if we need to perform any pre-submission validation or confirmation if (props.onBeforeSubmit && !props.onBeforeSubmit(evalTemplate)) { return; // Stop submission - the parent will handle it } createEvalTemplateMutation .mutateAsync(evalTemplate) .then((res) => { props.onFormSuccess?.(res); form.reset(); props.setIsEditing?.(false); if (props.preventRedirect) { return; } void router.push( `/project/${props.projectId}/evals/templates/${res.id}`, ); }) .catch((error) => { if ("message" in error && typeof error.message === "string") { setFormError(error.message as string); return; } else { setFormError(JSON.stringify(error)); console.error(error); } }); } const formBody = ( <> {!props.existingEvalTemplateId ? ( <>
( <> Name )} />
) : undefined} {/* Model Selection Section */}

Model

(
Use default evaluation model
)} /> {/* Only show model parameters if using custom model */} {!useDefaultModel && (!props.isEditing && !isCustomModelValid ? (

This evaluator is configured to use{" "} {modelParams.provider.value}s models but no API key exists. Add a key or choose another provider.

) : ( Custom model configuration

} {...{ modelParams, availableModels, providerModelCombinations, availableProviders, updateModelParamValue: updateModelParamValue, setModelParamEnabled, modelParamsDescription: "Select a model which supports function calling.", }} formDisabled={!props.isEditing} /> ))}

Prompt

( <> Evaluation prompt Define your llm-as-a-judge evaluation template. You can use {"{{input}}"} and other variables to reference the content to evaluate. )} />
( Score reasoning prompt Define how the LLM should explain its evaluation. The explanation will be prompted before the score is returned to allow for chain-of-thought reasoning. )} /> ( Score range prompt Define how the LLM should return the evaluation score in natural language. Needs to yield a numeric value. )} />
); const formFooter = (
{props.isEditing && ( )} {formError ? (

Error: {formError}

) : null}
); return (
{props.useDialog ? {formBody} : formBody} {props.useDialog ? ( {formFooter} ) : ( formFooter )}
); };