import { type UseFormReturn, useFieldArray, useForm } from "react-hook-form"; import { Input } from "@/src/components/ui/input"; import { Button } from "@/src/components/ui/button"; import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage, } from "@/src/components/ui/form"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/src/components/ui/select"; import { zodResolver } from "@hookform/resolvers/zod"; import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs"; import { tracesTableColsWithOptions, evalTraceTableCols, evalDatasetFormFilterCols, singleFilter, availableTraceEvalVariables, datasetFormFilterColsWithOptions, availableDatasetEvalVariables, type ObservationType, LangfuseInternalTraceEnvironment, } from "@langfuse/shared"; import { z } from "zod/v4"; import { useEffect, useMemo, useState, memo } from "react"; import { api } from "@/src/utils/api"; import { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder"; import { type EvalTemplate, variableMapping } from "@langfuse/shared"; import { useRouter } from "next/router"; import { Slider } from "@/src/components/ui/slider"; import { Card } from "@/src/components/ui/card"; import { JSONView } from "@/src/components/ui/CodeJsonViewer"; import DocPopup from "@/src/components/layouts/doc-popup"; import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture"; import { cn } from "@/src/utils/tailwind"; import { Checkbox } from "@/src/components/ui/checkbox"; import { evalConfigFormSchema, type EvalFormType, isTraceOrDatasetObject, isTraceTarget, type LangfuseObject, type VariableMapping, } from "@/src/features/evals/utils/evaluator-form-utils"; import { ExecutionCountTooltip } from "@/src/features/evals/components/execution-count-tooltip"; import { TimeScopeDescription, VariableMappingDescription, } from "@/src/features/evals/components/eval-form-descriptions"; import { Suspense, lazy } from "react"; import { getDateFromOption, type TableDateRange, } from "@/src/utils/date-range-utils"; import { useEvalConfigMappingData } from "@/src/features/evals/hooks/useEvalConfigMappingData"; import { type PartialConfig } from "@/src/features/evals/types"; import { Switch } from "@/src/components/ui/switch"; import { EvaluationPromptPreview, getVariableColor, } from "@/src/features/evals/components/evaluation-prompt-preview"; import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav"; import { Skeleton } from "@/src/components/ui/skeleton"; import { DialogBody, DialogFooter } from "@/src/components/ui/dialog"; import { Tooltip, TooltipTrigger, TooltipContent, } from "@/src/components/ui/tooltip"; import { InfoIcon } from "lucide-react"; // Lazy load TracesTable const TracesTable = lazy( () => import("@/src/components/table/use-cases/traces"), ); const OUTPUT_MAPPING = [ "generation", "output", "response", "answer", "completion", ]; const INTERNAL_ENVIRONMENTS = [ LangfuseInternalTraceEnvironment.LLMJudge, "langfuse-prompt-experiments", "langfuse-evaluation", "sdk-experiment", ] as const; // Default filter for new trace evaluators - excludes internal Langfuse environments // to prevent evaluators from running on their own traces const DEFAULT_TRACE_FILTER = [ { column: "environment", operator: "none of" as const, value: [...INTERNAL_ENVIRONMENTS], type: "stringOptions" as const, }, ]; const inferDefaultMapping = ( variable: string, ): Pick => { return { langfuseObject: "trace" as const, selectedColumnId: OUTPUT_MAPPING.includes(variable.toLowerCase()) ? "output" : "input", }; }; const fieldHasJsonSelectorOption = ( selectedColumnId: string | undefined | null, ): boolean => selectedColumnId === "input" || selectedColumnId === "output" || selectedColumnId === "metadata" || selectedColumnId === "expected_output"; const TracesPreview = memo( ({ projectId, filterState, }: { projectId: string; filterState: z.infer[]; }) => { const dateRange = useMemo(() => { return { from: getDateFromOption({ filterSource: "TABLE", option: "last1Day", }), } as TableDateRange; }, []); return ( <>
Preview sample matched traces Sample over the last 24 hours that match these filters
}>
); }, ); TracesPreview.displayName = "TracesPreview"; export const InnerEvaluatorForm = (props: { projectId: string; evalTemplate: EvalTemplate; useDialog: boolean; disabled?: boolean; existingEvaluator?: PartialConfig; onFormSuccess?: () => void; shouldWrapVariables?: boolean; mode?: "create" | "edit"; hideTargetSection?: boolean; preventRedirect?: boolean; preprocessFormValues?: (values: any) => any; }) => { const [formError, setFormError] = useState(null); const capture = usePostHogClientCapture(); const [showPreview, setShowPreview] = useState(false); const router = useRouter(); const traceId = router.query.traceId as string; const form = useForm({ resolver: zodResolver(evalConfigFormSchema), disabled: props.disabled, defaultValues: { scoreName: props.existingEvaluator?.scoreName ?? `${props.evalTemplate.name}`, target: props.existingEvaluator?.targetObject ?? "trace", filter: props.existingEvaluator?.filter ? z.array(singleFilter).parse(props.existingEvaluator.filter) : // For new trace evaluators, exclude internal environments by default (props.existingEvaluator?.targetObject ?? "trace") === "trace" ? DEFAULT_TRACE_FILTER : [], mapping: props.existingEvaluator?.variableMapping ? z .array(variableMapping) .parse(props.existingEvaluator.variableMapping) : z.array(variableMapping).parse( props.evalTemplate ? props.evalTemplate.vars.map((v) => ({ templateVariable: v, langfuseObject: "trace" as const, selectedColumnId: "input", })) : [], ), sampling: props.existingEvaluator?.sampling ? props.existingEvaluator.sampling.toNumber() : 1, delay: props.existingEvaluator?.delay ? props.existingEvaluator.delay / 1000 : 30, timeScope: (props.existingEvaluator?.timeScope ?? ["NEW"]).filter( (option): option is "NEW" | "EXISTING" => ["NEW", "EXISTING"].includes(option), ), }, }) as UseFormReturn; const traceFilterOptionsResponse = api.traces.filterOptions.useQuery( { projectId: props.projectId }, { trpc: { context: { skipBatch: true } }, refetchOnMount: false, refetchOnWindowFocus: false, refetchOnReconnect: false, staleTime: Infinity, }, ); const environmentFilterOptionsResponse = api.projects.environmentFilterOptions.useQuery( { projectId: props.projectId, }, { trpc: { context: { skipBatch: true } }, refetchOnMount: false, refetchOnWindowFocus: false, refetchOnReconnect: false, staleTime: Infinity, }, ); const traceFilterOptions = useMemo(() => { // Normalize API response to match TraceOptions type (count should be number, not string) const normalized = traceFilterOptionsResponse.data ? { name: traceFilterOptionsResponse.data.name?.map((n) => ({ value: n.value, count: Number(n.count), })), scores_avg: traceFilterOptionsResponse.data.scores_avg, score_categories: traceFilterOptionsResponse.data.score_categories, tags: traceFilterOptionsResponse.data.tags?.map((t) => ({ value: t.value, })), } : {}; return { ...normalized, environment: environmentFilterOptionsResponse.data?.map((e) => ({ value: e.environment, })), }; }, [traceFilterOptionsResponse.data, environmentFilterOptionsResponse.data]); const datasets = api.datasets.allDatasetMeta.useQuery( { projectId: props.projectId, }, { trpc: { context: { skipBatch: true, }, }, refetchOnMount: false, refetchOnWindowFocus: false, refetchOnReconnect: false, staleTime: Infinity, }, ); const shouldFetch = !props.disabled && form.watch("target") === "trace"; const { observationTypeToNames, traceWithObservations, isLoading } = useEvalConfigMappingData(props.projectId, form, traceId, shouldFetch); const datasetFilterOptions = useMemo(() => { if (!datasets.data) return undefined; return { datasetId: datasets.data?.map((d) => ({ value: d.id, displayValue: d.name, })), }; }, [datasets.data]); useEffect(() => { if (form.getValues("target") === "trace" && !props.disabled) { setShowPreview(true); } else if (form.getValues("target") === "dataset") { setShowPreview(false); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [form.watch("target"), props.disabled]); useEffect(() => { if (props.evalTemplate && form.getValues("mapping").length === 0) { form.setValue( "mapping", props.evalTemplate.vars.map((v) => ({ templateVariable: v, ...inferDefaultMapping(v), })), ); form.setValue("scoreName", `${props.evalTemplate.name}`); } }, [form, props.evalTemplate]); const { fields } = useFieldArray({ control: form.control, name: "mapping", }); const utils = api.useUtils(); const createJobMutation = api.evals.createJob.useMutation({ onSuccess: () => utils.models.invalidate(), onError: (error) => setFormError(error.message), }); const updateJobMutation = api.evals.updateEvalJob.useMutation({ onSuccess: () => utils.evals.invalidate(), onError: (error) => setFormError(error.message), }); const [availableVariables, setAvailableVariables] = useState< typeof availableTraceEvalVariables | typeof availableDatasetEvalVariables >( isTraceTarget(props.existingEvaluator?.targetObject ?? "trace") ? availableTraceEvalVariables : availableDatasetEvalVariables, ); function onSubmit(values: z.infer) { capture( props.mode === "edit" ? "eval_config:update" : "eval_config:new_form_submit", ); // Apply preprocessFormValues if it exists if (props.preprocessFormValues) { values = props.preprocessFormValues(values); } const validatedFilter = z.array(singleFilter).safeParse(values.filter); if ( props.existingEvaluator?.timeScope.includes("EXISTING") && props.mode === "edit" && !values.timeScope.includes("EXISTING") ) { form.setError("timeScope", { type: "manual", message: "The evaluator ran on existing traces already. This cannot be changed anymore.", }); return; } if (form.getValues("timeScope").length === 0) { form.setError("timeScope", { type: "manual", message: "Please select at least one.", }); return; } if (validatedFilter.success === false) { form.setError("filter", { type: "manual", message: "Please fill out all filter fields", }); return; } const validatedVarMapping = z .array(variableMapping) .safeParse(values.mapping); if (validatedVarMapping.success === false) { form.setError("mapping", { type: "manual", message: "Please fill out all variable mappings", }); return; } const delay = values.delay * 1000; // convert to ms const sampling = values.sampling; const mapping = validatedVarMapping.data; const filter = validatedFilter.data; const scoreName = values.scoreName; (props.mode === "edit" && props.existingEvaluator?.id ? updateJobMutation.mutateAsync({ projectId: props.projectId, evalConfigId: props.existingEvaluator.id, config: { delay, filter, variableMapping: mapping, sampling, scoreName, timeScope: values.timeScope, }, }) : createJobMutation.mutateAsync({ projectId: props.projectId, target: values.target, evalTemplateId: props.evalTemplate.id, scoreName, filter, mapping, sampling, delay, timeScope: values.timeScope, }) ) .then(() => { props.onFormSuccess?.(); form.reset(); if (props.mode !== "edit" && !props.preventRedirect) { void router.push(`/project/${props.projectId}/evals`); } }) .catch((error) => { if ("message" in error && typeof error.message === "string") { setFormError(error.message as string); return; } else { setFormError(JSON.stringify(error)); console.error(error); } }); } const mappingControlButtons = (
{form.watch("target") === "trace" && !props.disabled && ( <> Show Preview {showPreview && (traceWithObservations ? ( `/project/${props.projectId}/evals/new?evaluator=${props.evalTemplate.id}&traceId=${entry.id}` } /> ) : (
))} )}
); const formBody = (
( Generated Score Name )} /> {!props.hideTargetSection && ( Target
( Target data{" "} {props.mode === "edit" && ( An evaluator's target data may only be configured at creation. )} { const isTrace = isTraceTarget(value); const langfuseObject: LangfuseObject = isTrace ? "trace" : "dataset_item"; const newMapping = form .getValues("mapping") .map((field) => ({ ...field, langfuseObject, })); form.setValue("filter", []); form.setValue("mapping", newMapping); setAvailableVariables( isTrace ? availableTraceEvalVariables : availableDatasetEvalVariables, ); field.onChange(value); }} > Live tracing data Dataset runs )} /> ( Evaluator runs on
{ const newValue = checked ? [...field.value, "NEW"] : field.value.filter((v) => v !== "NEW"); field.onChange(newValue); }} disabled={props.disabled} />
{ const newValue = checked ? [...field.value, "EXISTING"] : field.value.filter((v) => v !== "EXISTING"); field.onChange(newValue); }} disabled={ props.disabled || (props.mode === "edit" && field.value.includes("EXISTING")) } />
{field.value.includes("EXISTING") && !props.disabled && (props.mode === "edit" ? ( This evaluator has already run on existing{" "} {form.watch("target") === "trace" ? "traces" : "dataset run items"}{" "} once. Set up a new evaluator to re-run on existing{" "} {form.watch("target") === "trace" ? "traces" : "dataset run items"} . ) : ( ))}
)} /> ( Target filter {isTraceTarget(form.watch("target")) ? ( <>
[], ) => { field.onChange(value); if (router.query.traceId) { const { traceId, ...otherParams } = router.query; router.replace( { pathname: router.pathname, query: otherParams, }, undefined, { shallow: true }, ); } }} disabled={props.disabled} columnsWithCustomSelect={["tags"]} />
) : ( <> )}
)} /> {form.watch("target") === "trace" && !props.disabled && ( )} ( Sampling
field.onChange(value[0])} showInput={true} displayAsPercentage={true} />
)} /> ( Delay (seconds) Time between first Trace/Dataset run event and evaluation execution to ensure all data is available )} />
)}
Variable mapping
{form.watch("target") === "trace" && !props.disabled && ( Preview of the evaluation prompt with the variables replaced with the first matched trace data subject to the filters. )}
( <>
{showPreview ? ( traceWithObservations ? ( ) : (
Evaluation Prompt Preview
{mappingControlButtons}

No trace data found, please adjust filters or switch to not show preview.

) ) : ( )}
{fields.map((mappingField, index) => (
{"{{"} {mappingField.templateVariable} {"}}"}
(
)} /> {!isTraceOrDatasetObject( form.watch(`mapping.${index}.langfuseObject`), ) ? ( { const type = String( form.watch(`mapping.${index}.langfuseObject`), ).toUpperCase() as ObservationType; const nameOptions = Array.from( observationTypeToNames.get(type) ?? [], ); const isCustomOption = field.value === "custom" || (field.value && !nameOptions.includes(field.value)); return (
{isCustomOption ? (
field.onChange(e.target.value) } placeholder="Enter langfuse object name" disabled={props.disabled} />
) : ( )}
); }} /> ) : undefined} (
)} /> {fieldHasJsonSelectorOption( form.watch(`mapping.${index}.selectedColumnId`), ) ? ( (
)} /> ) : undefined}
))}
)} />
); const formFooter = (
{!props.disabled ? ( ) : null} {formError ? (

Error: {formError}

) : null}
); return (
{ e.stopPropagation(); // Prevent event bubbling to parent forms form.handleSubmit(onSubmit)(e); }} onKeyDown={(e) => { if (e.key === "Enter" && e.target instanceof HTMLInputElement) { e.preventDefault(); } }} className="flex w-full flex-col gap-4" > {props.useDialog ? {formBody} : formBody} {props.useDialog ? ( {formFooter} ) : (
{formFooter}
)}
); };