diff --git a/api/oss/src/models/api/evaluation_model.py b/api/oss/src/models/api/evaluation_model.py index 38bcfa4f11..ef25bfa140 100644 --- a/api/oss/src/models/api/evaluation_model.py +++ b/api/oss/src/models/api/evaluation_model.py @@ -292,7 +292,6 @@ class LMProvidersEnum(str, Enum): class NewEvaluation(BaseModel): - app_id: str name: Optional[str] = None revisions_ids: List[str] evaluators_configs: List[str] @@ -302,7 +301,6 @@ class NewEvaluation(BaseModel): class NewEvaluatorConfig(BaseModel): - app_id: str name: str evaluator_key: str settings_values: dict diff --git a/api/oss/src/routers/evaluators_router.py b/api/oss/src/routers/evaluators_router.py index 6df148805e..0095a37e4d 100644 --- a/api/oss/src/routers/evaluators_router.py +++ b/api/oss/src/routers/evaluators_router.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from fastapi import HTTPException, Request from fastapi.responses import JSONResponse @@ -102,8 +102,8 @@ async def evaluator_run( @router.get("/configs/", response_model=List[EvaluatorConfig]) async def get_evaluator_configs( - app_id: str, request: Request, + app_id: Optional[str] = None, ): """Endpoint to fetch evaluator configurations for a specific app. @@ -113,25 +113,33 @@ async def get_evaluator_configs( Returns: List[EvaluatorConfigDB]: A list of evaluator configuration objects. """ + project_id: Optional[str] = None + if app_id: + app_db = await db_manager.fetch_app_by_id(app_id=app_id) + project_id = str(app_db.project_id) + else: + project_id = getattr(request.state, "project_id", None) + if project_id is None: + raise HTTPException(status_code=400, detail="project_id is required") - app_db = await db_manager.fetch_app_by_id(app_id=app_id) if is_ee(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app_db.project_id), + project_id=project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + error_msg = ( + "You do not have permission to perform this action. " + "Please contact your organization admin." + ) log.error(error_msg) return JSONResponse( {"detail": error_msg}, status_code=403, ) - evaluators_configs = await evaluator_manager.get_evaluators_configs( - str(app_db.project_id) - ) + evaluators_configs = await evaluator_manager.get_evaluators_configs(project_id) return evaluators_configs @@ -181,14 +189,10 @@ async def create_new_evaluator_config( EvaluatorConfigDB: Evaluator configuration api model. """ - app_db = await db_manager.get_app_instance_by_id( - project_id=request.state.project_id, - app_id=payload.app_id, - ) if is_ee(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app_db.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_EVALUATION, ) if not has_permission: @@ -200,8 +204,7 @@ async def create_new_evaluator_config( ) evaluator_config = await evaluator_manager.create_evaluator_config( - project_id=str(app_db.project_id), - app_name=app_db.app_name, + project_id=request.state.project_id, name=payload.name, evaluator_key=payload.evaluator_key, settings_values=payload.settings_values, diff --git a/api/oss/src/services/db_manager.py b/api/oss/src/services/db_manager.py index f1557f051b..6a73cf8339 100644 --- a/api/oss/src/services/db_manager.py +++ b/api/oss/src/services/db_manager.py @@ -3034,16 +3034,14 @@ async def create_evaluator_config( project_id: str, name: str, evaluator_key: str, - app_name: Optional[str] = None, settings_values: Optional[Dict[str, Any]] = None, ) -> EvaluatorConfigDB: """Create a new evaluator configuration in the database.""" async with engine.core_session() as session: - name_suffix = f" ({app_name})" if app_name else "" new_evaluator_config = EvaluatorConfigDB( project_id=uuid.UUID(project_id), - name=f"{name}{name_suffix}", + name=name, evaluator_key=evaluator_key, settings_values=settings_values, ) diff --git a/api/oss/src/services/evaluator_manager.py b/api/oss/src/services/evaluator_manager.py index 173a3198e5..68f44ee3d4 100644 --- a/api/oss/src/services/evaluator_manager.py +++ b/api/oss/src/services/evaluator_manager.py @@ -54,7 +54,6 @@ async def get_evaluator_config(evaluator_config: EvaluatorConfig) -> EvaluatorCo async def create_evaluator_config( project_id: str, - app_name: str, name: str, evaluator_key: str, settings_values: Optional[Dict[str, Any]] = None, @@ -75,7 +74,6 @@ async def create_evaluator_config( evaluator_config = await db_manager.create_evaluator_config( project_id=project_id, - app_name=app_name, name=name, evaluator_key=evaluator_key, settings_values=settings_values, @@ -149,7 +147,6 @@ async def create_ready_to_use_evaluators(project_id: str): ), f"'name' and 'key' does not exist in the evaluator: {evaluator}" await db_manager.create_evaluator_config( project_id=project_id, - app_name=None, name=evaluator.name, evaluator_key=evaluator.key, settings_values=settings_values, diff --git a/api/pyproject.toml b/api/pyproject.toml index fdd718dc1d..7251892db6 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "api" -version = "0.56.4" +version = "0.57.0" description = "Agenta API" authors = [ { name = "Mahmoud Mabrouk", email = "mahmoud@agenta.ai" }, diff --git a/docs/docs/reference/openapi.json b/docs/docs/reference/openapi.json index 2ab9146c38..5b6005d5e7 100644 --- a/docs/docs/reference/openapi.json +++ b/docs/docs/reference/openapi.json @@ -12706,10 +12706,6 @@ }, "NewEvaluation": { "properties": { - "app_id": { - "type": "string", - "title": "App Id" - }, "revisions_ids": { "items": { "type": "string" @@ -12744,15 +12740,11 @@ } }, "type": "object", - "required": ["app_id", "revisions_ids", "evaluators_configs", "testset_id", "rate_limit"], + "required": ["revisions_ids", "evaluators_configs", "testset_id", "rate_limit"], "title": "NewEvaluation" }, "NewEvaluatorConfig": { "properties": { - "app_id": { - "type": "string", - "title": "App Id" - }, "name": { "type": "string", "title": "Name" @@ -12768,7 +12760,7 @@ } }, "type": "object", - "required": ["app_id", "name", "evaluator_key", "settings_values"], + "required": ["name", "evaluator_key", "settings_values"], "title": "NewEvaluatorConfig" }, "NewHumanEvaluation": { diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index ab5189a2a1..5cd962d47c 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "agenta" -version = "0.56.4" +version = "0.57.0" description = "The SDK for agenta is an open-source LLMOps platform." readme = "README.md" authors = [ diff --git a/web/oss/package.json b/web/oss/package.json index fe336144de..c0a70666cc 100644 --- a/web/oss/package.json +++ b/web/oss/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/oss", - "version": "0.56.4", + "version": "0.57.0", "private": true, "engines": { "node": ">=18" diff --git a/web/oss/src/components/ErrorState/index.tsx b/web/oss/src/components/ErrorState/index.tsx new file mode 100644 index 0000000000..9c5b115455 --- /dev/null +++ b/web/oss/src/components/ErrorState/index.tsx @@ -0,0 +1,33 @@ +import {Result, Button} from "antd" +import type {FC} from "react" + +interface ErrorStateProps { + title?: string + subtitle?: string + status?: "error" | "warning" | "info" | "500" + onRetry?: () => void +} + +const ErrorState: FC = ({ + title = "Something went wrong", + subtitle = "Please try again", + status = "error", + onRetry, +}) => { + return ( + + Retry + + ) : null + } + /> + ) +} + +export default ErrorState diff --git a/web/oss/src/components/Layout/Layout.tsx b/web/oss/src/components/Layout/Layout.tsx index d03daf2c1d..e1ecc2126d 100644 --- a/web/oss/src/components/Layout/Layout.tsx +++ b/web/oss/src/components/Layout/Layout.tsx @@ -131,6 +131,7 @@ const AppWithVariants = memo( className={clsx(classes.content, { "[&.ant-layout-content]:p-0 [&.ant-layout-content]:m-0": isPlayground, + "flex flex-col min-h-0 grow": isHumanEval, })} > diff --git a/web/oss/src/components/Playground/Components/MainLayout/index.tsx b/web/oss/src/components/Playground/Components/MainLayout/index.tsx index a748d2a69d..123d3d2711 100644 --- a/web/oss/src/components/Playground/Components/MainLayout/index.tsx +++ b/web/oss/src/components/Playground/Components/MainLayout/index.tsx @@ -1,5 +1,5 @@ import React from "react" -import {memo, useCallback, useRef} from "react" +import {memo, useCallback, useEffect, useRef} from "react" import {Typography, Button, Splitter} from "antd" import clsx from "clsx" @@ -75,6 +75,26 @@ const PlaygroundMainView = ({className, isLoading = false, ...divProps}: MainLay enabled: isComparisonView, }) + useEffect(() => { + if (process.env.NODE_ENV !== "production") { + console.info("[PlaygroundMainView] state", { + displayedVariants, + isComparisonView, + shouldShowVariantConfigSkeleton, + shouldShowGenerationSkeleton, + appStatus, + appStatusLoading, + }) + } + }, [ + displayedVariants, + isComparisonView, + shouldShowVariantConfigSkeleton, + shouldShowGenerationSkeleton, + appStatus, + appStatusLoading, + ]) + const handleScroll = useCallback( (index: number) => { const targetRef = variantRefs.current[index] diff --git a/web/oss/src/components/Playground/Components/PlaygroundVariantConfig/assets/Editors.tsx b/web/oss/src/components/Playground/Components/PlaygroundVariantConfig/assets/Editors.tsx index 66da4e4b76..7b7f3dcadd 100644 --- a/web/oss/src/components/Playground/Components/PlaygroundVariantConfig/assets/Editors.tsx +++ b/web/oss/src/components/Playground/Components/PlaygroundVariantConfig/assets/Editors.tsx @@ -1,5 +1,6 @@ -import {memo} from "react" +import {useEffect, memo} from "react" +import {Spin} from "antd" import clsx from "clsx" import {useVariantPrompts} from "@/oss/components/Playground/hooks/useVariantPrompts" @@ -15,7 +16,27 @@ const PlaygroundVariantConfigEditors = ({ variantId: string className?: string }) => { - const {promptIds} = useVariantPrompts(variantId) + const {promptIds, variantExists, debug} = useVariantPrompts(variantId) + + useEffect(() => { + if (process.env.NODE_ENV !== "production") { + console.info("[PlaygroundVariantConfigEditors]", { + variantId, + promptCount: promptIds.length, + variantExists, + debug, + }) + } + }, [variantId, promptIds.length, variantExists, debug]) + + if (!variantExists) { + return ( +
+ + variantId: {variantId} +
+ ) + } return (
diff --git a/web/oss/src/components/Playground/Components/PlaygroundVariantModelConfig/index.tsx b/web/oss/src/components/Playground/Components/PlaygroundVariantModelConfig/index.tsx index 4dc9cf435c..6b2c84e2e7 100644 --- a/web/oss/src/components/Playground/Components/PlaygroundVariantModelConfig/index.tsx +++ b/web/oss/src/components/Playground/Components/PlaygroundVariantModelConfig/index.tsx @@ -81,21 +81,24 @@ const PlaygroundVariantModelConfig: React.FC } }, [variantId, promptId, propertyIds]) - const displayModel = resolvedModelName ?? rawModelName ?? "Choose a model" + const displayModel = resolvedModelName ?? "Choose a model" const canOpen = (propertyIds?.length || 0) > 0 - const handleOpenChange = useCallback( - (open: boolean) => { - setIsModalOpen(open) - }, - [variantId, promptId, viewOnly, propertyIds, modelPropertyId, resolvedModelName], - ) - return ( { + if (viewOnly) { + setIsModalOpen(false) + return + } + setIsModalOpen(open) + } + : undefined + } classNames={{ root: "w-full max-w-[300px]", }} @@ -119,7 +122,7 @@ const PlaygroundVariantModelConfig: React.FC } className={className} > - diff --git a/web/oss/src/components/Playground/hooks/useVariantPrompts/index.ts b/web/oss/src/components/Playground/hooks/useVariantPrompts/index.ts index 2743d3fcfe..c7140bc149 100644 --- a/web/oss/src/components/Playground/hooks/useVariantPrompts/index.ts +++ b/web/oss/src/components/Playground/hooks/useVariantPrompts/index.ts @@ -1,8 +1,10 @@ -import {useMemo, useEffect} from "react" +import {useEffect, useMemo} from "react" -import {atom, useAtomValue} from "jotai" +import {atom, useAtomValue, useSetAtom} from "jotai" import {promptsAtomFamily} from "@/oss/state/newPlayground/core/prompts" +import {appSchemaAtom, appUriInfoAtom} from "@/oss/state/variant/atoms/fetcher" +import {derivePromptsFromSpec} from "@/oss/lib/shared/variant/transformer/transformer" import {revisionListAtom, variantByRevisionIdAtomFamily} from "../../state/atoms" @@ -14,8 +16,17 @@ import {revisionListAtom, variantByRevisionIdAtomFamily} from "../../state/atoms export function useVariantPrompts(variantId: string | undefined): { promptIds: string[] hasPrompts: boolean + variantExists: boolean + debug: { + revisionCount: number + promptCount: number + variantId?: string + } } { const revisions = useAtomValue(revisionListAtom) + const revisionCount = revisions?.length ?? 0 + const spec = useAtomValue(appSchemaAtom) + const routePath = useAtomValue(appUriInfoAtom)?.routePath // No playground clone needed: prompts are stored per-revision locally useEffect(() => { @@ -27,6 +38,11 @@ export function useVariantPrompts(variantId: string | undefined): { } }, [variantId, revisions]) + const variantAtom = useMemo(() => { + if (!variantId) return atom(null) + return variantByRevisionIdAtomFamily(variantId) + }, [variantId]) + const variantPromptIdsAtom = useMemo(() => { if (!variantId) return atom([]) @@ -52,11 +68,42 @@ export function useVariantPrompts(variantId: string | undefined): { }) }, [variantId]) - // Subscribe only to the prompt IDs, not the entire variant + const variant = useAtomValue(variantAtom) const promptIds = useAtomValue(variantPromptIdsAtom) + const setPrompts = useSetAtom(variantId ? promptsAtomFamily(variantId) : atom([])) + + useEffect(() => { + if (!variantId || !variant || !spec) return + if (promptIds.length > 0) return + const derived = derivePromptsFromSpec(variant as any, spec as any, routePath) + if (Array.isArray(derived) && derived.length > 0) { + setPrompts(derived as any) + } + }, [variantId, promptIds.length, variant, spec, routePath, setPrompts]) + + const debug = useMemo( + () => ({ + revisionCount, + promptCount: promptIds.length, + variantId, + }), + [promptIds.length, revisionCount, variantId], + ) + + useEffect(() => { + if (process.env.NODE_ENV !== "production") { + console.info("[useVariantPrompts]", { + variantId, + variantExists: Boolean(variant), + ...debug, + }) + } + }, [variantId, variant, debug]) return { promptIds, hasPrompts: promptIds.length > 0, + variantExists: Boolean(variant), + debug, } } diff --git a/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx b/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx index 8ce4840162..c5cb6a261d 100644 --- a/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx +++ b/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx @@ -51,7 +51,13 @@ export const useSidebarConfig = () => { title: "Observability", link: `${projectURL}/observability`, icon: , - divider: appId || recentlyVisitedAppId ? true : false, + }, + { + key: "project-evaluations-link", + title: "Evaluations", + link: `${projectURL}/evaluations`, + isHidden: !isDemo(), + icon: , }, { key: `${currentApp?.app_name || ""}_key`, diff --git a/web/oss/src/components/VariantNameCell/index.tsx b/web/oss/src/components/VariantNameCell/index.tsx index 464cbe0717..a86dd8610f 100644 --- a/web/oss/src/components/VariantNameCell/index.tsx +++ b/web/oss/src/components/VariantNameCell/index.tsx @@ -23,17 +23,30 @@ interface VariantNameCellProps { revisionId?: string showBadges?: boolean showStable?: boolean + revision?: Rev + revisionName?: string | null } const VariantNameCell = memo( - ({revisionId, showBadges = false, showStable = false}: VariantNameCellProps) => { - // Resolve revision and derive stable keys; keep hooks unconditional - const rev = useAtomValue(variantByRevisionIdAtomFamily(revisionId || "")) as Rev + ({ + revisionId, + revision, + revisionName, + showBadges = false, + showStable = false, + }: VariantNameCellProps) => { + const resolvedRevision = useAtomValue( + variantByRevisionIdAtomFamily(revisionId || (revision?.id ?? "")), + ) as Rev + const rev = revision ?? resolvedRevision const variantId = (rev && rev.variantId) || "" - const name = useAtomValue(variantDisplayNameByIdAtomFamily(variantId)) + + const nameFromStore = useAtomValue(variantDisplayNameByIdAtomFamily(variantId)) const latestIdForVariant = useAtomValue(latestAppRevisionIdAtom) - const deployedIn = useAtomValue(revisionDeploymentAtomFamily((rev && rev.id) || "")) + const deployedInFromStore = useAtomValue( + revisionDeploymentAtomFamily((rev && rev.id) || ""), + ) if (!rev) { return ( @@ -43,17 +56,32 @@ const VariantNameCell = memo( ) } - const isLatestRevision = rev.id === latestIdForVariant + const resolvedName = + (nameFromStore && nameFromStore !== "-" ? nameFromStore : null) || + revisionName || + (rev as any)?.variantName || + "-" + + const deployedIn = + deployedInFromStore && deployedInFromStore.length > 0 + ? deployedInFromStore + : ((rev as any)?.deployedIn as Variant["deployedIn"]) || [] + + const isLatestRevision = + typeof (rev as any)?.isLatestRevision === "boolean" + ? (rev as any).isLatestRevision + : rev.id === latestIdForVariant + const variantMin: Pick = { id: rev.id, - deployedIn: deployedIn || [], + deployedIn, isLatestRevision, } return ( import("../../Dropdown/VariantDropdown"), const store = getDefaultStore() -const CreatedByCell = memo(({record}: {record: EnhancedVariant}) => ( - -)) +const CreatedByCell = memo(({record}: {record: EnhancedVariant}) => { + const fallbackName = + [ + (record as any)?.modifiedByDisplayName, + (record as any)?.modifiedBy, + (record as any)?.modifiedById, + (record as any)?.createdByDisplayName, + (record as any)?.createdBy, + (record as any)?.createdById, + ].find((value) => typeof value === "string" && value.trim().length > 0) ?? undefined + + return ( + + ) +}) const CreatedOnCell = memo(({record}: {record: EnhancedVariant}) => { return
{record.createdAt}
}) const ModelCell = memo(({record}: {record: EnhancedVariant}) => { - // const model = useAtomValue(modelNameByRevisionIdAtomFamily(record.id)) - const name = record.modelName || store.get(modelNameByRevisionIdAtomFamily(record.id)) + const modelFromStore = store.get(modelNameByRevisionIdAtomFamily(record.id)) + const inlineConfig = (record.parameters as any)?.prompt?.llm_config || record.parameters || {} + const inlineModel = + (record.modelName as string | undefined) || + (inlineConfig && typeof inlineConfig === "object" + ? (inlineConfig as any)?.model + : undefined) + + const name = [modelFromStore, inlineModel].find( + (value) => typeof value === "string" && value.trim().length > 0 && value !== "-", + ) + return
{name || "-"}
}) @@ -115,6 +141,8 @@ export const getColumns = ({ render: (_, record) => ( 0) { const evalSlugs = filteredAnnForEval - .filter((ann) => ann.origin === 'human' || ann.channel === 'web') // Only human annotations from other users + .filter((ann) => ann.origin === "human" || ann.channel === "web") // Only human annotations from other users .map((ann) => ann.references?.evaluator?.slug) .filter(Boolean) as string[] diff --git a/web/oss/src/components/ui/UserAvatarTag.tsx b/web/oss/src/components/ui/UserAvatarTag.tsx index eef0e0d637..af644b42ee 100644 --- a/web/oss/src/components/ui/UserAvatarTag.tsx +++ b/web/oss/src/components/ui/UserAvatarTag.tsx @@ -7,15 +7,20 @@ import {variantUserDisplayNameAtomFamily} from "@/oss/state/variant/selectors/va import Avatar from "../Avatar/Avatar" -interface UserAvatarTagProps { - modifiedBy?: string - variantId?: string +interface VariantUserAvatarTagProps { + variantId: string + fallback?: string + nameOverride?: string } const VariantUserAvatarTag = memo( - ({variantId, fallback}: {variantId: string; fallback?: string}) => { + ({variantId, fallback, nameOverride}: VariantUserAvatarTagProps) => { const derivedName = useAtomValue(variantUserDisplayNameAtomFamily(variantId)) - const name = derivedName || fallback || "-" + const name = + nameOverride || + (derivedName && derivedName !== "-" ? derivedName : undefined) || + fallback || + "-" return ( {name} @@ -24,11 +29,23 @@ const VariantUserAvatarTag = memo( }, ) -const UserAvatarTag = memo(({modifiedBy, variantId}: UserAvatarTagProps) => { +interface UserAvatarTagProps { + modifiedBy?: string + variantId?: string + nameOverride?: string +} + +const UserAvatarTag = memo(({modifiedBy, variantId, nameOverride}: UserAvatarTagProps) => { if (variantId) { - return + return ( + + ) } - const name = modifiedBy || "-" + const name = nameOverride || modifiedBy || "-" return ( {name} diff --git a/web/oss/src/hooks/useAppId.ts b/web/oss/src/hooks/useAppId.ts index 56f4ffbae2..7514a791b9 100644 --- a/web/oss/src/hooks/useAppId.ts +++ b/web/oss/src/hooks/useAppId.ts @@ -1,8 +1,8 @@ -import {useRouter} from "next/router" +import {useAtomValue} from "jotai" -export const useAppId = (): string => { - const router = useRouter() - const appId = (router.query.app_id ?? "") as string +import {routerAppIdAtom} from "../state/app" - return appId +export const useAppId = (): string => { + const appId = useAtomValue(routerAppIdAtom) + return appId || "" } diff --git a/web/oss/src/lib/Types.ts b/web/oss/src/lib/Types.ts index bfacb8b036..ff3ad6c493 100644 --- a/web/oss/src/lib/Types.ts +++ b/web/oss/src/lib/Types.ts @@ -103,6 +103,7 @@ export interface ListAppsItem { app_id: string app_name: string app_type?: string + created_at?: string updated_at: string } diff --git a/web/oss/src/lib/hooks/useAnnotations/assets/helpers.ts b/web/oss/src/lib/hooks/useAnnotations/assets/helpers.ts index 69fd76edb8..600d55126a 100644 --- a/web/oss/src/lib/hooks/useAnnotations/assets/helpers.ts +++ b/web/oss/src/lib/hooks/useAnnotations/assets/helpers.ts @@ -170,19 +170,18 @@ export function attachAnnotationsToTraces(traces: any[], annotations: Annotation function attach(trace: any): any { const invocationIds = trace.invocationIds - const matchingAnnotations = annotations.filter( - (annotation: AnnotationDto) => { - // Check if annotation links to this trace via ANY link key (including "invocation" and dynamic keys like "test-xxx") - if (annotation.links && typeof annotation.links === 'object') { - const linkValues = Object.values(annotation.links) - return linkValues.some((link: any) => + const matchingAnnotations = annotations.filter((annotation: AnnotationDto) => { + // Check if annotation links to this trace via ANY link key (including "invocation" and dynamic keys like "test-xxx") + if (annotation.links && typeof annotation.links === "object") { + const linkValues = Object.values(annotation.links) + return linkValues.some( + (link: any) => link?.trace_id === (invocationIds?.trace_id || "") && - link?.span_id === (invocationIds?.span_id || "") - ) - } - return false + link?.span_id === (invocationIds?.span_id || ""), + ) } - ) + return false + }) return { ...trace, diff --git a/web/oss/src/lib/hooks/useAppVariantRevisions.ts b/web/oss/src/lib/hooks/useAppVariantRevisions.ts new file mode 100644 index 0000000000..c7dcf26f7a --- /dev/null +++ b/web/oss/src/lib/hooks/useAppVariantRevisions.ts @@ -0,0 +1,165 @@ +import {useMemo} from "react" + +import {useQuery} from "@tanstack/react-query" +import {useAtomValue} from "jotai" + +import {formatDay, parseDate} from "@/oss/lib/helpers/dateTimeHelper" +import {adaptRevisionToVariant} from "@/oss/lib/shared/variant" +import type {EnhancedVariant} from "@/oss/lib/shared/variant/transformer/types" +import type { + ParentVariantObject, + RevisionObject, +} from "@/oss/lib/shared/variant/transformer/types/variant" +import {fetchRevisions} from "@/oss/lib/shared/variant/api" +import type {Variant, ApiRevision} from "@/oss/lib/Types" +import {projectIdAtom} from "@/oss/state/project/selectors/project" +import {fetchVariants} from "@/oss/services/api" + +const REVISION_INPUT_FORMAT = "YYYY-MM-DD HH:mm:ss.SSSZ" + +const toParentVariant = (variant: Variant): ParentVariantObject => ({ + id: variant.variantId, + variantId: variant.variantId, + variantName: variant.variantName, + baseId: variant.baseId, + baseName: variant.baseName, + configName: variant.configName, + appId: variant.appId, + uri: variant.uri, + parameters: variant.parameters, + createdAtTimestamp: variant.createdAtTimestamp, + updatedAtTimestamp: variant.updatedAtTimestamp, + modifiedBy: variant.modifiedBy, +}) + +const toRevisionObject = (revision: ApiRevision): RevisionObject => { + const parsedCreated = parseDate({date: revision.created_at, inputFormat: REVISION_INPUT_FORMAT}) + return { + id: revision.id, + revision: revision.revision, + config: revision.config, + createdAt: formatDay({ + date: revision.created_at, + inputFormat: REVISION_INPUT_FORMAT, + outputFormat: "DD MMM YYYY | h:mm a", + }), + createdAtTimestamp: parsedCreated.toDate().valueOf(), + updatedAtTimestamp: parsedCreated.toDate().valueOf(), + modifiedBy: revision.modified_by, + modifiedById: revision.modified_by, + commitMessage: revision.commit_message, + } +} + +const getRevisionNumber = (revision: EnhancedVariant | RevisionObject | ApiRevision): number => { + const value = + (revision as any).revision ?? + (revision as any).revisionNumber ?? + (revision as any).revision_label ?? + (revision as any).revisionLabel + const parsed = Number(value) + return Number.isNaN(parsed) ? 0 : parsed +} + +const buildEnhancedRevisions = async ( + variant: Variant, + projectId: string, +): Promise => { + const parent = toParentVariant(variant) + const revisions = await fetchRevisions(variant.variantId, projectId) + + if (!Array.isArray(revisions) || revisions.length === 0) { + const fallback: RevisionObject = { + id: variant.id || variant.variantId, + revision: variant.revision, + config: { + config_name: variant.configName, + parameters: variant.parameters, + }, + createdAt: variant.createdAt, + createdAtTimestamp: variant.createdAtTimestamp, + updatedAtTimestamp: variant.updatedAtTimestamp, + modifiedBy: variant.modifiedBy, + modifiedById: undefined, + commitMessage: variant.commitMessage, + } + const enhanced = adaptRevisionToVariant(fallback, parent) + enhanced.uri = variant.uri + enhanced.appId = variant.appId + enhanced.baseId = variant.baseId + enhanced.baseName = variant.baseName + enhanced.configName = variant.configName + enhanced.variantName = variant.variantName + enhanced.commitMessage = variant.commitMessage + enhanced.createdAt = variant.createdAt + enhanced.updatedAt = variant.updatedAt + enhanced.createdAtTimestamp = variant.createdAtTimestamp + enhanced.updatedAtTimestamp = variant.updatedAtTimestamp + return getRevisionNumber(enhanced) > 0 ? [enhanced] : [] + } + + return revisions + .map((revision) => { + const revisionObject = toRevisionObject(revision) + const enhanced = adaptRevisionToVariant(revisionObject, parent) + // Ensure core metadata is preserved + enhanced.uri = variant.uri + enhanced.appId = variant.appId + enhanced.baseId = variant.baseId + enhanced.baseName = variant.baseName + enhanced.configName = variant.configName + enhanced.variantName = variant.variantName + enhanced.commitMessage = revision.commit_message ?? enhanced.commitMessage + enhanced.createdAt = revisionObject.createdAt + enhanced.createdAtTimestamp = + revisionObject.createdAtTimestamp ?? variant.createdAtTimestamp + enhanced.updatedAtTimestamp = + revisionObject.updatedAtTimestamp ?? variant.updatedAtTimestamp + return getRevisionNumber(enhanced) > 0 ? enhanced : null + }) + .filter((rev): rev is EnhancedVariant => Boolean(rev)) +} + +export const useAppVariantRevisions = (appId?: string | null) => { + const projectId = useAtomValue(projectIdAtom) + + const query = useQuery({ + queryKey: ["appVariantRevisions", projectId, appId], + staleTime: 60_000, + enabled: Boolean(appId && projectId), + queryFn: async () => { + if (!appId || !projectId) return [] as EnhancedVariant[] + const variants = await fetchVariants(appId, false) + if (!variants.length) return [] + + const enhancedLists = await Promise.all( + variants.map((variant) => buildEnhancedRevisions(variant, projectId)), + ) + + return enhancedLists + .flat() + .sort((a, b) => (b.updatedAtTimestamp ?? 0) - (a.updatedAtTimestamp ?? 0)) + }, + }) + + const revisionMap = useMemo(() => { + const data = query.data ?? [] + return data.reduce>((acc, revision) => { + const key = revision.variantId + if (!acc[key]) { + acc[key] = [] + } + acc[key].push(revision) + return acc + }, {}) + }, [query.data]) + + return { + variants: query.data ?? [], + revisionMap, + isLoading: query.isLoading || query.isFetching, + refetch: query.refetch, + } +} + +export default useAppVariantRevisions diff --git a/web/oss/src/lib/hooks/useEvaluatorConfigs/index.ts b/web/oss/src/lib/hooks/useEvaluatorConfigs/index.ts index 436cd23fa8..6ebc39c643 100644 --- a/web/oss/src/lib/hooks/useEvaluatorConfigs/index.ts +++ b/web/oss/src/lib/hooks/useEvaluatorConfigs/index.ts @@ -1,11 +1,12 @@ -import {useCallback} from "react" +import {useMemo, useCallback} from "react" import {useAtomValue} from "jotai" -import useSWR from "swr" +import useSWR, {SWRResponse} from "swr" import {SWRConfiguration} from "swr" import {useAppId} from "@/oss/hooks/useAppId" import {fetchAllEvaluatorConfigs} from "@/oss/services/evaluators" +import {userAtom} from "@/oss/state/profile" import {projectIdAtom} from "@/oss/state/project" import {EvaluatorConfig} from "../../Types" @@ -16,27 +17,34 @@ type EvaluatorConfigResult = Preview extends true const useEvaluatorConfigs = ({ preview, + appId: appIdOverride, ...options -}: {preview?: Preview} & SWRConfiguration) => { +}: {preview?: Preview; appId?: string | null} & SWRConfiguration) => { const projectId = useAtomValue(projectIdAtom) - const appId = useAppId() - - const fetcher = useCallback(async () => { - const data = await fetchAllEvaluatorConfigs(appId) - return data as EvaluatorConfigResult + const user = useAtomValue(userAtom) + const routeAppId = useAppId() + const appId = appIdOverride ?? routeAppId + + const fetcher = useCallback(async (): Promise => { + if (!projectId) { + return [] + } + const data = await fetchAllEvaluatorConfigs(appId, projectId) + return data }, [projectId, appId]) - return useSWR, any>( - !preview && appId && !!projectId - ? `/preview/evaluator_configs/?project_id=${projectId}&app_id=${appId}` - : null, - fetcher, - { - revalidateOnFocus: false, - shouldRetryOnError: false, - ...options, - }, - ) + const swrKey = useMemo(() => { + if (!user || preview || !projectId) return null + return ["evaluator-configs", projectId, appId ?? null] as const + }, [user, preview, projectId, appId]) + + const response = useSWR(swrKey, fetcher, { + revalidateOnFocus: false, + shouldRetryOnError: false, + ...options, + }) as SWRResponse, any> + + return response } export default useEvaluatorConfigs diff --git a/web/oss/src/lib/hooks/useFetchEvaluatorsData/index.tsx b/web/oss/src/lib/hooks/useFetchEvaluatorsData/index.tsx index acbc63fd9b..4a51520e21 100644 --- a/web/oss/src/lib/hooks/useFetchEvaluatorsData/index.tsx +++ b/web/oss/src/lib/hooks/useFetchEvaluatorsData/index.tsx @@ -20,7 +20,11 @@ interface EvaluatorsData { } const useFetchEvaluatorsData = ( - {preview, queries}: {preview?: Preview; queries?: {is_human: boolean}} = { + { + preview, + queries, + appId, + }: {preview?: Preview; queries?: {is_human: boolean}; appId?: string | null} = { preview: false as Preview, }, ): EvaluatorsData => { @@ -40,6 +44,7 @@ const useFetchEvaluatorsData = ( setEvaluatorConfigs(data) }, preview, + appId, }) const refetchAll = useCallback(async () => { diff --git a/web/oss/src/lib/hooks/useStatelessVariants/index.ts b/web/oss/src/lib/hooks/useStatelessVariants/index.ts index 176a6448f3..18e3b5b56c 100644 --- a/web/oss/src/lib/hooks/useStatelessVariants/index.ts +++ b/web/oss/src/lib/hooks/useStatelessVariants/index.ts @@ -9,6 +9,11 @@ import {useAtomValue} from "jotai/react" import type {EnhancedVariant} from "@/oss/lib/shared/variant/transformer/types" import {routerAppIdAtom} from "@/oss/state/app/atoms/fetcher" import {projectIdAtom} from "@/oss/state/project/selectors/project" +import { + projectScopedVariantsAtom, + projectVariantConfigQueryKey, + projectVariantReferenceCountAtom, +} from "@/oss/state/projectVariantConfig" import { appSchemaAtom, appUriInfoAtom, @@ -29,6 +34,7 @@ export interface VariantsBundle { uriMap: Record isLoading: boolean refetch: () => Promise + revisions: EnhancedVariant[] } export interface UseStatelessVariantsOptions { @@ -58,6 +64,11 @@ function useStatelessVariants(options: UseStatelessVariantsOptions = {}): Varian | null | undefined const enabled = !!routerAppId && routerAppId !== null && !!projectId + const isProjectScope = !enabled + const projectReferenceCount = useAtomValue(projectVariantReferenceCountAtom, { + store: rootStore, + }) + const projectScoped = useAtomValue(projectScopedVariantsAtom, {store: rootStore}) // If there are no variants and we're not actively loading variants, do not block on revisions const noVariants = useAtomValue(variantsAtom, {store: rootStore}).length === 0 @@ -70,9 +81,21 @@ function useStatelessVariants(options: UseStatelessVariantsOptions = {}): Varian const lightLoadingState = variantsLoadingEff || effectiveRevisionsPending || enhancedPendingEff const refetch = useGlobalVariantsRefetch() - + const projectRefetch = useProjectScopedVariantsRefetch() const vars = useAtomValue(sortedEnhancedRevisionsAtom, {store: rootStore}) + if (isProjectScope || projectReferenceCount > 0) { + return { + variants: projectScoped.variants, + revisionMap: projectScoped.revisionMap, + specMap: projectScoped.specMap, + uriMap: projectScoped.uriMap, + isLoading: projectScoped.isLoading, + revisions: projectScoped.revisions, + refetch: projectRefetch, + } + } + // Synthesize per-variant maps from app-level atoms to preserve API shape const variantIds = Array.from(new Set(vars.map((v: EnhancedVariant) => v.variantId))) const specMap: Record = {} @@ -104,3 +127,8 @@ export const useGlobalVariantsRefetch = () => { const queryClient = useQueryClient() return () => queryClient.invalidateQueries({queryKey: ["variants"]}) } + +export const useProjectScopedVariantsRefetch = () => { + const queryClient = useQueryClient() + return () => queryClient.invalidateQueries({queryKey: [projectVariantConfigQueryKey]}) +} diff --git a/web/oss/src/services/evaluators/index.ts b/web/oss/src/services/evaluators/index.ts index 389da961cf..cee15694d8 100644 --- a/web/oss/src/services/evaluators/index.ts +++ b/web/oss/src/services/evaluators/index.ts @@ -64,12 +64,23 @@ export const fetchAllEvaluators = async () => { } // Evaluator Configs -export const fetchAllEvaluatorConfigs = async (appId: string) => { +export const fetchAllEvaluatorConfigs = async ( + appId?: string | null, + projectIdOverride?: string | null, +) => { const tagColors = getTagColors() - const {projectId} = getProjectValues() + const {projectId: projectIdFromStore} = getProjectValues() + const projectId = projectIdOverride ?? projectIdFromStore - const response = await axios.get(`/evaluators/configs?project_id=${projectId}`, { - params: {app_id: appId}, + if (!projectId) { + return [] as EvaluatorConfig[] + } + + const response = await axios.get("/evaluators/configs", { + params: { + project_id: projectId, + ...(appId ? {app_id: appId} : {}), + }, }) const evaluatorConfigs = (response.data || []).map((item: EvaluatorConfig) => ({ ...item, @@ -80,12 +91,15 @@ export const fetchAllEvaluatorConfigs = async (appId: string) => { } export type CreateEvaluationConfigData = Omit -export const createEvaluatorConfig = async (appId: string, config: CreateEvaluationConfigData) => { +export const createEvaluatorConfig = async ( + _appId: string | null | undefined, + config: CreateEvaluationConfigData, +) => { const {projectId} = getProjectValues() + void _appId return axios.post(`/evaluators/configs?project_id=${projectId}`, { ...config, - app_id: appId, }) } diff --git a/web/oss/src/state/projectVariantConfig/atoms.ts b/web/oss/src/state/projectVariantConfig/atoms.ts new file mode 100644 index 0000000000..e75c64e2ce --- /dev/null +++ b/web/oss/src/state/projectVariantConfig/atoms.ts @@ -0,0 +1,285 @@ +import {atom, getDefaultStore} from "jotai" +import {atomFamily} from "jotai/utils" +import {atomWithQuery} from "jotai-tanstack-query" +import {queryClientAtom} from "jotai-tanstack-query" + +import {adaptRevisionToVariant} from "@/oss/lib/shared/variant" +import {EnhancedVariant} from "@/oss/lib/shared/variant/transformer/types/transformedVariant" +import { + RevisionObject, + ParentVariantObject, +} from "@/oss/lib/shared/variant/transformer/types/variant" +import {fetchVariantConfig, VariantConfigResponse} from "@/oss/services/variantConfigs/api" +import {appsAtom} from "@/oss/state/app/selectors/app" + +export interface ProjectVariantConfigKey { + projectId?: string + appId?: string + appSlug?: string + variantId?: string + variantSlug?: string + variantVersion?: number | null +} + +export const serializeProjectVariantConfigKey = (key: ProjectVariantConfigKey): string => + JSON.stringify({ + projectId: key.projectId ?? "", + appId: key.appId ?? "", + appSlug: key.appSlug ?? "", + variantId: key.variantId ?? "", + variantSlug: key.variantSlug ?? "", + variantVersion: key.variantVersion ?? null, + }) + +const parseKey = (serialized: string): ProjectVariantConfigKey => { + try { + const parsed = JSON.parse(serialized) + return { + projectId: parsed.projectId || undefined, + appId: parsed.appId || undefined, + appSlug: parsed.appSlug || undefined, + variantId: parsed.variantId || undefined, + variantSlug: parsed.variantSlug || undefined, + variantVersion: parsed.variantVersion ?? undefined, + } + } catch (error) { + console.warn("Failed to parse project variant config key", error) + return {} + } +} + +export const projectVariantConfigQueryFamily = atomFamily((serializedKey: string) => + atomWithQuery((get) => { + const params = parseKey(serializedKey) + const {projectId, appId, appSlug, variantId, variantSlug, variantVersion} = params + const enabled = Boolean(projectId) && (Boolean(variantId) || Boolean(variantSlug)) + + return { + queryKey: [ + "projectVariantConfig", + projectId ?? "", + appId ?? "", + appSlug ?? "", + variantId ?? "", + variantSlug ?? "", + variantVersion ?? "", + ], + queryFn: async () => { + if (!enabled) return null + return fetchVariantConfig({ + projectId: projectId as string, + application: { + id: appId, + slug: appSlug, + }, + variant: { + id: variantId, + slug: variantSlug, + version: variantVersion ?? null, + }, + }) + }, + refetchOnReconnect: false, + refetchOnMount: false, + refetchOnWindowFocus: false, + staleTime: 60_000, + enabled, + } + }), +) + +const projectVariantReferenceMapAtom = atom>(new Map()) + +export const projectVariantReferenceCountAtom = atom( + (get) => get(projectVariantReferenceMapAtom).size, +) + +export const setProjectVariantReferencesAtom = atom( + null, + (get, set, references: ProjectVariantConfigKey[]) => { + const next = new Map() + references.forEach((reference) => { + if (!reference?.projectId) return + const serialized = serializeProjectVariantConfigKey(reference) + next.set(serialized, { + projectId: reference.projectId, + appId: reference.appId, + appSlug: reference.appSlug, + variantId: reference.variantId, + variantSlug: reference.variantSlug, + variantVersion: reference.variantVersion ?? null, + }) + }) + set(projectVariantReferenceMapAtom, next) + }, +) + +export interface ProjectScopedVariantsState { + variants: EnhancedVariant[] + revisionMap: Record + specMap: Record + uriMap: Record + isLoading: boolean + revisions: EnhancedVariant[] +} + +const buildRevisionFromConfig = ( + projectId: string, + params: ProjectVariantConfigKey, + response: VariantConfigResponse | null, + appNameLookup: Map, +): EnhancedVariant => { + const variantId = + response?.variant_ref?.id || params.variantId || params.variantSlug || "unknown" + const variantSlug = response?.variant_ref?.slug || params.variantSlug || variantId + const revisionValue = response?.variant_ref?.version ?? params.variantVersion ?? null + const appId = response?.application_ref?.id || params.appId || "" + const appSlug = response?.application_ref?.slug || params.appSlug || "" + const appName = appNameLookup.get(appId) || appSlug + const configParams = response?.params ? {...response.params} : undefined + + const revision: RevisionObject = { + id: variantId, + revision: revisionValue ?? "", + config: { + parameters: configParams ?? {}, + }, + createdAtTimestamp: Date.now(), + updatedAtTimestamp: Date.now(), + modifiedById: "", + modifiedBy: null, + } + + const parentVariant: ParentVariantObject = { + variantId, + variantName: variantSlug, + configName: variantSlug, + appId, + baseId: variantId, + baseName: variantSlug, + parameters: configParams ?? {}, + createdAtTimestamp: Date.now(), + updatedAtTimestamp: Date.now(), + } + + const adapted = adaptRevisionToVariant(revision, parentVariant) + + return { + ...adapted, + appId, + appName, + projectId, + uri: response?.url || "", + configParams: configParams, + revisionLabel: revisionValue ?? null, + } as EnhancedVariant & { + configParams?: Record + revisionLabel?: number | string | null + } +} + +export const projectScopedVariantsAtom = atom((get) => { + const referenceMap = get(projectVariantReferenceMapAtom) + + if (referenceMap.size === 0) { + return { + variants: [], + revisionMap: {}, + specMap: {}, + uriMap: {}, + isLoading: false, + revisions: [], + } + } + + const apps = get(appsAtom) || [] + const appNameLookup = new Map(apps.map((item) => [item.app_id, item.app_name])) + + const variants: EnhancedVariant[] = [] + const revisionMap: Record = {} + const uriMap: Record = {} + const specMap: Record = {} + + let isLoading = false + + referenceMap.forEach((params, serialized) => { + if (!params.projectId) return + const queryResult = get(projectVariantConfigQueryFamily(serialized)) + if (queryResult.isPending || queryResult.isLoading) { + isLoading = true + } + const response = queryResult.data + const enhanced = buildRevisionFromConfig(params.projectId, params, response, appNameLookup) + + variants.push(enhanced) + const revisionList = revisionMap[enhanced.variantId] || [] + revisionList.push(enhanced) + revisionMap[enhanced.variantId] = revisionList + + if (enhanced.uri) { + uriMap[enhanced.variantId] = { + runtimePrefix: enhanced.uri, + } + } + }) + + return { + variants, + revisionMap, + specMap, + uriMap, + isLoading, + revisions: variants.slice(), + } +}) + +export const clearProjectVariantReferencesAtom = atom(null, (get, set) => { + if (get(projectVariantReferenceMapAtom).size === 0) return + set(projectVariantReferenceMapAtom, new Map()) +}) + +export const projectVariantConfigQueryKey = "projectVariantConfig" + +export const prefetchProjectVariantConfigs = (references: ProjectVariantConfigKey[]) => { + if (!Array.isArray(references) || references.length === 0) return + const store = getDefaultStore() + const queryClient = store.get(queryClientAtom) + references.forEach((reference) => { + if (!reference?.projectId) return + const serialized = serializeProjectVariantConfigKey(reference) + const params = parseKey(serialized) + if (!params.projectId || (!params.variantId && !params.variantSlug)) return + + queryClient + .ensureQueryData({ + queryKey: [ + projectVariantConfigQueryKey, + params.projectId ?? "", + params.appId ?? "", + params.appSlug ?? "", + params.variantId ?? "", + params.variantSlug ?? "", + params.variantVersion ?? "", + ], + queryFn: async () => { + return fetchVariantConfig({ + projectId: params.projectId as string, + application: { + id: params.appId, + slug: params.appSlug, + }, + variant: { + id: params.variantId, + slug: params.variantSlug, + version: params.variantVersion ?? null, + }, + }) + }, + }) + .catch((error) => { + if (process.env.NODE_ENV !== "production") { + console.error("[projectVariantConfig] prefetch error", error) + } + }) + }) +} diff --git a/web/oss/src/state/projectVariantConfig/index.ts b/web/oss/src/state/projectVariantConfig/index.ts new file mode 100644 index 0000000000..89eeef800a --- /dev/null +++ b/web/oss/src/state/projectVariantConfig/index.ts @@ -0,0 +1 @@ +export * from "./atoms" diff --git a/web/package.json b/web/package.json index d507c77f88..eede97a037 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "agenta-web", - "version": "0.56.4", + "version": "0.57.0", "workspaces": [ "ee", "oss",