From 4c68d9598bb8162e4e35c3fa844fa032c56e3883 Mon Sep 17 00:00:00 2001 From: GitHub CI Date: Wed, 12 Nov 2025 01:16:30 +0000 Subject: [PATCH] AGE-3375-/-bug-the-breadcrumb-is-sometimes-wrong --- api/ee/docker/Dockerfile.dev | 4 +- api/ee/docker/Dockerfile.gh | 4 +- api/ee/src/apis/fastapi/billing/router.py | 91 +- api/{oss => ee}/src/crons/queries.sh | 0 api/{oss => ee}/src/crons/queries.txt | 0 .../src/dbs/postgres/shared}/__init__.py | 0 api/ee/src/main.py | 25 +- api/ee/src/models/db_models.py | 264 ++++++ .../src/routers/evaluation_router.py | 21 +- .../src/routers/human_evaluation_router.py | 27 +- api/ee/src/services/aggregation_service.py | 135 +++ api/ee/src/services/converters.py | 191 +++- api/ee/src/services/db_manager.py | 2 +- api/ee/src/services/db_manager_ee.py | 859 +++++++++++++++++- .../src/services/evaluation_service.py | 31 +- .../src/services/llm_apps_service.py | 0 .../src/services/results_service.py | 10 +- api/ee/src/services/utils.py | 21 + .../evaluations => ee/src/tasks}/__init__.py | 0 .../ee/src/tasks/evaluations/__init__.py | 0 .../src/tasks/evaluations/batch.py | 7 +- .../src/tasks/evaluations/legacy.py | 20 +- api/{oss => ee}/src/tasks/evaluations/live.py | 0 .../tests/manual/evaluations/live.http | 0 .../manual/evaluators/human-evaluator.http | 0 api/entrypoint.py | 22 +- api/oss/docker/Dockerfile.dev | 12 +- api/oss/docker/Dockerfile.gh | 18 +- .../src/apis/fastapi/applications/router.py | 15 +- api/oss/src/apis/fastapi/evaluators/router.py | 15 +- api/oss/src/core/evaluations/service.py | 73 +- api/oss/src/models/db_models.py | 276 +----- .../src/resources/evaluators/evaluators.py | 12 +- api/oss/src/services/app_manager.py | 2 +- api/oss/src/services/converters.py | 191 ---- api/oss/src/services/db_manager.py | 788 +--------------- api/pyproject.toml | 2 +- ...ustomize-llm-as-a-judge-output-schemas.mdx | 71 -- docs/blog/main.mdx | 27 - .../05-llm-as-a-judge.mdx | 28 +- .../changelog-llm-as-a-judge-response-1.png | Bin 68321 -> 0 bytes .../changelog-llm-as-a-judge-response-2.png | Bin 108020 -> 0 bytes .../docker-compose/ee/docker-compose.dev.yml | 1 - .../docker-compose/oss/docker-compose.dev.yml | 24 - .../oss/docker-compose.gh.ssl.yml | 20 - .../docker-compose/oss/docker-compose.gh.yml | 20 - sdk/agenta/sdk/tracing/exporters.py | 2 +- sdk/agenta/sdk/workflows/handlers.py | 12 +- .../DeleteEvaluationModal.tsx | 0 .../components/DeleteEvaluationModal/types.ts | 0 .../DeploymentHistory/DeploymentHistory.tsx | 2 +- .../assets/AutoEvalRunSkeleton.tsx | 0 .../AutoEvalRun/assets/EvalNameTag.tsx | 0 .../AutoEvalRun/assets/TagWithLink.tsx | 0 .../AutoEvalRun/assets/VariantTag.tsx | 0 .../AutoEvalRun/assets/types.ts | 0 .../AutoEvalRun/assets/utils.ts | 0 .../AutoEvalRun/assets/variantUtils.ts | 0 .../components/EvalRunCompareMenu/index.tsx | 0 .../FocusDrawerContent/assets/RunOutput.tsx | 0 .../assets/RunTraceHeader.tsx | 0 .../assets/FocusDrawerContent/index.tsx | 0 .../assets/FocusDrawerContent/lib/helpers.ts | 0 .../assets/FocusDrawerHeader/index.tsx | 0 .../assets/FocusDrawerSidePanel/index.tsx | 0 .../Skeletons/FocusDrawerContentSkeleton.tsx | 0 .../Skeletons/FocusDrawerHeaderSkeleton.tsx | 0 .../FocusDrawerSidePanelSkeleton.tsx | 0 .../components/EvalRunFocusDrawer/index.tsx | 0 .../assets/EvalRunHeaderSkeleton.tsx | 0 .../components/EvalRunHeader/index.tsx | 0 .../EvalRunPromptConfigViewerSkeleton.tsx | 0 .../assets/PromptConfigCard.tsx | 0 .../EvalRunPromptConfigViewer/index.tsx | 0 .../assets/EvalRunScoreTableSkeleton.tsx | 0 .../EvalRunScoreTable/assets/TraceMetrics.tsx | 0 .../EvalRunScoreTable/assets/constants.ts | 0 .../components/EvalRunScoreTable/index.tsx | 0 .../EvalRunSelectedEvaluations/index.tsx | 0 .../index.tsx | 0 .../assets/EvalRunTestcaseViewerSkeleton.tsx | 0 .../EvalRunTestcaseViewer/index.tsx | 0 .../EvaluatorMetircsSpiderChart/index.tsx | 0 .../EvaluatorMetircsSpiderChart/types.ts | 0 .../EvaluatorMetricsChart/TimeSeriesChart.tsx | 0 .../EvaluatorMetricsChart/assets/BarChart.tsx | 0 .../assets/EvaluatorMetricsChartSkeleton.tsx | 0 .../assets/HistogramChart.tsx | 0 .../assets/LowerBand.tsx | 0 .../assets/UpperBand.tsx | 0 .../EvaluatorMetricsChart/assets/helpers.ts | 0 .../EvaluatorMetricsChart/index.tsx | 0 .../components/shared/BarChartPlaceholder.tsx | 0 .../components/shared/PlaceholderOverlay.tsx | 0 .../shared/SpiderChartPlaceholder.tsx | 0 .../EvalRunDetails/AutoEvalRun/index.tsx | 0 .../HumanEvalRun/assets/annotationUtils.ts | 0 .../HumanEvalRun/assets/helpers.ts | 0 .../HumanEvalRun/assets/optimisticUtils.ts | 0 .../HumanEvalRun/assets/runnableSelectors.ts | 0 .../HumanEvalRun/assets/stepsMetricsUtils.ts | 0 .../HumanEvalRun/assets/types.ts | 0 .../AnnotateScenarioButton/index.tsx | 0 .../AnnotateScenarioButton/types.ts | 0 .../components/EvalResultsView/index.tsx | 0 .../components/EvalRunBatchActions.tsx | 0 .../components/EvalRunName/index.tsx | 0 .../components/EvalRunScenario/index.tsx | 0 .../components/EvalRunScenario/types.ts | 0 .../EvalRunScenarioCardBody.tsx | 0 .../EvalRunScenarioCard/InvocationInputs.tsx | 0 .../InvocationResponse.tsx | 0 .../EvalRunScenarioCard/InvocationRun.tsx | 0 .../EvalRunScenarioCard/assets/KeyValue.tsx | 0 .../EvalRunScenarioCard/assets/utils.tsx | 0 .../components/EvalRunScenarioCard/index.tsx | 0 .../components/EvalRunScenarioCard/types.ts | 0 .../EvalRunScenarioCardTitle/index.tsx | 0 .../EvalRunScenarioCardTitle/types.ts | 0 .../EvalRunScenarioCards.tsx | 0 .../EvalRunScenarioCards/assets/constants.ts | 0 .../components/EvalRunScenarioFilters.tsx | 0 .../assets/InstructionButton.tsx | 0 .../Modals/InstructionModal/index.tsx | 0 .../assets/RenameEvalButton.tsx | 0 .../assets/RenameEvalModalContent.tsx | 0 .../Modals/RenameEvalModal/index.tsx | 0 .../HumanEvalRun/components/Modals/types.d.ts | 0 .../RunEvalScenarioButton/index.tsx | 0 .../components/RunEvalScenarioButton/types.ts | 0 .../ScenarioAnnotationPanel/index.tsx | 0 .../ScenarioAnnotationPanel/types.ts | 0 .../ScenarioLoadingIndicator.tsx | 0 .../assets/constants.ts | 0 .../components/SingleScenarioViewer/index.tsx | 0 .../components/SingleScenarioViewer/types.ts | 0 .../EvalRunDetails/HumanEvalRun/index.tsx | 0 .../OnlineEvalRun/OnlineUrlSync.tsx | 0 .../components/ConfigurationViewer/index.tsx | 0 .../components/TracesViewer/index.tsx | 0 .../EvalRunDetails/OnlineEvalRun/index.tsx | 0 .../src/components/EvalRunDetails/UrlSync.tsx | 0 .../assets/renderChatMessages.tsx | 0 .../components/ComparisonDataFetcher.tsx | 0 .../assets/EvalRunOverviewViewerSkeleton.tsx | 0 .../EvalRunOverviewViewer/index.tsx | 0 .../EvalRunScenarioNavigator/index.tsx | 0 .../EvalRunScenarioStatusTag/assets/index.tsx | 0 .../EvalRunScenarioStatusTag/index.tsx | 0 .../assets/constants.ts | 0 .../EvalRunScenariosViewSelector/index.tsx | 0 .../SaveDataModal/assets/SaveDataButton.tsx | 0 .../assets/SaveDataModalContent.tsx | 0 .../components/SaveDataModal/assets/types.ts | 0 .../components/SaveDataModal/index.tsx | 0 .../ComparisonScenarioTable.tsx | 0 .../ScenarioTable.tsx | 0 .../assets/ActionCell.tsx | 0 .../assets/CellComponents.tsx | 2 +- .../assets/ComparisonModeToggle.tsx | 0 .../CollapsedAnnotationValueCell.tsx | 0 .../MetricCell/CollapsedMetricValueCell.tsx | 0 .../MetricCell/CollapsedMetricsCell.tsx | 0 .../assets/MetricCell/MetricCell.tsx | 0 .../assets/MetricCell/helpers.ts | 0 .../assets/MetricCell/types.ts | 0 .../assets/ScenarioTraceSummary.tsx | 0 .../assets/StatusCell.tsx | 0 .../assets/TimestampCell.tsx | 0 ...VirtualizedScenarioTableAnnotateDrawer.tsx | 0 .../assets/atoms/evaluatorFailures.ts | 0 .../assets/constants.ts | 0 .../assets/dataSourceBuilder.ts | 0 .../assets/evaluatorNameUtils.ts | 0 .../assets/evaluatorSchemaUtils.ts | 0 .../assets/flatDataSourceBuilder.ts | 0 .../VirtualizedScenarioTable/assets/types.ts | 0 .../VirtualizedScenarioTable/assets/utils.tsx | 0 .../useExpandableComparisonDataSource.tsx | 0 .../hooks/useScrollToScenario.ts | 0 .../hooks/useTableDataSource.ts | 0 .../VirtualizedScenarioTable/index.tsx | 0 .../VirtualizedScenarioTable/types.ts | 0 .../hooks/useCachedScenarioSteps.ts | 0 .../hooks/useMetricStepError.ts | 0 .../src/components/EvalRunDetails/index.tsx | 0 .../EvalRunDetails/state/evalType.ts | 0 .../EvalRunDetails/state/focusScenarioAtom.ts | 0 .../EvalRunDetails/state/urlState.ts | 2 +- .../ABTestingEvaluationTable.tsx | 0 .../SingleModelEvaluationTable.tsx | 4 +- .../EvaluationTable/assets/styles.ts | 0 .../components/ParamsFormWithRun.tsx | 0 .../src/components/EvaluationTable/types.d.ts | 0 .../EvaluationCardView/EvaluationCard.tsx | 0 .../EvaluationChatResponse.tsx | 0 .../EvaluationCardView/EvaluationInputs.tsx | 0 .../EvaluationVariantCard.tsx | 0 .../EvaluationVotePanel.tsx | 0 .../EvaluationCardView/VariantAlphabet.tsx | 0 .../EvaluationCardView/assets/styles.ts | 0 .../Evaluations/EvaluationCardView/index.tsx | 0 .../Evaluations/EvaluationCardView/types.d.ts | 0 .../Evaluations/EvaluationErrorModal.tsx | 0 .../Evaluations/HumanEvaluationResult.tsx} | 0 .../Evaluations/ShareEvaluationModal.tsx | 0 .../assets/cells/EvaluatorTagsCell.tsx | 0 .../assets/cells/EvaluatorTypePill.tsx | 0 .../assets/cells/TableDropdownMenu/index.tsx | 0 .../assets/cells/TableDropdownMenu/types.ts | 0 .../components/Evaluators/assets/constants.ts | 0 .../Evaluators/assets/getColumns.tsx | 0 .../src/components/Evaluators/assets/types.ts | 0 .../src/components/Evaluators/assets/utils.ts | 0 .../assets/ConfigureEvaluatorSkeleton.tsx | 0 .../components/ConfigureEvaluator/index.tsx | 0 .../DeleteEvaluatorsModalContent/index.tsx | 0 .../DeleteEvaluatorsModal/index.tsx | 0 .../components/DeleteEvaluatorsModal/types.ts | 0 .../SelectEvaluatorModalContent/index.tsx | 0 .../components/SelectEvaluatorModal/index.tsx | 0 .../components/SelectEvaluatorModal/types.ts | 0 .../hooks/useEvaluatorsRegistryData.ts | 0 .../src/components/Evaluators/index.tsx | 0 .../HumanEvaluationModal.tsx | 0 .../HumanEvaluationModal/assets/styles.ts | 0 .../HumanEvaluationModal/types.d.ts | 0 .../HumanEvaluations/AbTestingEvaluation.tsx | 0 .../SingleModelEvaluation.tsx | 0 .../assets/EvaluationStatusCell.tsx | 0 .../assets/LegacyEvalResultCell.tsx | 0 .../MetricDetailsPopover/assets/ChartAxis.tsx | 0 .../assets/ChartFrame.tsx | 0 .../assets/ResponsiveFrequencyChart.tsx | 0 .../assets/ResponsiveMetricChart.tsx | 0 .../MetricDetailsPopover/assets/chartUtils.ts | 0 .../MetricDetailsPopover/assets/utils.ts | 0 .../assets/MetricDetailsPopover/index.tsx | 0 .../assets/MetricDetailsPopover/types.ts | 0 .../SingleModelEvaluationHeader/index.tsx | 0 .../assets/TableDropdownMenu/index.tsx | 0 .../assets/TableDropdownMenu/types.ts | 0 .../HumanEvaluations/assets/styles.ts | 0 .../HumanEvaluations/assets/utils.tsx | 0 .../src/components/HumanEvaluations/types.ts | 0 .../SaveTestsetModal/SaveTestsetModal.tsx | 0 .../components/SaveTestsetModal/types.d.ts | 0 .../EvaluationErrorModal.tsx | 0 .../EvaluationErrorPopover.tsx | 0 .../EvaluationErrorText.tsx | 0 .../pages/evaluations/EvaluationsView.tsx | 0 .../FilterColumns/FilterColumns.tsx | 0 .../Components/AdvancedSettings.tsx | 0 .../Components/NewEvaluationModalContent.tsx | 0 .../Components/SelectAppSection.tsx | 0 .../SelectEvaluatorSection.tsx | 0 .../Components/SelectTestsetSection.tsx | 0 .../Components/SelectVariantSection.tsx | 0 .../NewEvaluation/assets/TabLabel/index.tsx | 0 .../NewEvaluation/assets/TabLabel/types.ts | 0 .../NewEvaluation/assets/constants.ts | 0 .../NewEvaluation/assets/styles.ts | 0 .../pages/evaluations/NewEvaluation/index.tsx | 0 .../pages/evaluations/NewEvaluation/types.ts | 0 .../autoEvaluation/AutoEvaluation.tsx | 0 .../ConfigureEvaluator/AdvancedSettings.tsx | 0 .../ConfigureEvaluator/DebugSection.tsx | 0 .../ConfigureEvaluator/DynamicFormField.tsx | 0 .../EvaluatorTestcaseModal.tsx | 0 .../EvaluatorVariantModal.tsx | 0 .../JSONSchema/JSONSchemaEditor.tsx | 0 .../JSONSchema/JSONSchemaGenerator.ts | 32 +- .../ConfigureEvaluator/JSONSchema/index.ts | 0 .../ConfigureEvaluator/JSONSchema/types.ts | 0 .../ConfigureEvaluator/Messages.tsx | 0 .../ConfigureEvaluator/assets/styles.ts | 0 .../ConfigureEvaluator/index.tsx | 0 .../ConfigureEvaluator/types.ts | 0 .../ConfigureEvaluator/variantUtils.ts | 0 .../Evaluators/DeleteModal.tsx | 0 .../Evaluators/EvaluatorCard.tsx | 0 .../Evaluators/EvaluatorList.tsx | 0 .../EvaluatorsModal/Evaluators/index.tsx | 0 .../EvaluatorsModal/EvaluatorsModal.tsx | 0 .../NewEvaluator/NewEvaluatorCard.tsx | 0 .../NewEvaluator/NewEvaluatorList.tsx | 0 .../EvaluatorsModal/NewEvaluator/index.tsx | 0 .../autoEvaluation/Filters/SearchFilter.tsx | 0 .../assets/AutoEvaluationHeader.tsx | 4 +- .../autoEvaluation/assets/styles.ts | 0 .../autoEvaluation/assets/types.ts | 0 .../cellRenderers/StatusRenderer.tsx | 0 .../cellRenderers/cellRenderers.tsx | 0 .../customEvaluation/CustomEvaluation.tsx | 0 .../evaluationCompare/EvaluationCompare.tsx | 0 .../EvaluationScenarios.tsx | 0 .../onlineEvaluation/OnlineEvaluation.tsx | 0 .../OnlineEvaluationDrawer.tsx | 0 .../onlineEvaluation/assets/helpers.ts | 0 .../onlineEvaluation/assets/state.ts | 0 .../onlineEvaluation/assets/styles.ts | 0 .../components/EvaluatorDetailsPreview.tsx | 0 .../components/EvaluatorTypeTag.tsx | 0 .../components/FiltersPreview.tsx | 0 .../components/OnlineEvaluationRowActions.tsx | 0 .../components/PromptPreview.tsx | 0 .../components/QueryFiltersCell.tsx | 0 .../components/QueryFiltersSummaryCard.tsx | 0 .../components/ReadOnlyBox.tsx | 0 .../components/SamplingRateControl.tsx | 0 .../evaluations/onlineEvaluation/constants.ts | 0 .../hooks/useEvaluatorDetails.ts | 0 .../hooks/useEvaluatorSelection.tsx | 0 .../hooks/useEvaluatorTypeFromConfigs.ts | 0 .../hooks/useEvaluatorTypeMeta.ts | 0 .../hooks/useOnlineEvaluations.ts | 0 .../evaluations/onlineEvaluation/types.ts | 0 .../utils/evaluatorDetails.ts | 0 .../src/components/pages/evaluations/utils.ts | 0 web/{oss => ee}/src/contexts/RunIdContext.tsx | 0 web/ee/src/lib/helpers/evaluate.ts | 469 ++++++++++ web/ee/src/lib/helpers/hashUtils.ts | 73 ++ .../src/lib/helpers/serviceValidations.ts | 0 web/{oss => ee}/src/lib/helpers/traceUtils.ts | 2 +- .../lib/hooks/useEvalScenarioQueue/index.ts | 4 +- .../useEvalScenarioQueue/responseQueue.ts | 0 .../assets/atoms/bulkFetch.ts | 0 .../assets/atoms/cache.ts | 0 .../assets/atoms/index.ts | 0 .../assets/atoms/migrationHelper.ts | 0 .../assets/atoms/progress.ts | 0 .../assets/atoms/runScopedAtoms.ts | 0 .../assets/atoms/runScopedMetrics.ts | 2 +- .../assets/atoms/runScopedScenarios.ts | 0 .../assets/atoms/store.ts | 0 .../assets/atoms/types.ts | 0 .../assets/atoms/utils.ts | 0 .../useEvaluationRunData/assets/constants.ts | 0 .../assets/helpers/buildRunIndex.ts | 0 .../helpers/fetchScenarioListViaWorker.ts | 2 +- .../assets/helpers/fetchScenarioViaWorker.ts | 2 +- .../assets/helpers/scenarioFilters.ts | 0 .../assets/helpers/workerContext/index.ts | 0 .../assets/helpers/workerContext/types.ts | 0 .../lib/hooks/useEvaluationRunData/index.ts | 2 +- .../useEvaluationRunData/refreshLiveRun.ts | 0 .../lib/hooks/useEvaluationRunData/types.ts | 0 .../useEvalRunScenarioData.tsx | 0 .../useScenarioStepSnapshot.ts | 0 .../useEvaluationRunMetrics/assets/utils.ts | 0 .../hooks/useEvaluationRunMetrics/index.ts | 0 .../hooks/useEvaluationRunMetrics/types.ts | 0 .../useEvaluationRunScenarioSteps/types.ts | 0 .../hooks/useEvaluationRunScenarios/index.ts | 0 .../hooks/useEvaluationRunScenarios/types.ts | 0 .../src/lib/hooks/useEvaluations.ts | 0 .../lib/hooks/useInvocationResult/index.ts | 2 +- .../lib/hooks/useInvocationResult/types.ts | 0 .../usePreviewEvaluations/assets/utils.ts | 0 .../lib/hooks/usePreviewEvaluations/index.ts | 0 .../projectVariantConfigs.ts | 0 .../states/queryFilterAtoms.ts | 0 .../lib/hooks/usePreviewEvaluations/types.ts | 0 .../usePreviewRunningEvaluations/index.ts | 0 .../states/runningEvalAtom.ts | 0 .../src/lib/hooks/useRunMetricsMap/index.ts | 0 .../src/lib/metricColumnFactory.tsx | 0 web/{oss => ee}/src/lib/metricSorter.ts | 0 web/{oss => ee}/src/lib/metricUtils.ts | 0 web/{oss => ee}/src/lib/metrics/utils.ts | 0 web/{oss => ee}/src/lib/tableUtils.ts | 0 web/{oss => ee}/src/lib/types_ee.ts | 0 .../src/lib/workers}/evalRunner/bulkWorker.ts | 0 .../workers}/evalRunner/evalRunner.worker.ts | 0 .../evalRunner/fetchRunMetrics.worker.ts | 0 .../workers}/evalRunner/fetchSteps.worker.ts | 0 .../lib/workers}/evalRunner/pureEnrichment.ts | 0 .../workers}/evalRunner/runMetricsWorker.ts | 0 .../workers}/evalRunner/scenarioListWorker.ts | 0 .../src/lib/workers}/evalRunner/types.ts | 2 +- .../lib/workers}/evalRunner/workerFetch.ts | 4 +- .../evaluators/configure/[evaluator_id].tsx | 19 +- .../p/[project_id]/evaluators/index.tsx | 6 +- .../src/services/evaluationRuns/api/index.ts | 0 .../src/services/evaluationRuns/api/types.ts | 0 web/ee/src/services/evaluationRuns/utils.ts | 0 .../src/services/evaluations/api/index.ts | 0 .../src/services/evaluations/api_ee/index.ts | 2 +- .../src/services/evaluations/workerUtils.ts | 0 .../services/human-evaluations/api/index.ts | 0 .../hooks/useEvaluationResults.ts | 0 web/ee/src/services/onlineEvaluations/api.ts | 188 ++++ .../services/promptVersioning/api/index.ts | 41 + .../runMetrics/api/assets/contants.ts | 0 .../src/services/runMetrics/api/index.ts | 0 .../src/services/runMetrics/api/types.ts | 0 .../src/services/variantConfigs/api/index.ts | 0 web/{oss => ee}/src/state/url/focusDrawer.ts | 0 web/oss/next.config.ts | 12 - web/oss/package.json | 3 +- .../Sidebar/hooks/useSidebarConfig/index.tsx | 6 +- web/oss/src/lib/atoms/breadcrumb/index.ts | 19 + web/oss/src/lib/helpers/buildBreadcrumbs.ts | 2 +- web/oss/src/lib/helpers/evaluate.ts | 340 +------ web/oss/src/lib/hooks/useBreadcrumbs.ts | 40 +- .../[evaluation_id]/index.tsx | 115 --- .../apps/[app_id]/evaluations/index.tsx | 7 - .../results/[evaluation_id]/index.tsx | 23 - .../evaluations/results/compare/index.tsx | 7 - .../[evaluation_id]/index.tsx | 7 - .../apps/[app_id]/overview/index.tsx | 12 +- .../p/[project_id]/evaluations/index.tsx | 7 - .../results/[evaluation_id]/index.tsx | 23 - .../evaluations/results/compare/index.tsx | 7 - .../[evaluation_id]/index.tsx | 7 - .../evaluators/configure/[evaluator_id].tsx | 20 - .../p/[project_id]/evaluators/index.tsx | 7 - web/oss/src/services/onlineEvaluations/api.ts | 168 ---- web/pnpm-lock.yaml | 21 +- 419 files changed, 2598 insertions(+), 2464 deletions(-) rename api/{oss => ee}/src/crons/queries.sh (100%) rename api/{oss => ee}/src/crons/queries.txt (100%) rename api/{oss/src/tasks => ee/src/dbs/postgres/shared}/__init__.py (100%) rename api/{oss => ee}/src/routers/evaluation_router.py (96%) rename api/{oss => ee}/src/routers/human_evaluation_router.py (93%) create mode 100644 api/ee/src/services/aggregation_service.py rename api/{oss => ee}/src/services/evaluation_service.py (93%) rename api/{oss => ee}/src/services/llm_apps_service.py (100%) rename api/{oss => ee}/src/services/results_service.py (91%) create mode 100644 api/ee/src/services/utils.py rename api/{oss/src/tasks/evaluations => ee/src/tasks}/__init__.py (100%) rename web/oss/src/components/Evaluations/HumanEvaluationResult.tsx => api/ee/src/tasks/evaluations/__init__.py (100%) rename api/{oss => ee}/src/tasks/evaluations/batch.py (97%) rename api/{oss => ee}/src/tasks/evaluations/legacy.py (99%) rename api/{oss => ee}/src/tasks/evaluations/live.py (100%) rename api/{oss => ee}/tests/manual/evaluations/live.http (100%) rename api/{oss => ee}/tests/manual/evaluators/human-evaluator.http (100%) delete mode 100644 api/oss/src/services/converters.py delete mode 100644 docs/blog/entries/customize-llm-as-a-judge-output-schemas.mdx delete mode 100644 docs/static/images/changelog/changelog-llm-as-a-judge-response-1.png delete mode 100644 docs/static/images/changelog/changelog-llm-as-a-judge-response-2.png rename web/{oss => ee}/src/components/DeleteEvaluationModal/DeleteEvaluationModal.tsx (100%) rename web/{oss => ee}/src/components/DeleteEvaluationModal/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/AutoEvalRunSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/EvalNameTag.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/TagWithLink.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/VariantTag.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/utils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/assets/variantUtils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunCompareMenu/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunOutput.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunTraceHeader.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/lib/helpers.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerHeader/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerSidePanel/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerContentSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerHeaderSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerSidePanelSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/assets/EvalRunHeaderSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/EvalRunPromptConfigViewerSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/PromptConfigCard.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/EvalRunScoreTableSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/TraceMetrics.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/constants.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunSelectedEvaluations/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewUtilityOptions/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/assets/EvalRunTestcaseViewerSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/TimeSeriesChart.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/BarChart.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/EvaluatorMetricsChartSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/HistogramChart.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/LowerBand.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/UpperBand.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/helpers.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/shared/BarChartPlaceholder.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/shared/PlaceholderOverlay.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/components/shared/SpiderChartPlaceholder.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/AutoEvalRun/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/assets/annotationUtils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/assets/helpers.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/assets/optimisticUtils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/assets/runnableSelectors.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/assets/stepsMetricsUtils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/assets/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalResultsView/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunBatchActions.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunName/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/EvalRunScenarioCardBody.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationInputs.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationResponse.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationRun.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/KeyValue.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/utils.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/EvalRunScenarioCards.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/assets/constants.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioFilters.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/assets/InstructionButton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalButton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalModalContent.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/Modals/types.d.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/ScenarioLoadingIndicator.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/assets/constants.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/HumanEvalRun/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/OnlineEvalRun/OnlineUrlSync.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/OnlineEvalRun/components/ConfigurationViewer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/OnlineEvalRun/components/TracesViewer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/OnlineEvalRun/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/UrlSync.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/assets/renderChatMessages.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/ComparisonDataFetcher.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunOverviewViewer/assets/EvalRunOverviewViewerSkeleton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunOverviewViewer/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/assets/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/assets/constants.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataButton.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataModalContent.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/SaveDataModal/assets/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/SaveDataModal/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx (99%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ComparisonModeToggle.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedAnnotationValueCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricValueCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricsCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/helpers.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ScenarioTraceSummary.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/StatusCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/TimestampCell.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/VirtualizedScenarioTableAnnotateDrawer.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/atoms/evaluatorFailures.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/constants.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorNameUtils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorSchemaUtils.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/flatDataSourceBuilder.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useScrollToScenario.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/components/VirtualizedScenarioTable/types.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/hooks/useCachedScenarioSteps.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/hooks/useMetricStepError.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/index.tsx (100%) rename web/{oss => ee}/src/components/EvalRunDetails/state/evalType.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/state/focusScenarioAtom.ts (100%) rename web/{oss => ee}/src/components/EvalRunDetails/state/urlState.ts (97%) rename web/{oss => ee}/src/components/EvaluationTable/ABTestingEvaluationTable.tsx (100%) rename web/{oss => ee}/src/components/EvaluationTable/SingleModelEvaluationTable.tsx (99%) rename web/{oss => ee}/src/components/EvaluationTable/assets/styles.ts (100%) rename web/{oss => ee}/src/components/EvaluationTable/components/ParamsFormWithRun.tsx (100%) rename web/{oss => ee}/src/components/EvaluationTable/types.d.ts (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/EvaluationCard.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/EvaluationChatResponse.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/EvaluationInputs.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/EvaluationVariantCard.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/VariantAlphabet.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/assets/styles.ts (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationCardView/types.d.ts (100%) rename web/{oss => ee}/src/components/Evaluations/EvaluationErrorModal.tsx (100%) rename web/{oss/src/services/evaluationRuns/utils.ts => ee/src/components/Evaluations/HumanEvaluationResult.tsx} (100%) rename web/{oss => ee}/src/components/Evaluations/ShareEvaluationModal.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/assets/cells/EvaluatorTagsCell.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/assets/cells/EvaluatorTypePill.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/assets/cells/TableDropdownMenu/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/assets/cells/TableDropdownMenu/types.ts (100%) rename web/{oss => ee}/src/components/Evaluators/assets/constants.ts (100%) rename web/{oss => ee}/src/components/Evaluators/assets/getColumns.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/assets/types.ts (100%) rename web/{oss => ee}/src/components/Evaluators/assets/utils.ts (100%) rename web/{oss => ee}/src/components/Evaluators/components/ConfigureEvaluator/assets/ConfigureEvaluatorSkeleton.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/components/ConfigureEvaluator/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/components/DeleteEvaluatorsModal/assets/DeleteEvaluatorsModalContent/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/components/DeleteEvaluatorsModal/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/components/DeleteEvaluatorsModal/types.ts (100%) rename web/{oss => ee}/src/components/Evaluators/components/SelectEvaluatorModal/assets/SelectEvaluatorModalContent/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/components/SelectEvaluatorModal/index.tsx (100%) rename web/{oss => ee}/src/components/Evaluators/components/SelectEvaluatorModal/types.ts (100%) rename web/{oss => ee}/src/components/Evaluators/hooks/useEvaluatorsRegistryData.ts (100%) rename web/{oss => ee}/src/components/Evaluators/index.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluationModal/assets/styles.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluationModal/types.d.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluations/AbTestingEvaluation.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/SingleModelEvaluation.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/EvaluationStatusCell.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/LegacyEvalResultCell.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartAxis.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartFrame.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveFrequencyChart.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveMetricChart.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/chartUtils.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/MetricDetailsPopover/types.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/SingleModelEvaluationHeader/index.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/TableDropdownMenu/index.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/TableDropdownMenu/types.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/styles.ts (100%) rename web/{oss => ee}/src/components/HumanEvaluations/assets/utils.tsx (100%) rename web/{oss => ee}/src/components/HumanEvaluations/types.ts (100%) rename web/{oss => ee}/src/components/SaveTestsetModal/SaveTestsetModal.tsx (100%) rename web/{oss => ee}/src/components/SaveTestsetModal/types.d.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorModal.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorPopover.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorText.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/EvaluationsView.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/FilterColumns/FilterColumns.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/Components/NewEvaluationModalContent.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/Components/SelectAppSection.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/Components/SelectEvaluatorSection/SelectEvaluatorSection.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/Components/SelectTestsetSection.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/Components/SelectVariantSection.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/index.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/types.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/assets/constants.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/assets/styles.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/index.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/NewEvaluation/types.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/AdvancedSettings.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DebugSection.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DynamicFormField.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorTestcaseModal.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorVariantModal.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaEditor.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts (82%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/index.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/types.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/Messages.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/assets/styles.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/index.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/types.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/variantUtils.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/DeleteModal.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorCard.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorList.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/index.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorCard.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorList.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/index.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/Filters/SearchFilter.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx (99%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/assets/styles.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/autoEvaluation/assets/types.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/cellRenderers/StatusRenderer.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/customEvaluation/CustomEvaluation.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluation.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluationDrawer.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/assets/helpers.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/assets/state.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/assets/styles.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorDetailsPreview.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorTypeTag.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/FiltersPreview.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/OnlineEvaluationRowActions.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/PromptPreview.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersCell.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersSummaryCard.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/ReadOnlyBox.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/components/SamplingRateControl.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/constants.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorDetails.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorSelection.tsx (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeFromConfigs.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeMeta.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/hooks/useOnlineEvaluations.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/types.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/onlineEvaluation/utils/evaluatorDetails.ts (100%) rename web/{oss => ee}/src/components/pages/evaluations/utils.ts (100%) rename web/{oss => ee}/src/contexts/RunIdContext.tsx (100%) create mode 100644 web/ee/src/lib/helpers/evaluate.ts create mode 100644 web/ee/src/lib/helpers/hashUtils.ts rename web/{oss => ee}/src/lib/helpers/serviceValidations.ts (100%) rename web/{oss => ee}/src/lib/helpers/traceUtils.ts (98%) rename web/{oss => ee}/src/lib/hooks/useEvalScenarioQueue/index.ts (99%) rename web/{oss => ee}/src/lib/hooks/useEvalScenarioQueue/responseQueue.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/bulkFetch.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/cache.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/index.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/migrationHelper.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/progress.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedAtoms.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts (99%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/atoms/utils.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/constants.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/helpers/buildRunIndex.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts (93%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts (99%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/helpers/scenarioFilters.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/index.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/index.ts (99%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/refreshLiveRun.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunData/useScenarioStepSnapshot.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunMetrics/assets/utils.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunMetrics/index.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunMetrics/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunScenarioSteps/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunScenarios/index.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluationRunScenarios/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/useEvaluations.ts (100%) rename web/{oss => ee}/src/lib/hooks/useInvocationResult/index.ts (99%) rename web/{oss => ee}/src/lib/hooks/useInvocationResult/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewEvaluations/assets/utils.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewEvaluations/index.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewEvaluations/projectVariantConfigs.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewEvaluations/states/queryFilterAtoms.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewEvaluations/types.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewRunningEvaluations/index.ts (100%) rename web/{oss => ee}/src/lib/hooks/usePreviewRunningEvaluations/states/runningEvalAtom.ts (100%) rename web/{oss => ee}/src/lib/hooks/useRunMetricsMap/index.ts (100%) rename web/{oss => ee}/src/lib/metricColumnFactory.tsx (100%) rename web/{oss => ee}/src/lib/metricSorter.ts (100%) rename web/{oss => ee}/src/lib/metricUtils.ts (100%) rename web/{oss => ee}/src/lib/metrics/utils.ts (100%) rename web/{oss => ee}/src/lib/tableUtils.ts (100%) rename web/{oss => ee}/src/lib/types_ee.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/bulkWorker.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/evalRunner.worker.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/fetchRunMetrics.worker.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/fetchSteps.worker.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/pureEnrichment.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/runMetricsWorker.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/scenarioListWorker.ts (100%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/types.ts (89%) rename web/{oss/src/lib => ee/src/lib/workers}/evalRunner/workerFetch.ts (98%) rename web/{oss => ee}/src/services/evaluationRuns/api/index.ts (100%) rename web/{oss => ee}/src/services/evaluationRuns/api/types.ts (100%) create mode 100644 web/ee/src/services/evaluationRuns/utils.ts rename web/{oss => ee}/src/services/evaluations/api/index.ts (100%) rename web/{oss => ee}/src/services/evaluations/api_ee/index.ts (97%) rename web/{oss => ee}/src/services/evaluations/workerUtils.ts (100%) rename web/{oss => ee}/src/services/human-evaluations/api/index.ts (100%) rename web/{oss => ee}/src/services/human-evaluations/hooks/useEvaluationResults.ts (100%) create mode 100644 web/ee/src/services/onlineEvaluations/api.ts create mode 100644 web/ee/src/services/promptVersioning/api/index.ts rename web/{oss => ee}/src/services/runMetrics/api/assets/contants.ts (100%) rename web/{oss => ee}/src/services/runMetrics/api/index.ts (100%) rename web/{oss => ee}/src/services/runMetrics/api/types.ts (100%) rename web/{oss => ee}/src/services/variantConfigs/api/index.ts (100%) rename web/{oss => ee}/src/state/url/focusDrawer.ts (100%) delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/human_a_b_testing/[evaluation_id]/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/[evaluation_id]/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/compare/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/single_model_test/[evaluation_id]/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/[evaluation_id]/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/compare/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/single_model_test/[evaluation_id]/index.tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx delete mode 100644 web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx diff --git a/api/ee/docker/Dockerfile.dev b/api/ee/docker/Dockerfile.dev index 2074141a18..814cbec3ff 100644 --- a/api/ee/docker/Dockerfile.dev +++ b/api/ee/docker/Dockerfile.dev @@ -34,8 +34,8 @@ RUN cat -A /etc/cron.d/meters-cron RUN chmod +x /meters.sh \ && chmod 0644 /etc/cron.d/meters-cron -COPY ./oss/src/crons/queries.sh /queries.sh -COPY ./oss/src/crons/queries.txt /etc/cron.d/queries-cron +COPY ./ee/src/crons/queries.sh /queries.sh +COPY ./ee/src/crons/queries.txt /etc/cron.d/queries-cron RUN sed -i -e '$a\' /etc/cron.d/queries-cron RUN cat -A /etc/cron.d/queries-cron diff --git a/api/ee/docker/Dockerfile.gh b/api/ee/docker/Dockerfile.gh index ab3a06b2ff..c3652a59df 100644 --- a/api/ee/docker/Dockerfile.gh +++ b/api/ee/docker/Dockerfile.gh @@ -34,8 +34,8 @@ RUN cat -A /etc/cron.d/meters-cron RUN chmod +x /meters.sh \ && chmod 0644 /etc/cron.d/meters-cron -COPY ./oss/src/crons/queries.sh /queries.sh -COPY ./oss/src/crons/queries.txt /etc/cron.d/queries-cron +COPY ./ee/src/crons/queries.sh /queries.sh +COPY ./ee/src/crons/queries.txt /etc/cron.d/queries-cron RUN sed -i -e '$a\' /etc/cron.d/queries-cron RUN cat -A /etc/cron.d/queries-cron diff --git a/api/ee/src/apis/fastapi/billing/router.py b/api/ee/src/apis/fastapi/billing/router.py index 08762eaa76..7ac23142c5 100644 --- a/api/ee/src/apis/fastapi/billing/router.py +++ b/api/ee/src/apis/fastapi/billing/router.py @@ -824,13 +824,12 @@ async def create_portal_user_route( self, request: Request, ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.EDIT_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.create_portal( organization_id=request.state.organization_id, @@ -852,13 +851,12 @@ async def create_checkout_user_route( plan: Plan = Query(...), success_url: str = Query(...), # find a way to make this optional or moot ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.EDIT_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.create_checkout( organization_id=request.state.organization_id, @@ -884,13 +882,12 @@ async def fetch_plan_user_route( self, request: Request, ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.VIEW_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.fetch_plans( organization_id=request.state.organization_id, @@ -902,13 +899,12 @@ async def switch_plans_user_route( request: Request, plan: Plan = Query(...), ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.EDIT_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.switch_plans( organization_id=request.state.organization_id, @@ -931,13 +927,12 @@ async def fetch_subscription_user_route( self, request: Request, ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.VIEW_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.fetch_subscription( organization_id=request.state.organization_id, @@ -948,13 +943,12 @@ async def cancel_subscription_user_route( self, request: Request, ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.EDIT_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.cancel_subscription( organization_id=request.state.organization_id, @@ -974,13 +968,12 @@ async def fetch_usage_user_route( self, request: Request, ): - if is_ee(): - if not await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.VIEW_BILLING, - ): - return FORBIDDEN_RESPONSE + if not await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_BILLING, + ): + return FORBIDDEN_RESPONSE return await self.fetch_usage( organization_id=request.state.organization_id, diff --git a/api/oss/src/crons/queries.sh b/api/ee/src/crons/queries.sh similarity index 100% rename from api/oss/src/crons/queries.sh rename to api/ee/src/crons/queries.sh diff --git a/api/oss/src/crons/queries.txt b/api/ee/src/crons/queries.txt similarity index 100% rename from api/oss/src/crons/queries.txt rename to api/ee/src/crons/queries.txt diff --git a/api/oss/src/tasks/__init__.py b/api/ee/src/dbs/postgres/shared/__init__.py similarity index 100% rename from api/oss/src/tasks/__init__.py rename to api/ee/src/dbs/postgres/shared/__init__.py diff --git a/api/ee/src/main.py b/api/ee/src/main.py index 036bda6f0f..86d8ecf618 100644 --- a/api/ee/src/main.py +++ b/api/ee/src/main.py @@ -2,7 +2,12 @@ from oss.src.utils.logging import get_module_logger -from ee.src.routers import workspace_router, organization_router +from ee.src.routers import ( + workspace_router, + organization_router, + evaluation_router, + human_evaluation_router, +) from ee.src.dbs.postgres.meters.dao import MetersDAO from ee.src.dbs.postgres.subscriptions.dao import SubscriptionsDAO @@ -66,11 +71,29 @@ def extend_main(app: FastAPI): prefix="/workspaces", ) + app.include_router( + evaluation_router.router, + prefix="/evaluations", + tags=["Evaluations"], + ) + + app.include_router( + human_evaluation_router.router, + prefix="/human-evaluations", + tags=["Human-Evaluations"], + ) + # -------------------------------------------------------------------------- return app +def load_tasks(): + import ee.src.tasks.evaluations.live + import ee.src.tasks.evaluations.legacy + import ee.src.tasks.evaluations.batch + + def extend_app_schema(app: FastAPI): app.openapi()["info"]["title"] = "Agenta API" app.openapi()["info"]["description"] = "Agenta API" diff --git a/api/ee/src/models/db_models.py b/api/ee/src/models/db_models.py index b05b633659..b5a4c194da 100644 --- a/api/ee/src/models/db_models.py +++ b/api/ee/src/models/db_models.py @@ -252,3 +252,267 @@ class ProjectMemberDB(Base): class DeploymentDB(OssDeploymentDB): pass + + +class HumanEvaluationVariantDB(Base): + __tablename__ = "human_evaluation_variants" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + human_evaluation_id = Column( + UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") + ) + variant_id = Column( + UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") + ) + variant_revision_id = Column( + UUID(as_uuid=True), ForeignKey("app_variant_revisions.id", ondelete="SET NULL") + ) + + variant = relationship("AppVariantDB", backref="evaluation_variant") + variant_revision = relationship( + "AppVariantRevisionsDB", backref="evaluation_variant_revision" + ) + + +class HumanEvaluationDB(Base): + __tablename__ = "human_evaluations" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) + status = Column(String) + evaluation_type = Column(String) + testset_id = Column(UUID(as_uuid=True), ForeignKey("testsets.id")) + created_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + + testset = relationship("TestsetDB") + evaluation_variant = relationship( + "HumanEvaluationVariantDB", + cascade=CASCADE_ALL_DELETE, + backref="human_evaluation", + ) + evaluation_scenario = relationship( + "HumanEvaluationScenarioDB", + cascade=CASCADE_ALL_DELETE, + backref="evaluation_scenario", + ) + + +class HumanEvaluationScenarioDB(Base): + __tablename__ = "human_evaluations_scenarios" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) + evaluation_id = Column( + UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") + ) + inputs = Column( + mutable_json_type(dbtype=JSONB, nested=True) + ) # List of HumanEvaluationScenarioInput + outputs = Column( + mutable_json_type(dbtype=JSONB, nested=True) + ) # List of HumanEvaluationScenarioOutput + vote = Column(String) + score = Column(String) + correct_answer = Column(String) + created_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + is_pinned = Column(Boolean) + note = Column(String) + + +class EvaluationAggregatedResultDB(Base): + __tablename__ = "auto_evaluation_aggregated_results" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + evaluation_id = Column( + UUID(as_uuid=True), ForeignKey("auto_evaluations.id", ondelete="CASCADE") + ) + evaluator_config_id = Column( + UUID(as_uuid=True), + ForeignKey("auto_evaluator_configs.id", ondelete="SET NULL"), + ) + result = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result + + evaluator_config = relationship("EvaluatorConfigDB", backref="evaluator_config") + + +class EvaluationScenarioResultDB(Base): + __tablename__ = "auto_evaluation_scenario_results" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + evaluation_scenario_id = Column( + UUID(as_uuid=True), + ForeignKey("auto_evaluation_scenarios.id", ondelete="CASCADE"), + ) + evaluator_config_id = Column( + UUID(as_uuid=True), + ForeignKey("auto_evaluator_configs.id", ondelete="SET NULL"), + ) + result = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result + + +class EvaluationDB(Base): + __tablename__ = "auto_evaluations" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) + status = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result + testset_id = Column( + UUID(as_uuid=True), ForeignKey("testsets.id", ondelete="SET NULL") + ) + variant_id = Column( + UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") + ) + variant_revision_id = Column( + UUID(as_uuid=True), ForeignKey("app_variant_revisions.id", ondelete="SET NULL") + ) + average_cost = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result + total_cost = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result + average_latency = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result + created_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + + project = relationship("ee.src.models.db_models.ProjectDB") + testset = relationship("TestsetDB") + variant = relationship("AppVariantDB") + variant_revision = relationship("AppVariantRevisionsDB") + aggregated_results = relationship( + "EvaluationAggregatedResultDB", + cascade=CASCADE_ALL_DELETE, + backref="evaluation", + ) + evaluation_scenarios = relationship( + "EvaluationScenarioDB", cascade=CASCADE_ALL_DELETE, backref="evaluation" + ) + evaluator_configs = relationship( + "EvaluationEvaluatorConfigDB", + cascade=CASCADE_ALL_DELETE, + backref="evaluation", + ) + + +class EvaluationEvaluatorConfigDB(Base): + __tablename__ = "auto_evaluation_evaluator_configs" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("auto_evaluations.id", ondelete="CASCADE"), + primary_key=True, + ) + evaluator_config_id = Column( + UUID(as_uuid=True), + ForeignKey("auto_evaluator_configs.id", ondelete="SET NULL"), + primary_key=True, + ) + + +class EvaluationScenarioDB(Base): + __tablename__ = "auto_evaluation_scenarios" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) + evaluation_id = Column( + UUID(as_uuid=True), ForeignKey("auto_evaluations.id", ondelete="CASCADE") + ) + variant_id = Column( + UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") + ) + inputs = Column( + mutable_json_type(dbtype=JSONB, nested=True) + ) # List of EvaluationScenarioInput + outputs = Column( + mutable_json_type(dbtype=JSONB, nested=True) + ) # List of EvaluationScenarioOutput + correct_answers = Column( + mutable_json_type(dbtype=JSONB, nested=True) + ) # List of CorrectAnswer + is_pinned = Column(Boolean) + note = Column(String) + latency = Column(Integer) + cost = Column(Integer) + created_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + + project = relationship("ee.src.models.db_models.ProjectDB") + variant = relationship("AppVariantDB") + results = relationship( + "EvaluationScenarioResultDB", + cascade=CASCADE_ALL_DELETE, + backref="evaluation_scenario", + ) diff --git a/api/oss/src/routers/evaluation_router.py b/api/ee/src/routers/evaluation_router.py similarity index 96% rename from api/oss/src/routers/evaluation_router.py rename to api/ee/src/routers/evaluation_router.py index cac2b06523..a01679d161 100644 --- a/api/oss/src/routers/evaluation_router.py +++ b/api/ee/src/routers/evaluation_router.py @@ -7,10 +7,10 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.caching import get_cache, set_cache -from oss.src.services import converters -from oss.src.services import evaluation_service +from ee.src.services import converters +from ee.src.services import evaluation_service -from oss.src.tasks.evaluations.legacy import ( +from ee.src.tasks.evaluations.legacy import ( setup_evaluation, annotate, ) @@ -21,6 +21,7 @@ NewEvaluation, DeleteEvaluation, ) +from ee.src.services import db_manager_ee from oss.src.services import app_manager, db_manager if is_ee(): @@ -81,7 +82,7 @@ async def fetch_evaluation_ids( {"detail": error_msg}, status_code=403, ) - evaluations = await db_manager.fetch_evaluations_by_resource( + evaluations = await db_manager_ee.fetch_evaluations_by_resource( resource_type, request.state.project_id, resource_ids, @@ -135,7 +136,7 @@ async def fetch_evaluation_status( status_code=403, ) - evaluation_status = await db_manager.fetch_evaluation_status_by_id( + evaluation_status = await db_manager_ee.fetch_evaluation_status_by_id( project_id=request.state.project_id, evaluation_id=evaluation_id, ) @@ -169,7 +170,7 @@ async def fetch_evaluation_results( _type_: _description_ """ - evaluation = await db_manager.fetch_evaluation_by_id( + evaluation = await db_manager_ee.fetch_evaluation_by_id( project_id=request.state.project_id, evaluation_id=evaluation_id, ) @@ -214,7 +215,7 @@ async def fetch_evaluation_scenarios( List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluation = await db_manager.fetch_evaluation_by_id( + evaluation = await db_manager_ee.fetch_evaluation_by_id( project_id=request.state.project_id, evaluation_id=evaluation_id, ) @@ -297,7 +298,7 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ - evaluation = await db_manager.fetch_evaluation_by_id( + evaluation = await db_manager_ee.fetch_evaluation_by_id( project_id=request.state.project_id, evaluation_id=evaluation_id, ) @@ -342,7 +343,7 @@ async def delete_evaluations( A list of the deleted comparison tables' IDs. """ - evaluation = await db_manager.fetch_evaluation_by_id( + evaluation = await db_manager_ee.fetch_evaluation_by_id( project_id=request.state.project_id, evaluation_id=payload.evaluations_ids[0], ) @@ -394,7 +395,7 @@ async def fetch_evaluation_scenarios_comparison_results( """ evaluations_ids_list = evaluations_ids.split(",") - evaluation = await db_manager.fetch_evaluation_by_id( + evaluation = await db_manager_ee.fetch_evaluation_by_id( project_id=request.state.project_id, evaluation_id=evaluations_ids_list[0], ) diff --git a/api/oss/src/routers/human_evaluation_router.py b/api/ee/src/routers/human_evaluation_router.py similarity index 93% rename from api/oss/src/routers/human_evaluation_router.py rename to api/ee/src/routers/human_evaluation_router.py index 681f7588bf..eb8e7e27f8 100644 --- a/api/oss/src/routers/human_evaluation_router.py +++ b/api/ee/src/routers/human_evaluation_router.py @@ -2,10 +2,11 @@ from fastapi import HTTPException, Body, Request, status, Response from oss.src.utils.logging import get_module_logger -from oss.src.services import converters +from ee.src.services import converters from oss.src.services import db_manager -from oss.src.services import results_service -from oss.src.services import evaluation_service +from ee.src.services import db_manager_ee +from ee.src.services import results_service +from ee.src.services import evaluation_service from oss.src.utils.common import APIRouter, is_ee from oss.src.models.api.evaluation_model import ( DeleteEvaluation, @@ -18,7 +19,7 @@ NewHumanEvaluation, SimpleEvaluationOutput, ) -from oss.src.services.evaluation_service import ( +from ee.src.services.evaluation_service import ( update_human_evaluation_scenario, update_human_evaluation_service, ) @@ -129,7 +130,7 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + human_evaluation = await db_manager_ee.fetch_human_evaluation_by_id(evaluation_id) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -170,7 +171,7 @@ async def fetch_human_evaluation_scenarios( List[EvaluationScenario]: A list of evaluation scenarios. """ - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + human_evaluation = await db_manager_ee.fetch_human_evaluation_by_id(evaluation_id) if human_evaluation is None: raise HTTPException( status_code=404, @@ -215,7 +216,9 @@ async def update_human_evaluation( """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + human_evaluation = await db_manager_ee.fetch_human_evaluation_by_id( + evaluation_id + ) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -261,7 +264,7 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ - evaluation_scenario_db = await db_manager.fetch_human_evaluation_scenario_by_id( + evaluation_scenario_db = await db_manager_ee.fetch_human_evaluation_scenario_by_id( evaluation_scenario_id ) if evaluation_scenario_db is None: @@ -306,7 +309,7 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ - evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id( + evaluation_scenario = db_manager_ee.fetch_evaluation_scenario_by_id( evaluation_scenario_id ) if evaluation_scenario is None: @@ -349,7 +352,7 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ - evaluation_scenario = await db_manager.fetch_evaluation_scenario_by_id( + evaluation_scenario = await db_manager_ee.fetch_evaluation_scenario_by_id( evaluation_scenario_id ) if evaluation_scenario is None: @@ -392,7 +395,7 @@ async def fetch_results( _description_ """ - evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + evaluation = await db_manager_ee.fetch_human_evaluation_by_id(evaluation_id) if evaluation is None: raise HTTPException( status_code=404, @@ -437,7 +440,7 @@ async def delete_evaluations( A list of the deleted comparison tables' IDs. """ - evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation = await db_manager_ee.fetch_human_evaluation_by_id( payload.evaluations_ids[0] ) if is_ee(): diff --git a/api/ee/src/services/aggregation_service.py b/api/ee/src/services/aggregation_service.py new file mode 100644 index 0000000000..55a14e5f8f --- /dev/null +++ b/api/ee/src/services/aggregation_service.py @@ -0,0 +1,135 @@ +import re +import traceback +from typing import List, Optional + +from oss.src.models.shared_models import InvokationResult, Result, Error + + +def aggregate_ai_critique(results: List[Result]) -> Result: + """Aggregates the results for the ai critique evaluation. + + Args: + results (List[Result]): list of result objects + + Returns: + Result: aggregated result + """ + + try: + numeric_scores = [] + for result in results: + # Extract the first number found in the result value + match = re.search(r"\d+", result.value) + if match: + try: + score = int(match.group()) + numeric_scores.append(score) + except ValueError: + # Ignore if the extracted value is not an integer + continue + + # Calculate the average of numeric scores if any are present + average_value = ( + sum(numeric_scores) / len(numeric_scores) if numeric_scores else None + ) + return Result( + type="number", + value=average_value, + ) + except Exception as exc: + return Result( + type="error", + value=None, + error=Error(message=str(exc), stacktrace=str(traceback.format_exc())), + ) + + +def aggregate_binary(results: List[Result]) -> Result: + """Aggregates the results for the binary (auto regex) evaluation. + + Args: + results (List[Result]): list of result objects + + Returns: + Result: aggregated result + """ + + if all(isinstance(result.value, bool) for result in results): + average_value = sum(int(result.value) for result in results) / len(results) + else: + average_value = None + return Result(type="number", value=average_value) + + +def aggregate_float(results: List[Result]) -> Result: + """Aggregates the results for evaluations aside from auto regex and ai critique. + + Args: + results (List[Result]): list of result objects + + Returns: + Result: aggregated result + """ + + try: + average_value = sum(result.value for result in results) / len(results) + return Result(type="number", value=average_value) + except Exception as exc: + return Result( + type="error", + value=None, + error=Error(message=str(exc), stacktrace=str(traceback.format_exc())), + ) + + +def aggregate_float_from_llm_app_response( + invocation_results: List[InvokationResult], key: Optional[str] +) -> Result: + try: + if not key: + raise ValueError("Key is required to aggregate InvokationResult objects.") + + values = [ + getattr(inv_result, key) + for inv_result in invocation_results + if hasattr(inv_result, key) and getattr(inv_result, key) is not None + ] + + if not values: + return Result(type=key, value=None) + + average_value = sum(values) / len(values) + return Result(type=key, value=average_value) + except Exception as exc: + return Result( + type="error", + value=None, + error=Error(message=str(exc), stacktrace=str(traceback.format_exc())), + ) + + +def sum_float_from_llm_app_response( + invocation_results: List[InvokationResult], key: Optional[str] +) -> Result: + try: + if not key: + raise ValueError("Key is required to aggregate InvokationResult objects.") + + values = [ + getattr(inv_result, key) + for inv_result in invocation_results + if hasattr(inv_result, key) and getattr(inv_result, key) is not None + ] + + if not values: + return Result(type=key, value=None) + + total_value = sum(values) + + return Result(type=key, value=total_value) + except Exception as exc: + return Result( + type="error", + value=None, + error=Error(message=str(exc), stacktrace=str(traceback.format_exc())), + ) diff --git a/api/ee/src/services/converters.py b/api/ee/src/services/converters.py index 2bfc1d330b..5b120899fc 100644 --- a/api/ee/src/services/converters.py +++ b/api/ee/src/services/converters.py @@ -3,9 +3,28 @@ from datetime import datetime, timezone from oss.src.services import db_manager +from oss.src.models.api.evaluation_model import ( + CorrectAnswer, + Evaluation, + HumanEvaluation, + EvaluationScenario, + SimpleEvaluationOutput, + EvaluationScenarioInput, + HumanEvaluationScenario, + EvaluationScenarioOutput, +) from ee.src.services import db_manager_ee -from ee.src.models.api.workspace_models import WorkspaceRole, WorkspaceResponse +from ee.src.models.api.workspace_models import ( + WorkspaceRole, + WorkspaceResponse, +) from ee.src.models.shared_models import Permission +from ee.src.models.db_models import ( + EvaluationDB, + HumanEvaluationDB, + EvaluationScenarioDB, + HumanEvaluationScenarioDB, +) from oss.src.models.db_models import WorkspaceDB @@ -130,3 +149,173 @@ def get_all_workspace_permissions_by_role(role_name: str) -> Dict[str, List[Any] getattr(WorkspaceRole, role_name.upper()) ) return workspace_permissions + + +async def human_evaluation_db_to_simple_evaluation_output( + human_evaluation_db: HumanEvaluationDB, +) -> SimpleEvaluationOutput: + evaluation_variants = await db_manager_ee.fetch_human_evaluation_variants( + human_evaluation_id=str(human_evaluation_db.id) + ) + return SimpleEvaluationOutput( + id=str(human_evaluation_db.id), + app_id=str(human_evaluation_db.app_id), + project_id=str(human_evaluation_db.project_id), + status=human_evaluation_db.status, # type: ignore + evaluation_type=human_evaluation_db.evaluation_type, # type: ignore + variant_ids=[ + str(evaluation_variant.variant_id) + for evaluation_variant in evaluation_variants + ], + ) + + +async def evaluation_db_to_pydantic( + evaluation_db: EvaluationDB, +) -> Evaluation: + variant_name = ( + evaluation_db.variant.variant_name + if evaluation_db.variant.variant_name + else str(evaluation_db.variant_id) + ) + aggregated_results = aggregated_result_of_evaluation_to_pydantic( + evaluation_db.aggregated_results + ) + + return Evaluation( + id=str(evaluation_db.id), + app_id=str(evaluation_db.app_id), + project_id=str(evaluation_db.project_id), + status=evaluation_db.status, + variant_ids=[str(evaluation_db.variant_id)], + variant_revision_ids=[str(evaluation_db.variant_revision_id)], + revisions=[str(evaluation_db.variant_revision.revision)], + variant_names=[variant_name], + testset_id=str(evaluation_db.testset_id), + testset_name=evaluation_db.testset.name, + aggregated_results=aggregated_results, + created_at=str(evaluation_db.created_at), + updated_at=str(evaluation_db.updated_at), + average_cost=evaluation_db.average_cost, + total_cost=evaluation_db.total_cost, + average_latency=evaluation_db.average_latency, + ) + + +async def human_evaluation_db_to_pydantic( + evaluation_db: HumanEvaluationDB, +) -> HumanEvaluation: + evaluation_variants = await db_manager_ee.fetch_human_evaluation_variants( + human_evaluation_id=str(evaluation_db.id) # type: ignore + ) + + revisions = [] + variants_ids = [] + variants_names = [] + variants_revision_ids = [] + for evaluation_variant in evaluation_variants: + variant_name = ( + evaluation_variant.variant.variant_name + if isinstance(evaluation_variant.variant_id, uuid.UUID) + else str(evaluation_variant.variant_id) + ) + variants_names.append(str(variant_name)) + variants_ids.append(str(evaluation_variant.variant_id)) + variant_revision = ( + str(evaluation_variant.variant_revision.revision) + if isinstance(evaluation_variant.variant_revision_id, uuid.UUID) + else " None" + ) + revisions.append(variant_revision) + variants_revision_ids.append(str(evaluation_variant.variant_revision_id)) + + return HumanEvaluation( + id=str(evaluation_db.id), + app_id=str(evaluation_db.app_id), + project_id=str(evaluation_db.project_id), + status=evaluation_db.status, # type: ignore + evaluation_type=evaluation_db.evaluation_type, # type: ignore + variant_ids=variants_ids, + variant_names=variants_names, + testset_id=str(evaluation_db.testset_id), + testset_name=evaluation_db.testset.name, + variants_revision_ids=variants_revision_ids, + revisions=revisions, + created_at=str(evaluation_db.created_at), # type: ignore + updated_at=str(evaluation_db.updated_at), # type: ignore + ) + + +def human_evaluation_scenario_db_to_pydantic( + evaluation_scenario_db: HumanEvaluationScenarioDB, evaluation_id: str +) -> HumanEvaluationScenario: + return HumanEvaluationScenario( + id=str(evaluation_scenario_db.id), + evaluation_id=evaluation_id, + inputs=evaluation_scenario_db.inputs, # type: ignore + outputs=evaluation_scenario_db.outputs, # type: ignore + vote=evaluation_scenario_db.vote, # type: ignore + score=evaluation_scenario_db.score, # type: ignore + correct_answer=evaluation_scenario_db.correct_answer, # type: ignore + is_pinned=evaluation_scenario_db.is_pinned or False, # type: ignore + note=evaluation_scenario_db.note or "", # type: ignore + ) + + +def aggregated_result_of_evaluation_to_pydantic( + evaluation_aggregated_results: List, +) -> List[dict]: + transformed_results = [] + for aggregated_result in evaluation_aggregated_results: + evaluator_config_dict = ( + { + "id": str(aggregated_result.evaluator_config.id), + "name": aggregated_result.evaluator_config.name, + "evaluator_key": aggregated_result.evaluator_config.evaluator_key, + "settings_values": aggregated_result.evaluator_config.settings_values, + "created_at": str(aggregated_result.evaluator_config.created_at), + "updated_at": str(aggregated_result.evaluator_config.updated_at), + } + if isinstance(aggregated_result.evaluator_config_id, uuid.UUID) + else None + ) + transformed_results.append( + { + "evaluator_config": ( + {} if evaluator_config_dict is None else evaluator_config_dict + ), + "result": aggregated_result.result, + } + ) + return transformed_results + + +async def evaluation_scenario_db_to_pydantic( + evaluation_scenario_db: EvaluationScenarioDB, evaluation_id: str +) -> EvaluationScenario: + scenario_results = [ + { + "evaluator_config": str(scenario_result.evaluator_config_id), + "result": scenario_result.result, + } + for scenario_result in evaluation_scenario_db.results + ] + return EvaluationScenario( + id=str(evaluation_scenario_db.id), + evaluation_id=evaluation_id, + inputs=[ + EvaluationScenarioInput(**scenario_input) # type: ignore + for scenario_input in evaluation_scenario_db.inputs + ], + outputs=[ + EvaluationScenarioOutput(**scenario_output) # type: ignore + for scenario_output in evaluation_scenario_db.outputs + ], + correct_answers=[ + CorrectAnswer(**correct_answer) # type: ignore + for correct_answer in evaluation_scenario_db.correct_answers + ], + is_pinned=evaluation_scenario_db.is_pinned or False, # type: ignore + note=evaluation_scenario_db.note or "", # type: ignore + results=scenario_results, # type: ignore + ) diff --git a/api/ee/src/services/db_manager.py b/api/ee/src/services/db_manager.py index 0d8e36c384..1091c4f736 100644 --- a/api/ee/src/services/db_manager.py +++ b/api/ee/src/services/db_manager.py @@ -1,7 +1,7 @@ import uuid from oss.src.dbs.postgres.shared.engine import engine -from ee.src.models.db_models import DeploymentDB +from ee.src.models.db_models import DeploymentDB_ as DeploymentDB async def create_deployment( diff --git a/api/ee/src/services/db_manager_ee.py b/api/ee/src/services/db_manager_ee.py index bc174918b6..b101f7b68d 100644 --- a/api/ee/src/services/db_manager_ee.py +++ b/api/ee/src/services/db_manager_ee.py @@ -1,12 +1,14 @@ import uuid -from typing import List, Union, NoReturn, Optional, Tuple +from typing import List, Dict, Union, Any, NoReturn, Optional, Tuple import sendgrid from fastapi import HTTPException +from sendgrid.helpers.mail import Mail +from sqlalchemy import func, asc from sqlalchemy.future import select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import joinedload, load_only +from sqlalchemy.orm import joinedload, load_only, aliased from sqlalchemy.exc import NoResultFound, MultipleResultsFound from oss.src.utils.logging import get_module_logger @@ -29,14 +31,36 @@ from ee.src.models.db_models import ( ProjectDB, WorkspaceDB, + EvaluationDB, OrganizationDB, ProjectMemberDB, WorkspaceMemberDB, + HumanEvaluationDB, OrganizationMemberDB, + EvaluationScenarioDB, + HumanEvaluationScenarioDB, + HumanEvaluationVariantDB, + EvaluationScenarioResultDB, + EvaluationEvaluatorConfigDB, + EvaluationAggregatedResultDB, ) from oss.src.models.db_models import ( + AppVariantDB, UserDB, + AppDB, + TestsetDB, InvitationDB, + EvaluatorConfigDB, + AppVariantRevisionsDB, +) +from oss.src.models.shared_models import ( + Result, + CorrectAnswer, + AggregatedResult, + EvaluationScenarioResult, + EvaluationScenarioInput, + EvaluationScenarioOutput, + HumanEvaluationScenarioInput, ) from ee.src.services.converters import get_workspace_in_format from ee.src.services.selectors import get_org_default_workspace @@ -1183,6 +1207,85 @@ async def get_all_workspace_roles() -> List[WorkspaceRole]: return workspace_roles +# async def get_project_id_from_db_entity( +# object_id: str, type: str, project_id: str +# ) -> dict: +# """ +# Get the project id of the object. + +# Args: +# object_id (str): The ID of the object. +# type (str): The type of the object. + +# Returns: +# dict: The project_id of the object. + +# Raises: +# ValueError: If the object type is unknown. +# Exception: If there is an error retrieving the project_id. +# """ +# try: +# if type == "app": +# app = await db_manager.fetch_app_by_id(object_id) +# project_id = app.project_id + +# elif type == "app_variant": +# app_variant = await db_manager.fetch_app_variant_by_id(object_id) +# project_id = app_variant.project_id + +# elif type == "base": +# base = await db_manager.fetch_base_by_id(object_id) +# project_id = base.project_id + +# elif type == "deployment": +# deployment = await db_manager.get_deployment_by_id(object_id) +# project_id = deployment.project_id + +# elif type == "testset": +# testset = await db_manager.fetch_testset_by_id(object_id) +# project_id = testset.project_id + +# elif type == "evaluation": +# evaluation = await db_manager.fetch_evaluation_by_id(object_id) +# project_id = evaluation.project_id + +# elif type == "evaluation_scenario": +# evaluation_scenario = await db_manager.fetch_evaluation_scenario_by_id( +# object_id +# ) +# project_id = evaluation_scenario.project_id + +# elif type == "evaluator_config": +# evaluator_config = await db_manager.fetch_evaluator_config(object_id) +# project_id = evaluator_config.project_id + +# elif type == "human_evaluation": +# human_evaluation = await db_manager.fetch_human_evaluation_by_id(object_id) +# project_id = human_evaluation.project_id + +# elif type == "human_evaluation_scenario": +# human_evaluation_scenario = ( +# await db_manager.fetch_human_evaluation_scenario_by_id(object_id) +# ) +# project_id = human_evaluation_scenario.project_id + +# elif type == "human_evaluation_scenario_by_evaluation_id": +# human_evaluation_scenario_by_evaluation = ( +# await db_manager.fetch_human_evaluation_scenario_by_evaluation_id( +# object_id +# ) +# ) +# project_id = human_evaluation_scenario_by_evaluation.project_id + +# else: +# raise ValueError(f"Unknown object type: {type}") + +# return str(project_id) + +# except Exception as e: +# raise e + + async def add_user_to_organization( organization_id: str, user_id: str, @@ -1274,3 +1377,755 @@ async def add_user_to_project( ) await session.commit() + + +async def fetch_evaluation_status_by_id( + project_id: str, + evaluation_id: str, +) -> Optional[str]: + """Fetch only the status of an evaluation by its ID.""" + assert evaluation_id is not None, "evaluation_id cannot be None" + + async with engine.core_session() as session: + query = ( + select(EvaluationDB) + .filter_by(project_id=project_id, id=uuid.UUID(evaluation_id)) + .options(load_only(EvaluationDB.status)) + ) + + result = await session.execute(query) + evaluation = result.scalars().first() + return evaluation.status if evaluation else None + + +async def fetch_evaluation_by_id( + project_id: str, + evaluation_id: str, +) -> Optional[EvaluationDB]: + """Fetches a evaluation by its ID. + + Args: + evaluation_id (str): The ID of the evaluation to fetch. + + Returns: + EvaluationDB: The fetched evaluation, or None if no evaluation was found. + """ + + assert evaluation_id is not None, "evaluation_id cannot be None" + async with engine.core_session() as session: + base_query = select(EvaluationDB).filter_by( + project_id=project_id, + id=uuid.UUID(evaluation_id), + ) + query = base_query.options( + joinedload(EvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore + ) + + result = await session.execute( + query.options( + joinedload(EvaluationDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore + joinedload(EvaluationDB.variant_revision.of_type(AppVariantRevisionsDB)).load_only(AppVariantRevisionsDB.revision), # type: ignore + joinedload( + EvaluationDB.aggregated_results.of_type( + EvaluationAggregatedResultDB + ) + ).joinedload(EvaluationAggregatedResultDB.evaluator_config), + ) + ) + evaluation = result.unique().scalars().first() + return evaluation + + +async def list_human_evaluations(app_id: str, project_id: str): + """ + Fetches human evaluations belonging to an App. + + Args: + app_id (str): The application identifier + """ + + async with engine.core_session() as session: + base_query = ( + select(HumanEvaluationDB) + .filter_by(app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id)) + .filter(HumanEvaluationDB.testset_id.isnot(None)) + ) + query = base_query.options( + joinedload(HumanEvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore + ) + + result = await session.execute(query) + human_evaluations = result.scalars().all() + return human_evaluations + + +async def create_human_evaluation( + app: AppDB, + status: str, + evaluation_type: str, + testset_id: str, + variants_ids: List[str], +): + """ + Creates a human evaluation. + + Args: + app (AppDB: The app object + status (str): The status of the evaluation + evaluation_type (str): The evaluation type + testset_id (str): The ID of the evaluation testset + variants_ids (List[str]): The IDs of the variants for the evaluation + """ + + async with engine.core_session() as session: + human_evaluation = HumanEvaluationDB( + app_id=app.id, + project_id=app.project_id, + status=status, + evaluation_type=evaluation_type, + testset_id=testset_id, + ) + + session.add(human_evaluation) + await session.commit() + await session.refresh(human_evaluation, attribute_names=["testset"]) + + # create variants for human evaluation + await create_human_evaluation_variants( + human_evaluation_id=str(human_evaluation.id), + variants_ids=variants_ids, + ) + return human_evaluation + + +async def fetch_human_evaluation_variants(human_evaluation_id: str): + """ + Fetches human evaluation variants. + + Args: + human_evaluation_id (str): The human evaluation ID + + Returns: + The human evaluation variants. + """ + + async with engine.core_session() as session: + base_query = select(HumanEvaluationVariantDB).filter_by( + human_evaluation_id=uuid.UUID(human_evaluation_id) + ) + query = base_query.options( + joinedload(HumanEvaluationVariantDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore + joinedload(HumanEvaluationVariantDB.variant_revision.of_type(AppVariantRevisionsDB)).load_only(AppVariantRevisionsDB.id, AppVariantRevisionsDB.revision), # type: ignore + ) + + result = await session.execute(query) + evaluation_variants = result.scalars().all() + return evaluation_variants + + +async def create_human_evaluation_variants( + human_evaluation_id: str, variants_ids: List[str] +): + """ + Creates human evaluation variants. + + Args: + human_evaluation_id (str): The human evaluation identifier + variants_ids (List[str]): The variants identifiers + project_id (str): The project ID + """ + + variants_dict = {} + for variant_id in variants_ids: + variant = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) + if variant: + variants_dict[variant_id] = variant + + variants_revisions_dict = {} + for variant_id, variant in variants_dict.items(): + variant_revision = await db_manager.fetch_app_variant_revision_by_variant( + app_variant_id=str(variant.id), project_id=str(variant.project_id), revision=variant.revision # type: ignore + ) + if variant_revision: + variants_revisions_dict[variant_id] = variant_revision + + if set(variants_dict.keys()) != set(variants_revisions_dict.keys()): + raise ValueError("Mismatch between variants and their revisions") + + async with engine.core_session() as session: + for variant_id in variants_ids: + variant = variants_dict[variant_id] + variant_revision = variants_revisions_dict[variant_id] + human_evaluation_variant = HumanEvaluationVariantDB( + human_evaluation_id=uuid.UUID(human_evaluation_id), + variant_id=variant.id, # type: ignore + variant_revision_id=variant_revision.id, # type: ignore + ) + session.add(human_evaluation_variant) + + await session.commit() + + +async def fetch_human_evaluation_by_id( + evaluation_id: str, +) -> Optional[HumanEvaluationDB]: + """ + Fetches a evaluation by its ID. + + Args: + evaluation_id (str): The ID of the evaluation to fetch. + + Returns: + EvaluationDB: The fetched evaluation, or None if no evaluation was found. + """ + + assert evaluation_id is not None, "evaluation_id cannot be None" + async with engine.core_session() as session: + base_query = select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + query = base_query.options( + joinedload(HumanEvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore + ) + result = await session.execute(query) + evaluation = result.scalars().first() + return evaluation + + +async def update_human_evaluation(evaluation_id: str, values_to_update: dict): + """Updates human evaluation with the specified values. + + Args: + evaluation_id (str): The evaluation ID + values_to_update (dict): The values to update + + Exceptions: + NoResultFound: if human evaluation is not found + """ + + async with engine.core_session() as session: + result = await session.execute( + select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + ) + human_evaluation = result.scalars().first() + if not human_evaluation: + raise NoResultFound(f"Human evaluation with id {evaluation_id} not found") + + for key, value in values_to_update.items(): + if hasattr(human_evaluation, key): + setattr(human_evaluation, key, value) + + await session.commit() + await session.refresh(human_evaluation) + + +async def delete_human_evaluation(evaluation_id: str): + """Delete the evaluation by its ID. + + Args: + evaluation_id (str): The ID of the evaluation to delete. + """ + + assert evaluation_id is not None, "evaluation_id cannot be None" + async with engine.core_session() as session: + result = await session.execute( + select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + ) + evaluation = result.scalars().first() + if not evaluation: + raise NoResultFound(f"Human evaluation with id {evaluation_id} not found") + + await session.delete(evaluation) + await session.commit() + + +async def create_human_evaluation_scenario( + inputs: List[HumanEvaluationScenarioInput], + project_id: str, + evaluation_id: str, + evaluation_extend: Dict[str, Any], +): + """ + Creates a human evaluation scenario. + + Args: + inputs (List[HumanEvaluationScenarioInput]): The inputs. + evaluation_id (str): The evaluation identifier. + evaluation_extend (Dict[str, any]): An extended required payload for the evaluation scenario. Contains score, vote, and correct_answer. + """ + + async with engine.core_session() as session: + evaluation_scenario = HumanEvaluationScenarioDB( + **evaluation_extend, + project_id=uuid.UUID(project_id), + evaluation_id=uuid.UUID(evaluation_id), + inputs=[input.model_dump() for input in inputs], + outputs=[], + ) + + session.add(evaluation_scenario) + await session.commit() + + +async def update_human_evaluation_scenario( + evaluation_scenario_id: str, values_to_update: dict +): + """Updates human evaluation scenario with the specified values. + + Args: + evaluation_scenario_id (str): The evaluation scenario ID + values_to_update (dict): The values to update + + Exceptions: + NoResultFound: if human evaluation scenario is not found + """ + + async with engine.core_session() as session: + result = await session.execute( + select(HumanEvaluationScenarioDB).filter_by( + id=uuid.UUID(evaluation_scenario_id) + ) + ) + human_evaluation_scenario = result.scalars().first() + if not human_evaluation_scenario: + raise NoResultFound( + f"Human evaluation scenario with id {evaluation_scenario_id} not found" + ) + + for key, value in values_to_update.items(): + if hasattr(human_evaluation_scenario, key): + setattr(human_evaluation_scenario, key, value) + + await session.commit() + await session.refresh(human_evaluation_scenario) + + +async def fetch_human_evaluation_scenarios(evaluation_id: str): + """ + Fetches human evaluation scenarios. + + Args: + evaluation_id (str): The evaluation identifier + + Returns: + The evaluation scenarios. + """ + + async with engine.core_session() as session: + result = await session.execute( + select(HumanEvaluationScenarioDB) + .filter_by(evaluation_id=uuid.UUID(evaluation_id)) + .order_by(asc(HumanEvaluationScenarioDB.created_at)) + ) + evaluation_scenarios = result.scalars().all() + return evaluation_scenarios + + +async def fetch_evaluation_scenarios(evaluation_id: str, project_id: str): + """ + Fetches evaluation scenarios. + + Args: + evaluation_id (str): The evaluation identifier + project_id (str): The ID of the project + + Returns: + The evaluation scenarios. + """ + + async with engine.core_session() as session: + result = await session.execute( + select(EvaluationScenarioDB) + .filter_by( + evaluation_id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) + .options(joinedload(EvaluationScenarioDB.results)) + ) + evaluation_scenarios = result.unique().scalars().all() + return evaluation_scenarios + + +async def fetch_evaluation_scenario_by_id( + evaluation_scenario_id: str, +) -> Optional[EvaluationScenarioDB]: + """Fetches and evaluation scenario by its ID. + + Args: + evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. + + Returns: + EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. + """ + + assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" + async with engine.core_session() as session: + result = await session.execute( + select(EvaluationScenarioDB).filter_by(id=uuid.UUID(evaluation_scenario_id)) + ) + evaluation_scenario = result.scalars().first() + return evaluation_scenario + + +async def fetch_human_evaluation_scenario_by_id( + evaluation_scenario_id: str, +) -> Optional[HumanEvaluationScenarioDB]: + """Fetches and evaluation scenario by its ID. + + Args: + evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. + + Returns: + EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. + """ + + assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" + async with engine.core_session() as session: + result = await session.execute( + select(HumanEvaluationScenarioDB).filter_by( + id=uuid.UUID(evaluation_scenario_id) + ) + ) + evaluation_scenario = result.scalars().first() + return evaluation_scenario + + +async def fetch_human_evaluation_scenario_by_evaluation_id( + evaluation_id: str, +) -> Optional[HumanEvaluationScenarioDB]: + """Fetches and evaluation scenario by its ID. + Args: + evaluation_id (str): The ID of the evaluation object to use in fetching the human evaluation. + Returns: + EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. + """ + + evaluation = await fetch_human_evaluation_by_id(evaluation_id) + async with engine.core_session() as session: + result = await session.execute( + select(HumanEvaluationScenarioDB).filter_by( + evaluation_id=evaluation.id # type: ignore + ) + ) + human_eval_scenario = result.scalars().first() + return human_eval_scenario + + +async def create_new_evaluation( + app: AppDB, + project_id: str, + testset: TestsetDB, + status: Result, + variant: str, + variant_revision: str, +) -> EvaluationDB: + """Create a new evaluation scenario. + Returns: + EvaluationScenarioDB: The created evaluation scenario. + """ + + async with engine.core_session() as session: + evaluation = EvaluationDB( + app_id=app.id, + project_id=uuid.UUID(project_id), + testset_id=testset.id, + status=status.model_dump(), + variant_id=uuid.UUID(variant), + variant_revision_id=uuid.UUID(variant_revision), + ) + + session.add(evaluation) + await session.commit() + await session.refresh( + evaluation, + attribute_names=[ + "testset", + "variant", + "variant_revision", + "aggregated_results", + ], + ) + + return evaluation + + +async def list_evaluations(app_id: str, project_id: str): + """Retrieves evaluations of the specified app from the db. + + Args: + app_id (str): The ID of the app + project_id (str): The ID of the project + """ + + async with engine.core_session() as session: + base_query = select(EvaluationDB).filter_by( + app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) + ) + query = base_query.options( + joinedload(EvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore + ) + + result = await session.execute( + query.options( + joinedload(EvaluationDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore + joinedload(EvaluationDB.variant_revision.of_type(AppVariantRevisionsDB)).load_only(AppVariantRevisionsDB.revision), # type: ignore + joinedload( + EvaluationDB.aggregated_results.of_type( + EvaluationAggregatedResultDB + ) + ).joinedload(EvaluationAggregatedResultDB.evaluator_config), + ) + ) + evaluations = result.unique().scalars().all() + return evaluations + + +async def fetch_evaluations_by_resource( + resource_type: str, project_id: str, resource_ids: List[str] +): + """ + Fetches an evaluations by resource. + + Args: + resource_type (str): The resource type + project_id (str): The ID of the project + resource_ids (List[str]): The resource identifiers + + Returns: + The evaluations by resource. + + Raises: + HTTPException:400 resource_type {type} is not supported + """ + + ids = list(map(uuid.UUID, resource_ids)) + + async with engine.core_session() as session: + if resource_type == "variant": + result_evaluations = await session.execute( + select(EvaluationDB) + .filter( + EvaluationDB.variant_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) + .options(load_only(EvaluationDB.id)) # type: ignore + ) + result_human_evaluations = await session.execute( + select(HumanEvaluationDB) + .join(HumanEvaluationVariantDB) + .filter( + HumanEvaluationVariantDB.variant_id.in_(ids), + HumanEvaluationDB.project_id == uuid.UUID(project_id), + ) + .options(load_only(HumanEvaluationDB.id)) # type: ignore + ) + res_evaluations = result_evaluations.scalars().all() + res_human_evaluations = result_human_evaluations.scalars().all() + return res_evaluations + res_human_evaluations + + elif resource_type == "testset": + result_evaluations = await session.execute( + select(EvaluationDB) + .filter( + EvaluationDB.testset_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) + .options(load_only(EvaluationDB.id)) # type: ignore + ) + result_human_evaluations = await session.execute( + select(HumanEvaluationDB) + .filter( + HumanEvaluationDB.testset_id.in_(ids), + HumanEvaluationDB.project_id + == uuid.UUID(project_id), # Fixed to match HumanEvaluationDB + ) + .options(load_only(HumanEvaluationDB.id)) # type: ignore + ) + res_evaluations = result_evaluations.scalars().all() + res_human_evaluations = result_human_evaluations.scalars().all() + return res_evaluations + res_human_evaluations + + elif resource_type == "evaluator_config": + query = ( + select(EvaluationDB) + .join(EvaluationDB.evaluator_configs) + .filter( + EvaluationEvaluatorConfigDB.evaluator_config_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) + ) + result = await session.execute(query) + res = result.scalars().all() + return res + + raise HTTPException( + status_code=400, + detail=f"resource_type {resource_type} is not supported", + ) + + +async def delete_evaluations(evaluation_ids: List[str]) -> None: + """Delete evaluations based on the ids provided from the db. + + Args: + evaluations_ids (list[str]): The IDs of the evaluation + """ + + async with engine.core_session() as session: + query = select(EvaluationDB).where(EvaluationDB.id.in_(evaluation_ids)) + result = await session.execute(query) + evaluations = result.scalars().all() + for evaluation in evaluations: + await session.delete(evaluation) + await session.commit() + + +async def create_new_evaluation_scenario( + project_id: str, + evaluation_id: str, + variant_id: str, + inputs: List[EvaluationScenarioInput], + outputs: List[EvaluationScenarioOutput], + correct_answers: Optional[List[CorrectAnswer]], + is_pinned: Optional[bool], + note: Optional[str], + results: List[EvaluationScenarioResult], +) -> EvaluationScenarioDB: + """Create a new evaluation scenario. + + Returns: + EvaluationScenarioDB: The created evaluation scenario. + """ + + async with engine.core_session() as session: + evaluation_scenario = EvaluationScenarioDB( + project_id=uuid.UUID(project_id), + evaluation_id=uuid.UUID(evaluation_id), + variant_id=uuid.UUID(variant_id), + inputs=[input.model_dump() for input in inputs], + outputs=[output.model_dump() for output in outputs], + correct_answers=( + [correct_answer.model_dump() for correct_answer in correct_answers] + if correct_answers is not None + else [] + ), + is_pinned=is_pinned, + note=note, + ) + + session.add(evaluation_scenario) + await session.commit() + await session.refresh(evaluation_scenario) + + # create evaluation scenario result + for result in results: + evaluation_scenario_result = EvaluationScenarioResultDB( + evaluation_scenario_id=evaluation_scenario.id, + evaluator_config_id=uuid.UUID(result.evaluator_config), + result=result.result.model_dump(), + ) + + session.add(evaluation_scenario_result) + + await session.commit() # ensures that scenario results insertion is committed + await session.refresh(evaluation_scenario) + + return evaluation_scenario + + +async def update_evaluation_with_aggregated_results( + evaluation_id: str, aggregated_results: List[AggregatedResult] +): + async with engine.core_session() as session: + for result in aggregated_results: + aggregated_result = EvaluationAggregatedResultDB( + evaluation_id=uuid.UUID(evaluation_id), + evaluator_config_id=uuid.UUID(result.evaluator_config), + result=result.result.model_dump(), + ) + session.add(aggregated_result) + + await session.commit() + + +async def fetch_eval_aggregated_results(evaluation_id: str): + """ + Fetches an evaluation aggregated results by evaluation identifier. + + Args: + evaluation_id (str): The evaluation identifier + + Returns: + The evaluation aggregated results by evaluation identifier. + """ + + async with engine.core_session() as session: + base_query = select(EvaluationAggregatedResultDB).filter_by( + evaluation_id=uuid.UUID(evaluation_id) + ) + query = base_query.options( + joinedload( + EvaluationAggregatedResultDB.evaluator_config.of_type(EvaluatorConfigDB) + ).load_only( + EvaluatorConfigDB.id, # type: ignore + EvaluatorConfigDB.name, # type: ignore + EvaluatorConfigDB.evaluator_key, # type: ignore + EvaluatorConfigDB.settings_values, # type: ignore + EvaluatorConfigDB.created_at, # type: ignore + EvaluatorConfigDB.updated_at, # type: ignore + ) + ) + + result = await session.execute(query) + aggregated_results = result.scalars().all() + return aggregated_results + + +async def update_evaluation( + evaluation_id: str, project_id: str, updates: Dict[str, Any] +) -> EvaluationDB: + """ + Update an evaluator configuration in the database with the provided id. + + Arguments: + evaluation_id (str): The ID of the evaluator configuration to be updated. + project_id (str): The ID of the project. + updates (Dict[str, Any]): The updates to apply to the evaluator configuration. + + Returns: + EvaluatorConfigDB: The updated evaluator configuration object. + """ + + async with engine.core_session() as session: + result = await session.execute( + select(EvaluationDB).filter_by( + id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) + ) + evaluation = result.scalars().first() + for key, value in updates.items(): + if hasattr(evaluation, key): + setattr(evaluation, key, value) + + await session.commit() + await session.refresh(evaluation) + + return evaluation + + +async def check_if_evaluation_contains_failed_evaluation_scenarios( + evaluation_id: str, +) -> bool: + async with engine.core_session() as session: + EvaluationResultAlias = aliased(EvaluationScenarioResultDB) + query = ( + select(func.count(EvaluationScenarioDB.id)) + .join(EvaluationResultAlias, EvaluationScenarioDB.results) + .where( + EvaluationScenarioDB.evaluation_id == uuid.UUID(evaluation_id), + EvaluationResultAlias.result["type"].astext == "error", + ) + ) + + result = await session.execute(query) + count = result.scalar() + if not count: + return False + return count > 0 diff --git a/api/oss/src/services/evaluation_service.py b/api/ee/src/services/evaluation_service.py similarity index 93% rename from api/oss/src/services/evaluation_service.py rename to api/ee/src/services/evaluation_service.py index ca40a70cef..9d7b61cb3d 100644 --- a/api/oss/src/services/evaluation_service.py +++ b/api/ee/src/services/evaluation_service.py @@ -3,8 +3,9 @@ from fastapi import HTTPException from oss.src.utils.logging import get_module_logger -from oss.src.services import converters +from ee.src.services import converters from oss.src.services import db_manager +from ee.src.services import db_manager_ee from oss.src.models.api.evaluation_model import ( Evaluation, @@ -17,7 +18,7 @@ NewHumanEvaluation, ) from oss.src.models.db_models import AppDB -from oss.src.models.db_models import ( +from ee.src.models.db_models import ( EvaluationDB, HumanEvaluationDB, HumanEvaluationScenarioDB, @@ -65,7 +66,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( for name in payload_inputs ] except KeyError: - await db_manager.delete_human_evaluation( + await db_manager_ee.delete_human_evaluation( evaluation_id=str(new_evaluation.id) ) msg = f""" @@ -91,7 +92,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( **_extend_with_evaluation(evaluation_type), **_extend_with_correct_answer(evaluation_type, datum), } - await db_manager.create_human_evaluation_scenario( + await db_manager_ee.create_human_evaluation_scenario( inputs=list_of_scenario_input, project_id=project_id, evaluation_id=str(new_evaluation.id), @@ -111,7 +112,7 @@ async def update_human_evaluation_service( """ # Update the evaluation - await db_manager.update_human_evaluation( + await db_manager_ee.update_human_evaluation( evaluation_id=str(evaluation.id), values_to_update=update_payload.model_dump() ) @@ -130,7 +131,7 @@ async def fetch_evaluation_scenarios_for_evaluation( List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluation_scenarios = await db_manager.fetch_evaluation_scenarios( + evaluation_scenarios = await db_manager_ee.fetch_evaluation_scenarios( evaluation_id=evaluation_id, project_id=project_id ) return [ @@ -156,7 +157,7 @@ async def fetch_human_evaluation_scenarios_for_evaluation( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - human_evaluation_scenarios = await db_manager.fetch_human_evaluation_scenarios( + human_evaluation_scenarios = await db_manager_ee.fetch_human_evaluation_scenarios( evaluation_id=str(human_evaluation.id) ) eval_scenarios = [ @@ -224,7 +225,7 @@ async def update_human_evaluation_scenario( if "correct_answer" in payload: values_to_update["correct_answer"] = payload["correct_answer"] - await db_manager.update_human_evaluation_scenario( + await db_manager_ee.update_human_evaluation_scenario( evaluation_scenario_id=str(evaluation_scenario_db.id), values_to_update=values_to_update, ) @@ -259,7 +260,7 @@ async def fetch_list_evaluations(app: AppDB, project_id: str) -> List[Evaluation List[Evaluation]: A list of evaluations. """ - evaluations_db = await db_manager.list_evaluations( + evaluations_db = await db_manager_ee.list_evaluations( app_id=str(app.id), project_id=project_id ) return [ @@ -282,7 +283,7 @@ async def fetch_list_human_evaluations( List[Evaluation]: A list of evaluations. """ - evaluations_db = await db_manager.list_human_evaluations( + evaluations_db = await db_manager_ee.list_human_evaluations( app_id=app_id, project_id=project_id ) return [ @@ -318,7 +319,7 @@ async def delete_human_evaluations(evaluation_ids: List[str]) -> None: """ for evaluation_id in evaluation_ids: - await db_manager.delete_human_evaluation(evaluation_id=evaluation_id) + await db_manager_ee.delete_human_evaluation(evaluation_id=evaluation_id) async def delete_evaluations(evaluation_ids: List[str]) -> None: @@ -332,7 +333,7 @@ async def delete_evaluations(evaluation_ids: List[str]) -> None: HTTPException: If evaluation not found or access denied. """ - await db_manager.delete_evaluations(evaluation_ids=evaluation_ids) + await db_manager_ee.delete_evaluations(evaluation_ids=evaluation_ids) async def create_new_human_evaluation(payload: NewHumanEvaluation) -> HumanEvaluationDB: @@ -353,7 +354,7 @@ async def create_new_human_evaluation(payload: NewHumanEvaluation) -> HumanEvalu detail=f"App with id {payload.app_id} does not exist", ) - human_evaluation = await db_manager.create_human_evaluation( + human_evaluation = await db_manager_ee.create_human_evaluation( app=app, status=payload.status, evaluation_type=payload.evaluation_type, @@ -410,7 +411,7 @@ async def create_new_evaluation( assert testset is not None, f"Testset with id {testset_id} does not exist" - evaluation_db = await db_manager.create_new_evaluation( + evaluation_db = await db_manager_ee.create_new_evaluation( app=app, project_id=project_id, testset=testset, @@ -424,7 +425,7 @@ async def create_new_evaluation( async def compare_evaluations_scenarios(evaluations_ids: List[str], project_id: str): - evaluation = await db_manager.fetch_evaluation_by_id( + evaluation = await db_manager_ee.fetch_evaluation_by_id( project_id=project_id, evaluation_id=evaluations_ids[0], ) diff --git a/api/oss/src/services/llm_apps_service.py b/api/ee/src/services/llm_apps_service.py similarity index 100% rename from api/oss/src/services/llm_apps_service.py rename to api/ee/src/services/llm_apps_service.py diff --git a/api/oss/src/services/results_service.py b/api/ee/src/services/results_service.py similarity index 91% rename from api/oss/src/services/results_service.py rename to api/ee/src/services/results_service.py index cccb32164d..ca52151315 100644 --- a/api/oss/src/services/results_service.py +++ b/api/ee/src/services/results_service.py @@ -1,16 +1,16 @@ import uuid from typing import Sequence, Dict, Any -from oss.src.services import db_manager +from ee.src.services import db_manager_ee from oss.src.models.api.evaluation_model import EvaluationType -from oss.src.models.db_models import ( +from ee.src.models.db_models import ( HumanEvaluationDB, EvaluationScenarioDB, ) async def fetch_results_for_evaluation(evaluation: HumanEvaluationDB): - evaluation_scenarios = await db_manager.fetch_human_evaluation_scenarios( + evaluation_scenarios = await db_manager_ee.fetch_human_evaluation_scenarios( evaluation_id=str(evaluation.id) ) @@ -18,7 +18,7 @@ async def fetch_results_for_evaluation(evaluation: HumanEvaluationDB): if len(evaluation_scenarios) == 0: return results - evaluation_variants = await db_manager.fetch_human_evaluation_variants( + evaluation_variants = await db_manager_ee.fetch_human_evaluation_variants( human_evaluation_id=str(evaluation.id) ) results["variants"] = [ @@ -99,7 +99,7 @@ async def _compute_stats_for_human_a_b_testing_evaluation( async def fetch_results_for_single_model_test(evaluation_id: str): - evaluation_scenarios = await db_manager.fetch_human_evaluation_scenarios( + evaluation_scenarios = await db_manager_ee.fetch_human_evaluation_scenarios( evaluation_id=str(evaluation_id) ) scores_and_counts: Dict[str, Any] = {} diff --git a/api/ee/src/services/utils.py b/api/ee/src/services/utils.py new file mode 100644 index 0000000000..0eaedde4ff --- /dev/null +++ b/api/ee/src/services/utils.py @@ -0,0 +1,21 @@ +# Stdlib Imports +import asyncio +from functools import partial +from typing import Callable, Coroutine + + +async def run_in_separate_thread(func: Callable, *args, **kwargs) -> Coroutine: + """ + Run a synchronous function in a separate thread. + + Args: + func (callable): The synchronous function to be executed. + args (tuple): Positional arguments to be passed to `func`. + kwargs (dict): Keyword arguments to be passed to `func`. + + Returns: + The result of the synchronous function. + """ + + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, partial(func, *args, **kwargs)) diff --git a/api/oss/src/tasks/evaluations/__init__.py b/api/ee/src/tasks/__init__.py similarity index 100% rename from api/oss/src/tasks/evaluations/__init__.py rename to api/ee/src/tasks/__init__.py diff --git a/web/oss/src/components/Evaluations/HumanEvaluationResult.tsx b/api/ee/src/tasks/evaluations/__init__.py similarity index 100% rename from web/oss/src/components/Evaluations/HumanEvaluationResult.tsx rename to api/ee/src/tasks/evaluations/__init__.py diff --git a/api/oss/src/tasks/evaluations/batch.py b/api/ee/src/tasks/evaluations/batch.py similarity index 97% rename from api/oss/src/tasks/evaluations/batch.py rename to api/ee/src/tasks/evaluations/batch.py index 324ed74e49..5fdef15b3c 100644 --- a/api/oss/src/tasks/evaluations/batch.py +++ b/api/ee/src/tasks/evaluations/batch.py @@ -10,9 +10,8 @@ from oss.src.utils.helpers import parse_url, get_slug_from_name_and_id from oss.src.utils.logging import get_module_logger -from oss.src.utils.common import is_ee from oss.src.services.auth_helper import sign_secret_token -from oss.src.services import llm_apps_service +from ee.src.services import llm_apps_service from oss.src.models.shared_models import InvokationResult from oss.src.services.db_manager import ( fetch_app_by_id, @@ -22,9 +21,7 @@ get_project_by_id, ) from oss.src.core.secrets.utils import get_llm_providers_secrets - -if is_ee(): - from ee.src.utils.entitlements import check_entitlements, Counter +from ee.src.utils.entitlements import check_entitlements, Counter from oss.src.dbs.postgres.queries.dbes import ( QueryArtifactDBE, diff --git a/api/oss/src/tasks/evaluations/legacy.py b/api/ee/src/tasks/evaluations/legacy.py similarity index 99% rename from api/oss/src/tasks/evaluations/legacy.py rename to api/ee/src/tasks/evaluations/legacy.py index d3bc69f9cc..579c6853b9 100644 --- a/api/oss/src/tasks/evaluations/legacy.py +++ b/api/ee/src/tasks/evaluations/legacy.py @@ -9,9 +9,8 @@ from oss.src.utils.helpers import parse_url, get_slug_from_name_and_id from oss.src.utils.logging import get_module_logger -from oss.src.utils.common import is_ee from oss.src.services.auth_helper import sign_secret_token -from oss.src.services import llm_apps_service +from ee.src.services import llm_apps_service from oss.src.models.shared_models import InvokationResult from oss.src.services.db_manager import ( fetch_app_by_id, @@ -22,9 +21,7 @@ get_project_by_id, ) from oss.src.core.secrets.utils import get_llm_providers_secrets - -if is_ee(): - from ee.src.utils.entitlements import check_entitlements, Counter +from ee.src.utils.entitlements import check_entitlements, Counter from oss.src.dbs.postgres.queries.dbes import ( QueryArtifactDBE, @@ -1544,14 +1541,13 @@ def annotate( # edit meters to avoid conting failed evaluations -------------------------- if run_status == EvaluationStatus.FAILURE: - if is_ee(): - loop.run_until_complete( - check_entitlements( - organization_id=project.organization_id, - key=Counter.EVALUATIONS, - delta=-1, - ) + loop.run_until_complete( + check_entitlements( + organization_id=project.organization_id, + key=Counter.EVALUATIONS, + delta=-1, ) + ) log.info("[DONE] ", run_id=run_id, project_id=project_id, user_id=user_id) diff --git a/api/oss/src/tasks/evaluations/live.py b/api/ee/src/tasks/evaluations/live.py similarity index 100% rename from api/oss/src/tasks/evaluations/live.py rename to api/ee/src/tasks/evaluations/live.py diff --git a/api/oss/tests/manual/evaluations/live.http b/api/ee/tests/manual/evaluations/live.http similarity index 100% rename from api/oss/tests/manual/evaluations/live.http rename to api/ee/tests/manual/evaluations/live.http diff --git a/api/oss/tests/manual/evaluators/human-evaluator.http b/api/ee/tests/manual/evaluators/human-evaluator.http similarity index 100% rename from api/oss/tests/manual/evaluators/human-evaluator.http rename to api/ee/tests/manual/evaluators/human-evaluator.http diff --git a/api/entrypoint.py b/api/entrypoint.py index 36b37b699c..aa50cf2de1 100644 --- a/api/entrypoint.py +++ b/api/entrypoint.py @@ -24,8 +24,6 @@ from oss.src.services.auth_helper import authentication_middleware from oss.src.services.analytics_service import analytics_middleware -from oss.src.routers import evaluation_router, human_evaluation_router - # DBEs from oss.src.dbs.postgres.queries.dbes import ( QueryArtifactDBE, @@ -474,18 +472,6 @@ async def lifespan(*args, **kwargs): tags=["Evaluations"], ) -app.include_router( - evaluation_router.router, - prefix="/evaluations", - tags=["Evaluations"], -) - -app.include_router( - human_evaluation_router.router, - prefix="/human-evaluations", - tags=["Human-Evaluations"], -) - app.include_router( admin_router.router, prefix="/admin", @@ -582,11 +568,7 @@ async def lifespan(*args, **kwargs): # ------------------------------------------------------------------------------ - -import oss.src.tasks.evaluations.live -import oss.src.tasks.evaluations.legacy -import oss.src.tasks.evaluations.batch - - if ee and is_ee(): app = ee.extend_app_schema(app) + + ee.load_tasks() diff --git a/api/oss/docker/Dockerfile.dev b/api/oss/docker/Dockerfile.dev index 8b500fd96e..647b46c960 100644 --- a/api/oss/docker/Dockerfile.dev +++ b/api/oss/docker/Dockerfile.dev @@ -34,12 +34,12 @@ ENV PYTHONPATH=/sdk:$PYTHONPATH # # -COPY ./oss/src/crons/queries.sh /queries.sh -COPY ./oss/src/crons/queries.txt /etc/cron.d/queries-cron -RUN sed -i -e '$a\' /etc/cron.d/queries-cron -RUN cat -A /etc/cron.d/queries-cron +# +# +# +# -RUN chmod +x /queries.sh \ - && chmod 0644 /etc/cron.d/queries-cron +# +# EXPOSE 8000 diff --git a/api/oss/docker/Dockerfile.gh b/api/oss/docker/Dockerfile.gh index a9bd7c8365..cf9817f0b0 100644 --- a/api/oss/docker/Dockerfile.gh +++ b/api/oss/docker/Dockerfile.gh @@ -18,11 +18,11 @@ RUN pip install --upgrade pip \ # COPY ./oss /app/oss/ COPY ./entrypoint.py ./pyproject.toml /app/ -COPY ./sdk /sdk/ +# RUN poetry config virtualenvs.create false \ - && poetry install --no-interaction --no-ansi \ - && pip install --force-reinstall --upgrade /sdk/ + && poetry install --no-interaction --no-ansi +# # @@ -34,12 +34,12 @@ RUN poetry config virtualenvs.create false \ # # -COPY ./oss/src/crons/queries.sh /queries.sh -COPY ./oss/src/crons/queries.txt /etc/cron.d/queries-cron -RUN sed -i -e '$a\' /etc/cron.d/queries-cron -RUN cat -A /etc/cron.d/queries-cron +# +# +# +# -RUN chmod +x /queries.sh \ - && chmod 0644 /etc/cron.d/queries-cron +# +# EXPOSE 8000 diff --git a/api/oss/src/apis/fastapi/applications/router.py b/api/oss/src/apis/fastapi/applications/router.py index e179e0f04c..2f03ea8c8b 100644 --- a/api/oss/src/apis/fastapi/applications/router.py +++ b/api/oss/src/apis/fastapi/applications/router.py @@ -104,14 +104,13 @@ async def retrieve_application_revision( *, application_revision_retrieve_request: ApplicationRevisionRetrieveRequest, ): - if is_ee(): - if not await check_action_access( # type: ignore - project_id=request.state.project_id, - user_uid=request.state.user_id, - # - permission=Permission.VIEW_APPLICATIONS, # type: ignore - ): - raise FORBIDDEN_EXCEPTION # type: ignore + if not await check_action_access( # type: ignore + project_id=request.state.project_id, + user_uid=request.state.user_id, + # + permission=Permission.VIEW_APPLICATIONS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore cache_key = { "artifact_ref": application_revision_retrieve_request.application_ref, # type: ignore diff --git a/api/oss/src/apis/fastapi/evaluators/router.py b/api/oss/src/apis/fastapi/evaluators/router.py index 4ee7f5cbd8..4461df9072 100644 --- a/api/oss/src/apis/fastapi/evaluators/router.py +++ b/api/oss/src/apis/fastapi/evaluators/router.py @@ -748,14 +748,13 @@ async def retrieve_evaluator_revision( *, evaluator_revision_retrieve_request: EvaluatorRevisionRetrieveRequest, ) -> EvaluatorRevisionResponse: - if is_ee(): - if not await check_action_access( # type: ignore - project_id=request.state.project_id, - user_uid=request.state.user_id, - # - permission=Permission.VIEW_EVALUATORS, # type: ignore - ): - raise FORBIDDEN_EXCEPTION # type: ignore + if not await check_action_access( # type: ignore + project_id=request.state.project_id, + user_uid=request.state.user_id, + # + permission=Permission.VIEW_EVALUATORS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore cache_key = { "artifact_ref": evaluator_revision_retrieve_request.evaluator_ref, # type: ignore diff --git a/api/oss/src/core/evaluations/service.py b/api/oss/src/core/evaluations/service.py index b36a9d47ef..a6fff35361 100644 --- a/api/oss/src/core/evaluations/service.py +++ b/api/oss/src/core/evaluations/service.py @@ -6,6 +6,7 @@ from celery import current_app as celery_dispatch +from oss.src.utils.common import is_ee from oss.src.utils.logging import get_module_logger from oss.src.core.shared.dtos import Reference, Windowing, Tags, Meta, Data @@ -178,7 +179,7 @@ async def refresh_runs( try: log.info( - "[LIVE] Dispatching...", + "[LIVE]", project_id=project_id, run_id=run.id, # @@ -186,24 +187,19 @@ async def refresh_runs( oldest=oldest, ) - celery_dispatch.send_task( # type: ignore - "src.tasks.evaluations.live.evaluate", - kwargs=dict( - project_id=project_id, - user_id=user_id, - # - run_id=run.id, - # - newest=newest, - oldest=oldest, - ), - ) - - log.info( - "[LIVE] Dispatched. ", - project_id=project_id, - run_id=run.id, - ) + if is_ee(): + celery_dispatch.send_task( # type: ignore + "src.tasks.evaluations.live.evaluate", + kwargs=dict( + project_id=project_id, + user_id=user_id, + # + run_id=run.id, + # + newest=newest, + oldest=oldest, + ), + ) except Exception as e: # pylint: disable=broad-exception-caught log.error(f"[LIVE] Error refreshing run {run.id}: {e}", exc_info=True) @@ -1561,26 +1557,29 @@ async def start( return None if _evaluation.data.query_steps: - celery_dispatch.send_task( # type: ignore - "src.tasks.evaluations.batch.evaluate_queries", - kwargs=dict( - project_id=project_id, - user_id=user_id, - # - run_id=run.id, - ), - ) + if is_ee(): + celery_dispatch.send_task( # type: ignore + "src.tasks.evaluations.batch.evaluate_queries", + kwargs=dict( + project_id=project_id, + user_id=user_id, + # + run_id=run.id, + ), + ) elif _evaluation.data.testset_steps: - celery_dispatch.send_task( # type: ignore - "src.tasks.evaluations.batch.evaluate_testsets", - kwargs=dict( - project_id=project_id, - user_id=user_id, - # - run_id=run.id, - ), - ) + if is_ee(): + # TODO: Fix typing ? + celery_dispatch.send_task( # type: ignore + "src.tasks.evaluations.batch.evaluate_testsets", + kwargs=dict( + project_id=project_id, + user_id=user_id, + # + run_id=run.id, + ), + ) return _evaluation diff --git a/api/oss/src/models/db_models.py b/api/oss/src/models/db_models.py index 867cb30156..3afa51acdb 100644 --- a/api/oss/src/models/db_models.py +++ b/api/oss/src/models/db_models.py @@ -89,6 +89,7 @@ class WorkspaceDB(Base): ) +# KEEP in oss/ class UserDB(Base): __tablename__ = "users" @@ -110,6 +111,7 @@ class UserDB(Base): ) +# KEEP in oss/ class ProjectDB(Base): __tablename__ = "projects" @@ -153,6 +155,7 @@ class ProjectDB(Base): testset = relationship("TestsetDB", cascade=CASCADE_ALL_DELETE, backref="project") +# KEEP in oss/ class AppDB(Base): __tablename__ = "app_db" @@ -187,6 +190,7 @@ class AppDB(Base): ) +# KEEP in oss/ class DeploymentDB(Base): __tablename__ = "deployments" @@ -213,6 +217,7 @@ class DeploymentDB(Base): app = relationship("AppDB", back_populates="deployment") +# KEEP in oss/ class VariantBaseDB(Base): __tablename__ = "bases" @@ -243,6 +248,7 @@ class VariantBaseDB(Base): project = relationship("oss.src.models.db_models.ProjectDB") +# KEEP in oss/ class AppVariantDB(Base): __tablename__ = "app_variants" @@ -287,6 +293,7 @@ class AppVariantDB(Base): ) +# KEEP in oss/ class AppVariantRevisionsDB(Base): __tablename__ = "app_variant_revisions" @@ -329,6 +336,7 @@ def get_config(self) -> dict: return {"config_name": self.config_name, "parameters": self.config_parameters} +# KEEP in oss/ class AppEnvironmentDB(Base): __tablename__ = "environments" @@ -366,6 +374,7 @@ class AppEnvironmentDB(Base): deployed_app_variant_revision = relationship("AppVariantRevisionsDB") +# KEEP in oss/ class AppEnvironmentRevisionDB(Base): __tablename__ = "environments_revisions" @@ -399,6 +408,7 @@ class AppEnvironmentRevisionDB(Base): modified_by = relationship("UserDB") +# KEEP in oss/ class TestsetDB(Base): __tablename__ = "testsets" @@ -422,6 +432,7 @@ class TestsetDB(Base): ) +# KEEP in oss/ class EvaluatorConfigDB(Base): __tablename__ = "auto_evaluator_configs" @@ -447,6 +458,7 @@ class EvaluatorConfigDB(Base): ) +# KEEP in oss/ or KILL class IDsMappingDB(Base): __tablename__ = "ids_mapping" @@ -519,267 +531,3 @@ class APIKeyDB(Base): project = relationship( "oss.src.models.db_models.ProjectDB", backref="api_key_project" ) - - -class HumanEvaluationVariantDB(Base): - __tablename__ = "human_evaluation_variants" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - human_evaluation_id = Column( - UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") - ) - variant_id = Column( - UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") - ) - variant_revision_id = Column( - UUID(as_uuid=True), ForeignKey("app_variant_revisions.id", ondelete="SET NULL") - ) - - variant = relationship("AppVariantDB", backref="evaluation_variant") - variant_revision = relationship( - "AppVariantRevisionsDB", backref="evaluation_variant_revision" - ) - - -class HumanEvaluationDB(Base): - __tablename__ = "human_evaluations" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) - status = Column(String) - evaluation_type = Column(String) - testset_id = Column(UUID(as_uuid=True), ForeignKey("testsets.id")) - created_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - updated_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - - testset = relationship("TestsetDB") - evaluation_variant = relationship( - "HumanEvaluationVariantDB", - cascade=CASCADE_ALL_DELETE, - backref="human_evaluation", - ) - evaluation_scenario = relationship( - "HumanEvaluationScenarioDB", - cascade=CASCADE_ALL_DELETE, - backref="evaluation_scenario", - ) - - -class HumanEvaluationScenarioDB(Base): - __tablename__ = "human_evaluations_scenarios" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) - evaluation_id = Column( - UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") - ) - inputs = Column( - mutable_json_type(dbtype=JSONB, nested=True) - ) # List of HumanEvaluationScenarioInput - outputs = Column( - mutable_json_type(dbtype=JSONB, nested=True) - ) # List of HumanEvaluationScenarioOutput - vote = Column(String) - score = Column(String) - correct_answer = Column(String) - created_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - updated_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - is_pinned = Column(Boolean) - note = Column(String) - - -class EvaluationAggregatedResultDB(Base): - __tablename__ = "auto_evaluation_aggregated_results" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - evaluation_id = Column( - UUID(as_uuid=True), ForeignKey("auto_evaluations.id", ondelete="CASCADE") - ) - evaluator_config_id = Column( - UUID(as_uuid=True), - ForeignKey("auto_evaluator_configs.id", ondelete="SET NULL"), - ) - result = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result - - evaluator_config = relationship("EvaluatorConfigDB", backref="evaluator_config") - - -class EvaluationScenarioResultDB(Base): - __tablename__ = "auto_evaluation_scenario_results" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - evaluation_scenario_id = Column( - UUID(as_uuid=True), - ForeignKey("auto_evaluation_scenarios.id", ondelete="CASCADE"), - ) - evaluator_config_id = Column( - UUID(as_uuid=True), - ForeignKey("auto_evaluator_configs.id", ondelete="SET NULL"), - ) - result = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result - - -class EvaluationDB(Base): - __tablename__ = "auto_evaluations" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) - status = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result - testset_id = Column( - UUID(as_uuid=True), ForeignKey("testsets.id", ondelete="SET NULL") - ) - variant_id = Column( - UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") - ) - variant_revision_id = Column( - UUID(as_uuid=True), ForeignKey("app_variant_revisions.id", ondelete="SET NULL") - ) - average_cost = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result - total_cost = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result - average_latency = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result - created_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - updated_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - - project = relationship("oss.src.models.db_models.ProjectDB") - testset = relationship("TestsetDB") - variant = relationship("AppVariantDB") - variant_revision = relationship("AppVariantRevisionsDB") - aggregated_results = relationship( - "EvaluationAggregatedResultDB", - cascade=CASCADE_ALL_DELETE, - backref="evaluation", - ) - evaluation_scenarios = relationship( - "EvaluationScenarioDB", cascade=CASCADE_ALL_DELETE, backref="evaluation" - ) - evaluator_configs = relationship( - "EvaluationEvaluatorConfigDB", - cascade=CASCADE_ALL_DELETE, - backref="evaluation", - ) - - -class EvaluationEvaluatorConfigDB(Base): - __tablename__ = "auto_evaluation_evaluator_configs" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("auto_evaluations.id", ondelete="CASCADE"), - primary_key=True, - ) - evaluator_config_id = Column( - UUID(as_uuid=True), - ForeignKey("auto_evaluator_configs.id", ondelete="SET NULL"), - primary_key=True, - ) - - -class EvaluationScenarioDB(Base): - __tablename__ = "auto_evaluation_scenarios" - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid7, - unique=True, - nullable=False, - ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) - evaluation_id = Column( - UUID(as_uuid=True), ForeignKey("auto_evaluations.id", ondelete="CASCADE") - ) - variant_id = Column( - UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") - ) - inputs = Column( - mutable_json_type(dbtype=JSONB, nested=True) - ) # List of EvaluationScenarioInput - outputs = Column( - mutable_json_type(dbtype=JSONB, nested=True) - ) # List of EvaluationScenarioOutput - correct_answers = Column( - mutable_json_type(dbtype=JSONB, nested=True) - ) # List of CorrectAnswer - is_pinned = Column(Boolean) - note = Column(String) - latency = Column(Integer) - cost = Column(Integer) - created_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - updated_at = Column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - - project = relationship("oss.src.models.db_models.ProjectDB") - variant = relationship("AppVariantDB") - results = relationship( - "EvaluationScenarioResultDB", - cascade=CASCADE_ALL_DELETE, - backref="evaluation_scenario", - ) diff --git a/api/oss/src/resources/evaluators/evaluators.py b/api/oss/src/resources/evaluators/evaluators.py index cbca48d4fc..53a2d48542 100644 --- a/api/oss/src/resources/evaluators/evaluators.py +++ b/api/oss/src/resources/evaluators/evaluators.py @@ -229,12 +229,12 @@ "description": "Extract information from the user's response.", "type": "object", "properties": { - "score": { + "correctness": { "type": "boolean", "description": "The grade results", } }, - "required": ["score"], + "required": ["correctness"], "strict": True, }, }, @@ -264,12 +264,12 @@ "description": "Extract information from the user's response.", "type": "object", "properties": { - "score": { + "correctness": { "type": "boolean", "description": "The hallucination detection result", } }, - "required": ["score"], + "required": ["correctness"], "strict": True, }, }, @@ -339,12 +339,12 @@ "description": "Extract information from the user's response.", "type": "object", "properties": { - "score": { + "correctness": { "type": "boolean", "description": "The grade results", } }, - "required": ["score"], + "required": ["correctness"], "strict": True, }, }, diff --git a/api/oss/src/services/app_manager.py b/api/oss/src/services/app_manager.py index f644399ebd..7e11084485 100644 --- a/api/oss/src/services/app_manager.py +++ b/api/oss/src/services/app_manager.py @@ -42,7 +42,7 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: raise db_manager.NoResultFound(f"Variant with id {object_id} not found") return str(app_variant_db.app_id) elif object_type == "evaluation": - evaluation_db = await db_manager.fetch_evaluation_by_id( + evaluation_db = await db_manager_ee.fetch_evaluation_by_id( project_id=project_id, evaluation_id=object_id, ) diff --git a/api/oss/src/services/converters.py b/api/oss/src/services/converters.py deleted file mode 100644 index ad9cb64169..0000000000 --- a/api/oss/src/services/converters.py +++ /dev/null @@ -1,191 +0,0 @@ -import uuid -from typing import List, Dict, Any -from datetime import datetime, timezone - -from oss.src.services import db_manager -from oss.src.models.api.evaluation_model import ( - CorrectAnswer, - Evaluation, - HumanEvaluation, - EvaluationScenario, - SimpleEvaluationOutput, - EvaluationScenarioInput, - HumanEvaluationScenario, - EvaluationScenarioOutput, -) -from oss.src.models.db_models import ( - EvaluationDB, - HumanEvaluationDB, - EvaluationScenarioDB, - HumanEvaluationScenarioDB, -) - - -async def human_evaluation_db_to_simple_evaluation_output( - human_evaluation_db: HumanEvaluationDB, -) -> SimpleEvaluationOutput: - evaluation_variants = await db_manager.fetch_human_evaluation_variants( - human_evaluation_id=str(human_evaluation_db.id) - ) - return SimpleEvaluationOutput( - id=str(human_evaluation_db.id), - app_id=str(human_evaluation_db.app_id), - project_id=str(human_evaluation_db.project_id), - status=human_evaluation_db.status, # type: ignore - evaluation_type=human_evaluation_db.evaluation_type, # type: ignore - variant_ids=[ - str(evaluation_variant.variant_id) - for evaluation_variant in evaluation_variants - ], - ) - - -async def evaluation_db_to_pydantic( - evaluation_db: EvaluationDB, -) -> Evaluation: - variant_name = ( - evaluation_db.variant.variant_name - if evaluation_db.variant.variant_name - else str(evaluation_db.variant_id) - ) - aggregated_results = aggregated_result_of_evaluation_to_pydantic( - evaluation_db.aggregated_results - ) - - return Evaluation( - id=str(evaluation_db.id), - app_id=str(evaluation_db.app_id), - project_id=str(evaluation_db.project_id), - status=evaluation_db.status, - variant_ids=[str(evaluation_db.variant_id)], - variant_revision_ids=[str(evaluation_db.variant_revision_id)], - revisions=[str(evaluation_db.variant_revision.revision)], - variant_names=[variant_name], - testset_id=str(evaluation_db.testset_id), - testset_name=evaluation_db.testset.name, - aggregated_results=aggregated_results, - created_at=str(evaluation_db.created_at), - updated_at=str(evaluation_db.updated_at), - average_cost=evaluation_db.average_cost, - total_cost=evaluation_db.total_cost, - average_latency=evaluation_db.average_latency, - ) - - -async def human_evaluation_db_to_pydantic( - evaluation_db: HumanEvaluationDB, -) -> HumanEvaluation: - evaluation_variants = await db_manager.fetch_human_evaluation_variants( - human_evaluation_id=str(evaluation_db.id) # type: ignore - ) - - revisions = [] - variants_ids = [] - variants_names = [] - variants_revision_ids = [] - for evaluation_variant in evaluation_variants: - variant_name = ( - evaluation_variant.variant.variant_name - if isinstance(evaluation_variant.variant_id, uuid.UUID) - else str(evaluation_variant.variant_id) - ) - variants_names.append(str(variant_name)) - variants_ids.append(str(evaluation_variant.variant_id)) - variant_revision = ( - str(evaluation_variant.variant_revision.revision) - if isinstance(evaluation_variant.variant_revision_id, uuid.UUID) - else " None" - ) - revisions.append(variant_revision) - variants_revision_ids.append(str(evaluation_variant.variant_revision_id)) - - return HumanEvaluation( - id=str(evaluation_db.id), - app_id=str(evaluation_db.app_id), - project_id=str(evaluation_db.project_id), - status=evaluation_db.status, # type: ignore - evaluation_type=evaluation_db.evaluation_type, # type: ignore - variant_ids=variants_ids, - variant_names=variants_names, - testset_id=str(evaluation_db.testset_id), - testset_name=evaluation_db.testset.name, - variants_revision_ids=variants_revision_ids, - revisions=revisions, - created_at=str(evaluation_db.created_at), # type: ignore - updated_at=str(evaluation_db.updated_at), # type: ignore - ) - - -def human_evaluation_scenario_db_to_pydantic( - evaluation_scenario_db: HumanEvaluationScenarioDB, evaluation_id: str -) -> HumanEvaluationScenario: - return HumanEvaluationScenario( - id=str(evaluation_scenario_db.id), - evaluation_id=evaluation_id, - inputs=evaluation_scenario_db.inputs, # type: ignore - outputs=evaluation_scenario_db.outputs, # type: ignore - vote=evaluation_scenario_db.vote, # type: ignore - score=evaluation_scenario_db.score, # type: ignore - correct_answer=evaluation_scenario_db.correct_answer, # type: ignore - is_pinned=evaluation_scenario_db.is_pinned or False, # type: ignore - note=evaluation_scenario_db.note or "", # type: ignore - ) - - -def aggregated_result_of_evaluation_to_pydantic( - evaluation_aggregated_results: List, -) -> List[dict]: - transformed_results = [] - for aggregated_result in evaluation_aggregated_results: - evaluator_config_dict = ( - { - "id": str(aggregated_result.evaluator_config.id), - "name": aggregated_result.evaluator_config.name, - "evaluator_key": aggregated_result.evaluator_config.evaluator_key, - "settings_values": aggregated_result.evaluator_config.settings_values, - "created_at": str(aggregated_result.evaluator_config.created_at), - "updated_at": str(aggregated_result.evaluator_config.updated_at), - } - if isinstance(aggregated_result.evaluator_config_id, uuid.UUID) - else None - ) - transformed_results.append( - { - "evaluator_config": ( - {} if evaluator_config_dict is None else evaluator_config_dict - ), - "result": aggregated_result.result, - } - ) - return transformed_results - - -async def evaluation_scenario_db_to_pydantic( - evaluation_scenario_db: EvaluationScenarioDB, evaluation_id: str -) -> EvaluationScenario: - scenario_results = [ - { - "evaluator_config": str(scenario_result.evaluator_config_id), - "result": scenario_result.result, - } - for scenario_result in evaluation_scenario_db.results - ] - return EvaluationScenario( - id=str(evaluation_scenario_db.id), - evaluation_id=evaluation_id, - inputs=[ - EvaluationScenarioInput(**scenario_input) # type: ignore - for scenario_input in evaluation_scenario_db.inputs - ], - outputs=[ - EvaluationScenarioOutput(**scenario_output) # type: ignore - for scenario_output in evaluation_scenario_db.outputs - ], - correct_answers=[ - CorrectAnswer(**correct_answer) # type: ignore - for correct_answer in evaluation_scenario_db.correct_answers - ], - is_pinned=evaluation_scenario_db.is_pinned or False, # type: ignore - note=evaluation_scenario_db.note or "", # type: ignore - results=scenario_results, # type: ignore - ) diff --git a/api/oss/src/services/db_manager.py b/api/oss/src/services/db_manager.py index 9833c104dc..54fef759d3 100644 --- a/api/oss/src/services/db_manager.py +++ b/api/oss/src/services/db_manager.py @@ -10,7 +10,7 @@ from sqlalchemy import func, or_, asc from sqlalchemy.ext.asyncio import AsyncSession from supertokens_python.types import AccountInfo -from sqlalchemy.orm import joinedload, load_only, aliased +from sqlalchemy.orm import joinedload, load_only, selectinload from sqlalchemy.exc import NoResultFound, MultipleResultsFound, SQLAlchemyError from supertokens_python.asyncio import list_users_by_account_info from supertokens_python.asyncio import delete_user as delete_user_from_supertokens @@ -22,6 +22,7 @@ from oss.src.dbs.postgres.shared.engine import engine from oss.src.services.json_importer_helper import get_json + if is_ee(): from ee.src.models.db_models import ProjectDB, WorkspaceDB else: @@ -47,25 +48,6 @@ AppType, ConfigDB, ) -from oss.src.models.shared_models import ( - Result, - CorrectAnswer, - AggregatedResult, - EvaluationScenarioResult, - EvaluationScenarioInput, - EvaluationScenarioOutput, - HumanEvaluationScenarioInput, -) -from oss.src.models.db_models import ( - EvaluationDB, - HumanEvaluationDB, - EvaluationScenarioDB, - HumanEvaluationScenarioDB, - HumanEvaluationVariantDB, - EvaluationScenarioResultDB, - EvaluationEvaluatorConfigDB, - EvaluationAggregatedResultDB, -) log = get_module_logger(__name__) @@ -2976,7 +2958,7 @@ async def find_previous_variant_from_base_id( async def update_base( base_id: str, **kwargs: dict, -) -> Optional[VariantBaseDB]: +) -> VariantBaseDB: """Update the base object in the database with the provided id. Arguments: @@ -3084,9 +3066,7 @@ async def fetch_evaluators_configs(project_id: str): return evaluators_configs -async def fetch_evaluator_config( - evaluator_config_id: str, -) -> Optional[EvaluatorConfigDB]: +async def fetch_evaluator_config(evaluator_config_id: str) -> EvaluatorConfigDB: """Fetch evaluator configurations from the database. Args: @@ -3139,7 +3119,7 @@ async def check_if_evaluators_exist_in_list_of_evaluators_configs( async def fetch_evaluator_config_by_appId( app_id: str, evaluator_name: str -) -> Optional[EvaluatorConfigDB]: +) -> EvaluatorConfigDB: """Fetch the evaluator config from the database using the app Id and evaluator name. Args: @@ -3290,7 +3270,7 @@ async def fetch_corresponding_object_uuid(table_name: str, object_id: str) -> st return str(object_mapping.uuid) -async def fetch_default_project() -> Optional[ProjectDB]: +async def fetch_default_project() -> ProjectDB: """ Fetch the default project from the database. Returns: @@ -3303,9 +3283,7 @@ async def fetch_default_project() -> Optional[ProjectDB]: return default_project -async def get_user_api_key_by_prefix( - api_key_prefix: str, user_id: str -) -> Optional[APIKeyDB]: +async def get_user_api_key_by_prefix(api_key_prefix: str, user_id: str) -> APIKeyDB: """ Gets the user api key by prefix. @@ -3351,755 +3329,3 @@ async def update_api_key_timestamp(api_key_id: str) -> None: await session.commit() await session.refresh(api_key) - - -async def fetch_evaluation_status_by_id( - project_id: str, - evaluation_id: str, -) -> Optional[str]: - """Fetch only the status of an evaluation by its ID.""" - assert evaluation_id is not None, "evaluation_id cannot be None" - - async with engine.core_session() as session: - query = ( - select(EvaluationDB) - .filter_by(project_id=project_id, id=uuid.UUID(evaluation_id)) - .options(load_only(EvaluationDB.status)) - ) - - result = await session.execute(query) - evaluation = result.scalars().first() - return evaluation.status if evaluation else None - - -async def fetch_evaluation_by_id( - project_id: str, - evaluation_id: str, -) -> Optional[EvaluationDB]: - """Fetches a evaluation by its ID. - - Args: - evaluation_id (str): The ID of the evaluation to fetch. - - Returns: - EvaluationDB: The fetched evaluation, or None if no evaluation was found. - """ - - assert evaluation_id is not None, "evaluation_id cannot be None" - async with engine.core_session() as session: - base_query = select(EvaluationDB).filter_by( - project_id=project_id, - id=uuid.UUID(evaluation_id), - ) - query = base_query.options( - joinedload(EvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore - ) - - result = await session.execute( - query.options( - joinedload(EvaluationDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore - joinedload(EvaluationDB.variant_revision.of_type(AppVariantRevisionsDB)).load_only(AppVariantRevisionsDB.revision), # type: ignore - joinedload( - EvaluationDB.aggregated_results.of_type( - EvaluationAggregatedResultDB - ) - ).joinedload(EvaluationAggregatedResultDB.evaluator_config), - ) - ) - evaluation = result.unique().scalars().first() - return evaluation - - -async def list_human_evaluations(app_id: str, project_id: str): - """ - Fetches human evaluations belonging to an App. - - Args: - app_id (str): The application identifier - """ - - async with engine.core_session() as session: - base_query = ( - select(HumanEvaluationDB) - .filter_by(app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id)) - .filter(HumanEvaluationDB.testset_id.isnot(None)) - ) - query = base_query.options( - joinedload(HumanEvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore - ) - - result = await session.execute(query) - human_evaluations = result.scalars().all() - return human_evaluations - - -async def create_human_evaluation( - app: AppDB, - status: str, - evaluation_type: str, - testset_id: str, - variants_ids: List[str], -): - """ - Creates a human evaluation. - - Args: - app (AppDB: The app object - status (str): The status of the evaluation - evaluation_type (str): The evaluation type - testset_id (str): The ID of the evaluation testset - variants_ids (List[str]): The IDs of the variants for the evaluation - """ - - async with engine.core_session() as session: - human_evaluation = HumanEvaluationDB( - app_id=app.id, - project_id=app.project_id, - status=status, - evaluation_type=evaluation_type, - testset_id=testset_id, - ) - - session.add(human_evaluation) - await session.commit() - await session.refresh(human_evaluation, attribute_names=["testset"]) - - # create variants for human evaluation - await create_human_evaluation_variants( - human_evaluation_id=str(human_evaluation.id), - variants_ids=variants_ids, - ) - return human_evaluation - - -async def fetch_human_evaluation_variants(human_evaluation_id: str): - """ - Fetches human evaluation variants. - - Args: - human_evaluation_id (str): The human evaluation ID - - Returns: - The human evaluation variants. - """ - - async with engine.core_session() as session: - base_query = select(HumanEvaluationVariantDB).filter_by( - human_evaluation_id=uuid.UUID(human_evaluation_id) - ) - query = base_query.options( - joinedload(HumanEvaluationVariantDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore - joinedload(HumanEvaluationVariantDB.variant_revision.of_type(AppVariantRevisionsDB)).load_only(AppVariantRevisionsDB.id, AppVariantRevisionsDB.revision), # type: ignore - ) - - result = await session.execute(query) - evaluation_variants = result.scalars().all() - return evaluation_variants - - -async def create_human_evaluation_variants( - human_evaluation_id: str, variants_ids: List[str] -): - """ - Creates human evaluation variants. - - Args: - human_evaluation_id (str): The human evaluation identifier - variants_ids (List[str]): The variants identifiers - project_id (str): The project ID - """ - - variants_dict = {} - for variant_id in variants_ids: - variant = await fetch_app_variant_by_id(app_variant_id=variant_id) - if variant: - variants_dict[variant_id] = variant - - variants_revisions_dict = {} - for variant_id, variant in variants_dict.items(): - variant_revision = await fetch_app_variant_revision_by_variant( - app_variant_id=str(variant.id), project_id=str(variant.project_id), revision=variant.revision # type: ignore - ) - if variant_revision: - variants_revisions_dict[variant_id] = variant_revision - - if set(variants_dict.keys()) != set(variants_revisions_dict.keys()): - raise ValueError("Mismatch between variants and their revisions") - - async with engine.core_session() as session: - for variant_id in variants_ids: - variant = variants_dict[variant_id] - variant_revision = variants_revisions_dict[variant_id] - human_evaluation_variant = HumanEvaluationVariantDB( - human_evaluation_id=uuid.UUID(human_evaluation_id), - variant_id=variant.id, # type: ignore - variant_revision_id=variant_revision.id, # type: ignore - ) - session.add(human_evaluation_variant) - - await session.commit() - - -async def fetch_human_evaluation_by_id( - evaluation_id: str, -) -> Optional[HumanEvaluationDB]: - """ - Fetches a evaluation by its ID. - - Args: - evaluation_id (str): The ID of the evaluation to fetch. - - Returns: - EvaluationDB: The fetched evaluation, or None if no evaluation was found. - """ - - assert evaluation_id is not None, "evaluation_id cannot be None" - async with engine.core_session() as session: - base_query = select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) - query = base_query.options( - joinedload(HumanEvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore - ) - result = await session.execute(query) - evaluation = result.scalars().first() - return evaluation - - -async def update_human_evaluation(evaluation_id: str, values_to_update: dict): - """Updates human evaluation with the specified values. - - Args: - evaluation_id (str): The evaluation ID - values_to_update (dict): The values to update - - Exceptions: - NoResultFound: if human evaluation is not found - """ - - async with engine.core_session() as session: - result = await session.execute( - select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) - ) - human_evaluation = result.scalars().first() - if not human_evaluation: - raise NoResultFound(f"Human evaluation with id {evaluation_id} not found") - - for key, value in values_to_update.items(): - if hasattr(human_evaluation, key): - setattr(human_evaluation, key, value) - - await session.commit() - await session.refresh(human_evaluation) - - -async def delete_human_evaluation(evaluation_id: str): - """Delete the evaluation by its ID. - - Args: - evaluation_id (str): The ID of the evaluation to delete. - """ - - assert evaluation_id is not None, "evaluation_id cannot be None" - async with engine.core_session() as session: - result = await session.execute( - select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) - ) - evaluation = result.scalars().first() - if not evaluation: - raise NoResultFound(f"Human evaluation with id {evaluation_id} not found") - - await session.delete(evaluation) - await session.commit() - - -async def create_human_evaluation_scenario( - inputs: List[HumanEvaluationScenarioInput], - project_id: str, - evaluation_id: str, - evaluation_extend: Dict[str, Any], -): - """ - Creates a human evaluation scenario. - - Args: - inputs (List[HumanEvaluationScenarioInput]): The inputs. - evaluation_id (str): The evaluation identifier. - evaluation_extend (Dict[str, any]): An extended required payload for the evaluation scenario. Contains score, vote, and correct_answer. - """ - - async with engine.core_session() as session: - evaluation_scenario = HumanEvaluationScenarioDB( - **evaluation_extend, - project_id=uuid.UUID(project_id), - evaluation_id=uuid.UUID(evaluation_id), - inputs=[input.model_dump() for input in inputs], - outputs=[], - ) - - session.add(evaluation_scenario) - await session.commit() - - -async def update_human_evaluation_scenario( - evaluation_scenario_id: str, values_to_update: dict -): - """Updates human evaluation scenario with the specified values. - - Args: - evaluation_scenario_id (str): The evaluation scenario ID - values_to_update (dict): The values to update - - Exceptions: - NoResultFound: if human evaluation scenario is not found - """ - - async with engine.core_session() as session: - result = await session.execute( - select(HumanEvaluationScenarioDB).filter_by( - id=uuid.UUID(evaluation_scenario_id) - ) - ) - human_evaluation_scenario = result.scalars().first() - if not human_evaluation_scenario: - raise NoResultFound( - f"Human evaluation scenario with id {evaluation_scenario_id} not found" - ) - - for key, value in values_to_update.items(): - if hasattr(human_evaluation_scenario, key): - setattr(human_evaluation_scenario, key, value) - - await session.commit() - await session.refresh(human_evaluation_scenario) - - -async def fetch_human_evaluation_scenarios(evaluation_id: str): - """ - Fetches human evaluation scenarios. - - Args: - evaluation_id (str): The evaluation identifier - - Returns: - The evaluation scenarios. - """ - - async with engine.core_session() as session: - result = await session.execute( - select(HumanEvaluationScenarioDB) - .filter_by(evaluation_id=uuid.UUID(evaluation_id)) - .order_by(asc(HumanEvaluationScenarioDB.created_at)) - ) - evaluation_scenarios = result.scalars().all() - return evaluation_scenarios - - -async def fetch_evaluation_scenarios(evaluation_id: str, project_id: str): - """ - Fetches evaluation scenarios. - - Args: - evaluation_id (str): The evaluation identifier - project_id (str): The ID of the project - - Returns: - The evaluation scenarios. - """ - - async with engine.core_session() as session: - result = await session.execute( - select(EvaluationScenarioDB) - .filter_by( - evaluation_id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) - ) - .options(joinedload(EvaluationScenarioDB.results)) - ) - evaluation_scenarios = result.unique().scalars().all() - return evaluation_scenarios - - -async def fetch_evaluation_scenario_by_id( - evaluation_scenario_id: str, -) -> Optional[EvaluationScenarioDB]: - """Fetches and evaluation scenario by its ID. - - Args: - evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. - - Returns: - EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. - """ - - assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" - async with engine.core_session() as session: - result = await session.execute( - select(EvaluationScenarioDB).filter_by(id=uuid.UUID(evaluation_scenario_id)) - ) - evaluation_scenario = result.scalars().first() - return evaluation_scenario - - -async def fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id: str, -) -> Optional[HumanEvaluationScenarioDB]: - """Fetches and evaluation scenario by its ID. - - Args: - evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. - - Returns: - EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. - """ - - assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" - async with engine.core_session() as session: - result = await session.execute( - select(HumanEvaluationScenarioDB).filter_by( - id=uuid.UUID(evaluation_scenario_id) - ) - ) - evaluation_scenario = result.scalars().first() - return evaluation_scenario - - -async def fetch_human_evaluation_scenario_by_evaluation_id( - evaluation_id: str, -) -> Optional[HumanEvaluationScenarioDB]: - """Fetches and evaluation scenario by its ID. - Args: - evaluation_id (str): The ID of the evaluation object to use in fetching the human evaluation. - Returns: - EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. - """ - - evaluation = await fetch_human_evaluation_by_id(evaluation_id) - async with engine.core_session() as session: - result = await session.execute( - select(HumanEvaluationScenarioDB).filter_by( - evaluation_id=evaluation.id # type: ignore - ) - ) - human_eval_scenario = result.scalars().first() - return human_eval_scenario - - -async def create_new_evaluation( - app: AppDB, - project_id: str, - testset: TestsetDB, - status: Result, - variant: str, - variant_revision: str, -) -> EvaluationDB: - """Create a new evaluation scenario. - Returns: - EvaluationScenarioDB: The created evaluation scenario. - """ - - async with engine.core_session() as session: - evaluation = EvaluationDB( - app_id=app.id, - project_id=uuid.UUID(project_id), - testset_id=testset.id, - status=status.model_dump(), - variant_id=uuid.UUID(variant), - variant_revision_id=uuid.UUID(variant_revision), - ) - - session.add(evaluation) - await session.commit() - await session.refresh( - evaluation, - attribute_names=[ - "testset", - "variant", - "variant_revision", - "aggregated_results", - ], - ) - - return evaluation - - -async def list_evaluations(app_id: str, project_id: str): - """Retrieves evaluations of the specified app from the db. - - Args: - app_id (str): The ID of the app - project_id (str): The ID of the project - """ - - async with engine.core_session() as session: - base_query = select(EvaluationDB).filter_by( - app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) - ) - query = base_query.options( - joinedload(EvaluationDB.testset.of_type(TestsetDB)).load_only(TestsetDB.id, TestsetDB.name), # type: ignore - ) - - result = await session.execute( - query.options( - joinedload(EvaluationDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore - joinedload(EvaluationDB.variant_revision.of_type(AppVariantRevisionsDB)).load_only(AppVariantRevisionsDB.revision), # type: ignore - joinedload( - EvaluationDB.aggregated_results.of_type( - EvaluationAggregatedResultDB - ) - ).joinedload(EvaluationAggregatedResultDB.evaluator_config), - ) - ) - evaluations = result.unique().scalars().all() - return evaluations - - -async def fetch_evaluations_by_resource( - resource_type: str, project_id: str, resource_ids: List[str] -): - """ - Fetches an evaluations by resource. - - Args: - resource_type (str): The resource type - project_id (str): The ID of the project - resource_ids (List[str]): The resource identifiers - - Returns: - The evaluations by resource. - - Raises: - HTTPException:400 resource_type {type} is not supported - """ - - ids = list(map(uuid.UUID, resource_ids)) - - async with engine.core_session() as session: - if resource_type == "variant": - result_evaluations = await session.execute( - select(EvaluationDB) - .filter( - EvaluationDB.variant_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), - ) - .options(load_only(EvaluationDB.id)) # type: ignore - ) - result_human_evaluations = await session.execute( - select(HumanEvaluationDB) - .join(HumanEvaluationVariantDB) - .filter( - HumanEvaluationVariantDB.variant_id.in_(ids), - HumanEvaluationDB.project_id == uuid.UUID(project_id), - ) - .options(load_only(HumanEvaluationDB.id)) # type: ignore - ) - res_evaluations = list(result_evaluations.scalars().all()) - res_human_evaluations = list(result_human_evaluations.scalars().all()) - return res_evaluations + res_human_evaluations - - elif resource_type == "testset": - result_evaluations = await session.execute( - select(EvaluationDB) - .filter( - EvaluationDB.testset_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), - ) - .options(load_only(EvaluationDB.id)) # type: ignore - ) - result_human_evaluations = await session.execute( - select(HumanEvaluationDB) - .filter( - HumanEvaluationDB.testset_id.in_(ids), - HumanEvaluationDB.project_id - == uuid.UUID(project_id), # Fixed to match HumanEvaluationDB - ) - .options(load_only(HumanEvaluationDB.id)) # type: ignore - ) - res_evaluations = list(result_evaluations.scalars().all()) - res_human_evaluations = list(result_human_evaluations.scalars().all()) - return res_evaluations + res_human_evaluations - - elif resource_type == "evaluator_config": - query = ( - select(EvaluationDB) - .join(EvaluationDB.evaluator_configs) - .filter( - EvaluationEvaluatorConfigDB.evaluator_config_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), - ) - ) - result = await session.execute(query) - res = result.scalars().all() - return res - - raise HTTPException( - status_code=400, - detail=f"resource_type {resource_type} is not supported", - ) - - -async def delete_evaluations(evaluation_ids: List[str]) -> None: - """Delete evaluations based on the ids provided from the db. - - Args: - evaluations_ids (list[str]): The IDs of the evaluation - """ - - async with engine.core_session() as session: - query = select(EvaluationDB).where(EvaluationDB.id.in_(evaluation_ids)) - result = await session.execute(query) - evaluations = result.scalars().all() - for evaluation in evaluations: - await session.delete(evaluation) - await session.commit() - - -async def create_new_evaluation_scenario( - project_id: str, - evaluation_id: str, - variant_id: str, - inputs: List[EvaluationScenarioInput], - outputs: List[EvaluationScenarioOutput], - correct_answers: Optional[List[CorrectAnswer]], - is_pinned: Optional[bool], - note: Optional[str], - results: List[EvaluationScenarioResult], -) -> EvaluationScenarioDB: - """Create a new evaluation scenario. - - Returns: - EvaluationScenarioDB: The created evaluation scenario. - """ - - async with engine.core_session() as session: - evaluation_scenario = EvaluationScenarioDB( - project_id=uuid.UUID(project_id), - evaluation_id=uuid.UUID(evaluation_id), - variant_id=uuid.UUID(variant_id), - inputs=[input.model_dump() for input in inputs], - outputs=[output.model_dump() for output in outputs], - correct_answers=( - [correct_answer.model_dump() for correct_answer in correct_answers] - if correct_answers is not None - else [] - ), - is_pinned=is_pinned, - note=note, - ) - - session.add(evaluation_scenario) - await session.commit() - await session.refresh(evaluation_scenario) - - # create evaluation scenario result - for result in results: - evaluation_scenario_result = EvaluationScenarioResultDB( - evaluation_scenario_id=evaluation_scenario.id, - evaluator_config_id=uuid.UUID(result.evaluator_config), - result=result.result.model_dump(), - ) - - session.add(evaluation_scenario_result) - - await session.commit() # ensures that scenario results insertion is committed - await session.refresh(evaluation_scenario) - - return evaluation_scenario - - -async def update_evaluation_with_aggregated_results( - evaluation_id: str, aggregated_results: List[AggregatedResult] -): - async with engine.core_session() as session: - for result in aggregated_results: - aggregated_result = EvaluationAggregatedResultDB( - evaluation_id=uuid.UUID(evaluation_id), - evaluator_config_id=uuid.UUID(result.evaluator_config), - result=result.result.model_dump(), - ) - session.add(aggregated_result) - - await session.commit() - - -async def fetch_eval_aggregated_results(evaluation_id: str): - """ - Fetches an evaluation aggregated results by evaluation identifier. - - Args: - evaluation_id (str): The evaluation identifier - - Returns: - The evaluation aggregated results by evaluation identifier. - """ - - async with engine.core_session() as session: - base_query = select(EvaluationAggregatedResultDB).filter_by( - evaluation_id=uuid.UUID(evaluation_id) - ) - query = base_query.options( - joinedload( - EvaluationAggregatedResultDB.evaluator_config.of_type(EvaluatorConfigDB) - ).load_only( - EvaluatorConfigDB.id, # type: ignore - EvaluatorConfigDB.name, # type: ignore - EvaluatorConfigDB.evaluator_key, # type: ignore - EvaluatorConfigDB.settings_values, # type: ignore - EvaluatorConfigDB.created_at, # type: ignore - EvaluatorConfigDB.updated_at, # type: ignore - ) - ) - - result = await session.execute(query) - aggregated_results = result.scalars().all() - return aggregated_results - - -async def update_evaluation( - evaluation_id: str, project_id: str, updates: Dict[str, Any] -) -> Optional[EvaluationDB]: - """ - Update an evaluator configuration in the database with the provided id. - - Arguments: - evaluation_id (str): The ID of the evaluator configuration to be updated. - project_id (str): The ID of the project. - updates (Dict[str, Any]): The updates to apply to the evaluator configuration. - - Returns: - EvaluatorConfigDB: The updated evaluator configuration object. - """ - - async with engine.core_session() as session: - result = await session.execute( - select(EvaluationDB).filter_by( - id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) - ) - ) - evaluation = result.scalars().first() - for key, value in updates.items(): - if hasattr(evaluation, key): - setattr(evaluation, key, value) - - await session.commit() - await session.refresh(evaluation) - - return evaluation - - -async def check_if_evaluation_contains_failed_evaluation_scenarios( - evaluation_id: str, -) -> bool: - async with engine.core_session() as session: - EvaluationResultAlias = aliased(EvaluationScenarioResultDB) - query = ( - select(func.count(EvaluationScenarioDB.id)) - .join(EvaluationResultAlias, EvaluationScenarioDB.results) - .where( - EvaluationScenarioDB.evaluation_id == uuid.UUID(evaluation_id), - EvaluationResultAlias.result["type"].astext == "error", - ) - ) - - result = await session.execute(query) - count = result.scalar() - if not count: - return False - return count > 0 diff --git a/api/pyproject.toml b/api/pyproject.toml index e8c38e1e19..23123058db 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "api" -version = "0.62.1" +version = "0.61.2" description = "Agenta API" authors = [ { name = "Mahmoud Mabrouk", email = "mahmoud@agenta.ai" }, diff --git a/docs/blog/entries/customize-llm-as-a-judge-output-schemas.mdx b/docs/blog/entries/customize-llm-as-a-judge-output-schemas.mdx deleted file mode 100644 index 033e29371b..0000000000 --- a/docs/blog/entries/customize-llm-as-a-judge-output-schemas.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: "Customize LLM-as-a-Judge Output Schemas" -slug: customize-llm-as-a-judge-output-schemas -date: 2025-11-10 -tags: [v0.62.0] -description: "Learn how to customize LLM-as-a-Judge evaluator output schemas with binary, multiclass, or custom JSON formats. Enable reasoning for better evaluation quality and structure feedback to match your workflow needs." ---- - -import Image from "@theme/IdealImage"; - -The LLM-as-a-Judge evaluator now supports custom output schemas. You can define exactly what feedback structure you need for your evaluations. - - -
- Custom output schemas in LLM-as-a-Judge - Example 1 - Custom output schemas in LLM-as-a-Judge - Example 2 -
- -## What's New - -### **Flexible Output Types** -Configure the evaluator to return different types of outputs: -- **Binary**: Return a simple yes/no or pass/fail score -- **Multiclass**: Choose from multiple predefined categories -- **Custom JSON**: Define any structure that fits your use case - -### **Include Reasoning for Better Quality** -Enable the reasoning option to have the LLM explain its evaluation. This improves prediction quality because the model thinks through its assessment before providing a score. - -When you include reasoning, the evaluator returns both the score and a detailed explanation of how it arrived at that judgment. - -### **Advanced: Raw JSON Schema** -For complete control, provide a raw JSON schema. The evaluator will return responses that match your exact structure. - -This lets you capture multiple scores, categorical labels, confidence levels, and custom fields in a single evaluation pass. You can structure the output however your workflow requires. - -### **Use Custom Schemas in Evaluation** -Once configured, your custom schemas work seamlessly in the evaluation workflow. The results display in the evaluation dashboard with all your custom fields visible. - -This makes it easy to analyze multiple dimensions of quality in a single evaluation run. - -## Example Use Cases - -**Binary Score with Reasoning:** -Return a simple correct/incorrect judgment along with an explanation of why the output succeeded or failed. - -**Multi-dimensional Feedback:** -Capture separate scores for accuracy, relevance, completeness, and tone in one evaluation. Include reasoning for each dimension. - -**Structured Classification:** -Return categorical labels (excellent/good/fair/poor) along with specific issues found and suggestions for improvement. - -## Getting Started - -To use custom output schemas with LLM-as-a-Judge: - -1. Open the evaluator configuration -2. Select your desired output type (binary, multiclass, or custom) -3. Enable reasoning if you want explanations -4. For advanced use, provide your JSON schema -5. Run your evaluation - -Learn more in the [LLM-as-a-Judge documentation](/evaluation/configure-evaluators/llm-as-a-judge). diff --git a/docs/blog/main.mdx b/docs/blog/main.mdx index 66a0256cb0..e55eed8a9c 100644 --- a/docs/blog/main.mdx +++ b/docs/blog/main.mdx @@ -10,33 +10,6 @@ import Image from "@theme/IdealImage";
-### [Customize LLM-as-a-Judge Output Schemas](/changelog/customize-llm-as-a-judge-output-schemas) - -_10 November 2025_ - -**v0.62.0** - -
- Custom output schemas in LLM-as-a-Judge - Example 1 - Custom output schemas in LLM-as-a-Judge - Example 2 -
- -The LLM-as-a-Judge evaluator now supports custom output schemas. Create multiple feedback outputs per evaluator with any structure you need. - -You can configure output types (binary, multiclass), include reasoning to improve prediction quality, or provide a raw JSON schema with any structure you define. Use these custom schemas in your evaluations to capture exactly the feedback you need. - -Learn more in the [LLM-as-a-Judge documentation](/evaluation/configure-evaluators/llm-as-a-judge). - ---- - ### [Documentation Overhaul](/changelog/documentation-architecture-overhaul) _3 November 2025_ diff --git a/docs/docs/evaluation/configure-evaluators/05-llm-as-a-judge.mdx b/docs/docs/evaluation/configure-evaluators/05-llm-as-a-judge.mdx index a6489b156d..399dfde99e 100644 --- a/docs/docs/evaluation/configure-evaluators/05-llm-as-a-judge.mdx +++ b/docs/docs/evaluation/configure-evaluators/05-llm-as-a-judge.mdx @@ -2,8 +2,6 @@ title: "LLM-as-a-Judge" --- -import Image from "@theme/IdealImage"; - LLM-as-a-Judge is an evaluator that uses an LLM to assess LLM outputs. It's particularly useful for evaluating text generation tasks or chatbots where there's no single correct answer. ![Configuration of LLM-as-a-judge](/images/evaluation/configure-evaluators-3.png) @@ -58,28 +56,4 @@ ANSWER ONLY THE SCORE. DO NOT USE MARKDOWN. DO NOT PROVIDE ANYTHING OTHER THAN T ### The Model -The model can be configured to select one of the supported options (`gpt-4o`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `claude-3-5-sonnet`, `claude-3-5-haiku`, `claude-3-5-opus`). To use LLM-as-a-Judge, you'll need to set your OpenAI or Anthropic API key in the settings. The key is saved locally and only sent to our servers for evaluation; it's not stored there. - -### Output Schema - -You can configure the output schema to control what the LLM evaluator returns. This allows you to get structured feedback tailored to your evaluation needs. - -#### Basic Configuration - -The basic configuration lets you choose from common output types: - -- **Binary**: Returns a simple pass/fail or yes/no judgment -- **Multiclass**: Returns a classification from a predefined set of categories -- **Continuous**: Returns a score between a minimum and maximum value - -You can also enable **Include Reasoning** to have the evaluator explain its judgment. This option significantly improves the quality of evaluations by making the LLM's decision process transparent. - -Basic output schema configuration - - -#### Advanced Configuration - -For complete control, you can provide a custom JSON schema. This lets you define any output structure you need. For example, you could return multiple scores, confidence levels, detailed feedback categories, or any combination of fields. - - -Advanced output schema configuration +The model can be configured to select one of the supported options (`gpt-3.5-turbo`, `gpt-4o`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `claude-3-5-sonnet`, `claude-3-5-haiku`, `claude-3-5-opus`). To use LLM-as-a-Judge, you'll need to set your OpenAI or Anthropic API key in the settings. The key is saved locally and only sent to our servers for evaluation—it's not stored there. diff --git a/docs/static/images/changelog/changelog-llm-as-a-judge-response-1.png b/docs/static/images/changelog/changelog-llm-as-a-judge-response-1.png deleted file mode 100644 index 452c1b718a4ad4c8655af1c71fb6b05874fe7c4e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 68321 zcmeFY^;?_WvNw(tC=_W+3&klED8-5w4aF&L#hu{6T|y~sDN@|ExVr=oQc7`XaHmLs z;1(bxe0lac@80|D^L+n+_qyI+?knq_wPwwl`K+0B&#Z)Ns40*TQ4?WdVUZ{)%4%U@ z-P^;$!im0*fA{8T>6e4MMwY#fzNfybil~(fh}**2#S+Nv3v#`qV_}H{d|fT99D$yU zmOxv3X9=dmHYgLLy|o0Bo}lV$RaY6HoxP&JJ5bwSO~=aL(MrUc2_Q)%?kjr700i{3 zVDtq!IeUouN-+I{ujt+Hzh3h&G5({Ar=tXuzN!YJjEg&vQHc9B_iH9eB1Um{Ya3B5 zS^57SeAkj-vh(zG73Jaa@$upI;pcX7x8>m#5fR~e&Bw#X$930(%frvv)54d_*@O8n z7XQUV7U*H+Ztv=8@8Zn(7f%aI7cWl16-k zJpR%8KTO2`p*Ye1nCZVp{F^=QhW^*PJ3HS2hUcG%xoi9rb3o@ipt;{6>Wh)63l%VJ1{y&dmN9^jMI`?wJ7GXg=8`ziWfsSR0;`1jM|HiZ^6TD?`yBV< za2`>Nx&EPrW~^Nr$gkU4^~fwedXI~xZdv;&94nUi5r8Wr>+8nzD@LbL_|fHkIOASs za`Ncg=BE}iFXTTQset`&%uZflxNO^Butg@lTx+QxCRz?nVOIoJUI7-4p)6dw0?;t* zyn+|G>Mtz|I*{=*Pr4Jq&=fe)mwWnb0bYwCK` zjHAB#;G@)=uh<;eG%rE~81I#Li`Ec6`M^~3U=Q!fXWjyu zN$7ESpJDTK-}5CaV!Yq;*7_?ByG(j4y)}-3wC69XlTSCf8ZYr9LLWbueDQJmt|M5}JI*|Bf1eD1hG)BH8;g^&UspZjEI7y`o;?Su)Zvy{>Rq{LBaN zUa?iaW@<@N;ohtxuS+4$FJ?UnFF(XfYib(4)P6#@8BHYJm-kphdJ6VQd&0$$m-v(|72F%k>+p;` z3inHR_|F%X@2!Yl7pCT?vJSHhgNE;KNl-)oYhJ3sJREw9&OOEPUPXA~g@I zO?X4AMrHfaqX%zIznRk*yNcF_)(2boD@}LM59Kw%<78i+K+f!#l_;LRnYFQXg5&&4 z_K%!zV^*c^Gr8V(ye~$maDV?3&8J5_Pc)$VB%nAa?5jmZspji6t5KEfOBkR zA}FDQLyTP^u0HlbymG<&VnyJ~cb~bYkPq02F@8&h0+(d4(_G}_oB6Fg0 z;v>9@k0ChP&kb@F*FH(-^nA)|&ug?Su#C1W{39aA`&4eKn6I>SGHX(MQfzW8a*~pl z!i^GSG+ATkIzhmq%c93prIo9d{~J^|HYfm8-+Z#UG{`j=qdWfU=d0(h%5_OgjJ2rr zbTzzmfaMfr*;+y0@yn^oKbKYNid9UP3zb&o9#nE`@aREHB&*&|xlSogznpke&0C~2 z;W@owpKND39{5(ffY+|HfzMDFB(1DEq@Y~jrC6@+_-;W;Ogf<6JO9FpE|1iN?QOAa zv9hhSp0==*PnL7X0j|VHdJ`TcmmJfWWr84F^hXjkk9^@{$yLXVr|ZAQ;W95{MqgEQB^7;hB?9f zVbA%?(x7uT^%V7b%AP4dxaT?wYzKrrXijxbnYYNeHn;-0gp3-D+G_=xtc~6qF4r*` z`5Ay392)&WAj^?PN%NqwFP7xy^;1PdTjiBmmE%p_-ot7$+@n>42izth^eQWAP|KKF zpZXQM;Au=kXr)YYFMAk(PL)oZ7D1cCBd9-U;%~iElh8yyv=f^!o3=#@zsawkl^goJ zskMJHsoS5l?my|3Xz@eQhSs*-o!^JNRj?(<)yLKBaCt4{|m^IE=4XvI-uvJMS0I|&vIDX=R@M) zXBN+%M{GwVh8xox&|EzqWG;L2j<(~4f6&?av!rLHDvq4oLVOo?dmDUXz<`u*oDc)k znwwFg(5E6CRaH6$X{{%VuhMiYbc@$tyJ@<4xtZy|t#FvO7U%DL1^&1{y4(*QaNA6D z{n@OxXnA1wQCr%r=Tx@%heWuQ61P&J(z>{hDc+%1o%%`l1{wQX_I|A*?R+g7*pa=P zJt)u*Oav7(VU}PEk}_v{`S2@0E#HNW%6;@F1^(ROr)iZ2M87^va_~I~Vxu1P9<3V2 zG>f{Q@6fi;g4GVG*98}*qqh7vF%1+Q$sJ9OS<|hmh%eLp(+u-ipsy?AjhePrn4fXt zhwbUAKUCQ>z)Mb@ExxC1T$)Dnr&Fh=Ct=W2OjV~;tCYU0Px<_JEteNg7ZM;JA;T}i?}D1)kk^n;QgEqaS+mK``#Jr>_l=diz2H$yH*OUH zHE9_Gr|9LC!g36f)Q`cTYPyQDN3`hj8F z{JdymYu34IM+tcE@?;&&Ca%^Xlqy8)P#{E+b&pMs&FqF2d77mSE^bwWsS9ULh-pvB z+P`x8l0GA#-teOFtC5upa~;2ACJMIcrc^p9Ft0wlA(Oo)ZkdmOl7AK>D`d z*VYp5I#X5^Mw!pEyi2-s(6-^TGYG66q6~N@L0R?2L?mEp&0#_Cx-l8iG?Nhk^3MP_ zUNx4EHC11IS^72&Mqw^O3UT2CCir45{)_E_k$#Bl{jLREAEtxFwziw9OM3Y7DXJ7{ z2KWVjeg69J(U#_#)m(p}`ee4wt<9NH{_Oie=U-MONc&k>Snu9i3e*;Z0lv7Ux-!@) z*|OXB7=qjOxBt90;|$Q>XkR630hF0ib&8%ABXT;(C&-zrKUWv8I0cvAT+E6g zd`uF$6V#WBJ94fyE;X;LM%b>4v&S@Z6pG&mcL%Yf9L^fi87mM4m^}=JlN>83M)dvR zXY$8bz8=I_?!;J$V63l0TU)yZAzPRS!;df3#f1aLBDktHwhq zEbM##@9}?^4ptY*6@3n}+@tu_NM;_c=au}O5fy8?O7FR(;RvY=bXO$?Ir{GID-?n}SsbK{`)Vx_0|BS8QR z&{@)CBwY{|B#JD*z&qUp|Gu|9_!Bie_czOs!U`#%oORoqtzDD(LQExL-t}SK2FWc2 zFY=eY4LqXy(ZuPy6gy^u#kJ9MpTlY)EI2Jg9b!EYFP&7b6U1-QdN#8)9LLquTSNW7 zFz^oxD|A^ea~XC9UV)~{^;T@6w?Dc%m$9wokV%%#CgWAptovO*6Fe?=ecP;Ibd|Ex z9>i(XRAKjxTCfvlZc_5CMCR{=W^BPzi5wr|X+P?55jY;sjHFyR$w~oSY?e3G*K$2D z75;hpC<_kLaKgP<{E*S=@A(s+L>C%dAQl|F0lL1i6q$VQ-ev#&X@ zcsnNKs%+oMY3Sn&iEs8e$@=^#*G#FT!0Xh5qqSZ#o1tV-YIb+GjDjbVKzJkZP1xUo zQW}qx_RTnp&%vVh@Q%IqOu~fr%-)o0Z&r};kd;$PEU_46RAg=aHW6eIpgPm$b68O% zPVLOR`{-}?XgH3D$5!AX@%QxZ%#5FHH@rL&CP-trOj2`l|14`Kjci?~QNPVp8uRYy z*+LVjty3|MiJ_IBv3FGIwcOumQgH`|wP~tZzt++$aWApG%HU_j;GQWTGxGLgq+Oo7%awl;vj@#q>jf^mKmZXdR&JEnpRK6RYG?wYo9Kv>{Wh~A)k@8fji#tBFr7QFHlW`Wk4BX?Ak}u|O zEc~HMwo_tjw=FoJjnZ@xYT&lg@c&eAv z@12==iQ}sFw`#igz%Gj8jSa<(xNnB*Sg&(Rs+3_#)-__WJ>!d*$lqi3jov-u3gu$V z)#|Cw&_~32U{%&;)tzY;EfP@wz&T(Y z@$0%&bv4m?I8(_TIQGiVeKWQP??my!$v}HS?n`Sn|MeOf2;GPVZ4|_kQXD1?u`4DmVd4YMM|5z6o4FP zq6+}I2B1beveGU*qK9M$doN@yA(E(vCj>ABZv7{HMVb{czF^;2*u7 z4pAh;_cc7Fy^Cgl9*+;h%AUSc9^IR>o@%YU@JJN#_%fMnPf3gT4d?yn>}Jr{BtH4F zwwvJGg~7p~L=>XxG_)wH^1qn#;>_D-358Bc*Kqf>b7uY$Vg3Hr8GU0nF_r+X(lQ9= z+IsreaD~TK1|=V|j6|Npo)IYiQk(r>2Hto)j?LOICJMq=wlQ(Pm_dGPlv_hO0u^OQWQ8$#E1F{eFK!6+Xhp=TyHD#k*%4OaxH8!o~* zShCFCNu;cPFmalI&L;1iU!{a)sF#vvQvT<@|7%;6$KHWQ;7*9}dyDXapw4(Z$UsYm zs~b;nP|0Z|$givJj!{Y0=kf=$n$1Zi|A7suRpUo*T;G%pm}VQ#jU{Psi9fpYo(HEJ zl4k6fPYDP2$Z5rtRqmW0O#LWauFm6sxot*vyy#n0&Jx{-MS@mRP@xw#($77oTIY|O zyp^fa)0)$qf?&y{c6?*}|0%rG{KM5`AyE6MG4SMm!TeatZO_dqp^aME2NPqq*cWUx zGjH{yr5q?OX98i6JqTc9=4tS+PBcNg^3?ST_*PjhOLTLpaFa^>_euF!;fNz>5piXmjgC`7< z3B6%4ifC4l^~`~UJ8(b!WE#O!)O-raU~9PTdMiH4`les{pL_VLzWLI%DjzcAaMDPqSc_7`?U#{w48~_ZAlS(!J_;mC4+i8(zoqL#V-^KuYav`M(k{g zGr8?gRq0^qPhz>Q=e4FL?mWhA&iUeEf0Dkc%yrmSOU(FtRVmhe$ijTwlQ1v2Y2HZrFyp_akh1_!8_|G; z=J@IxHrhR|5pT&^TtO&Zxi9y8fR*?i*gZ|g{z(=*lY-(Fz%m_{#_yK1vGoAJRTMWPS}irW{$kWM!F zno+1r%D}}WKXZqQ(Cx`aD!}(BHc;YL;c$Fk)f?VSnVlTQux zZ*BvXkxuX6sr%U`FV0t$2dM-hamV^b3Ixe~mj2BfYBhamN=yS84mb08To?-v2|T^u zj_mF~Sqat*P5R*_;_{_ACft!GxmYoz+0w*~E9NxshRQp9*&tIvjNtt^ajWo8(gy)W z;i^WYZB`VSydiX~%m|Fa^}-&%gd%=I+Vt^gvR0do7rQ}wiPLO@+;TSWLIFQd6a4$z zTyZ~lM-yhooEo z^5R+%UBaiqu@ZMNjfFCcE%>J6edo1v?tY4>`hhX~<`a<^+52{sayda)SppzWlZt?e zPf=u&S4`7h`l(xZXFr#PQqBqX;A0TH1-=tKaZZojvx2?%FYxt$Ib3HB zCGji#Iyc_@Zjo=T?n%B7SX6rC$Roj#V(Ol2#0gK6+;etv{br}VNZGH}a1OH#;*>Zz zd~)F>?tWGhv_wiP{!&gQ=TTqeQ$iH=HkpeTEDzg4Z_;WkJ4mZG&|qS#8I!j%x+hpY zLSeNZ;oNqbQ?m74WJC}Vu=#23(78UCYiA^&N(vL`eEtl!ij2%o$j(S!1)<@YD|`Hx z=)mo93z5K$uS-G-(X`6}QmA|4=INUb1KC^|8^7X~Mjbsi_Qh`Fh8&EoV}Km_w=grq zj3(z4W}D#oLmACe%zZGR4NWf@IJ=-T<3sPibhO8CSJH(pcKbXB1 z2)NjbXATf*lLfRvZyODXp8jH}#=Oe&8M^n+Lk}YfVLL#qj3!$Mvcj~&bCW|dA9%Ji z^Sep6{_B$l=!xnUxZXS@Fb`?I7IAxMaoeGAbWJe4vZ^p5@$L4|2ok-_wjH#P3=$iW zFe7C*Ytos;Tr7f5(m#c%q~#shpV52t{^Zi?b}B}4#U*lWfz2~o%wAf>Ke22pK0b7W zw50sTY!EM1~JzC zutgC0Ozo1O&1j;gKnU}qgz^!XDF}McrFcxRskZwEb#%s$xQNXm#c56h=}P$}gNJXl zvD@lhciGn;rgxk#{?fA^g`1^4nJvM_cHj7A(g{d2~QgKTN$?O+nu zei?32J>W{q6zo-3VOT#}K4V6&H`ywSFS*Qb3V6Y$=BI~G!T>_%{!QZwXIZ7 zZ;&+N9Xl*#UV`r6{QiYP6?DI%hdh`XWBE34b4|s#5E)D>)6zMk8VPpy<(OS92R8*49F_R_WkcBAr ze>)nzmpWAH#CYdgkEF{v^M&K!jePe{2MVs|kNqK@wPzSb53kJlySVS_=rEl7^9UB) z<`?N|WIyX+i_VF~ADN%q>zF%iXQj+yxHjEM7D5j0HX=5Y;zA2G&%P5}%bjOz9U|OZ z+PTtStVJx_Hni;9`2A>*BEnd99DZ24F!lXCyXJkK%hTz=cV906?jPE++`W}uU9%G` zs&n-x=&8K>s6%sP+sFUdnvgobr+uB3GpbENUxCVN+=XHS~@$=5K9tH8vLDyl()x%ahBspnkU_vJi{FaQ13Wa zxyf=PBmA?SN9&RUP_9#(7is(&OiH-Q!y)AL`&7+~1xtmUxnj$=CBx~bEtC~7W?E5C zS&Ew)Sy7*WVPcA-pkJXEf}spi^nktx(>w3dYxA)MM()EHl}moxuDVhIc0fG(#fZIy zU@89#I?kNnbCVKP%Sk!`1B=Ro-2*&?=Vx33(zG~t_#EQc)@z3k-Ca3TiY*^pqn2VH zs}*YO;|cy1xL&VYjuU}jOux(jlJQL*hn-C#{>d3-U%D3pw}}vGJ!)$1eRdW)V%k=u#$~^} z;Sc-GTEBQYzJ7v^QT`}g%jJ{4=vrBMd$i7D#W_PncP4cUo~mfZKd(BO-oeaD`OPo7 z*w?5TVK4a^@$)-4=q?!AN45CS32XpemR8d?Ic{UCIk4txl|#!5|6viWYqZqpcD-cNkD z&6-a!d_Z#MeXzW+oKJAb`23I(LN-!_xON3hSmS|AbgR`s_I}FD2_^>f_cDTG5gCw_ zm0RP@_qP6qjJ&+nZy>6J#BFcLysy)>4i-xjirYI!rL%x)upjFA!_+ks4Dr{!YC}_c zCagY?HklLn7)RtSvOemWSU3JbtK?0A3YBn=b;8t+`im;x#d&Fwk_E-ibYUoDiGzG4k&1P7_1%!ow+-skUNCaGLr7b_K8cC_ zFzBds;N?3W?XTTb@ zXf9#-ko#Gq1CwwTBmNYMmY=4ym{F7=?cUcVN8-3PVJ-l?cvuf;&eQfQZ_Z1I?il<&aU(0;DqYw?ZG zdcbyjch!GO({>RK3#54(I@|y5Qfpkil@IdUHU5;cMJeWG*42!-R+Hh`w#wizuGX zY?lm&PX0CE!pYFSQNp|Q#vMz;+2PpiE#t;MHCHNL|BvD`P>53OS0dw`69C4PnQMNT z4Vn(EKOZ_~6_&?)?3D&Ct;S~BkKgTsU*mDYP(wxwD?If8^l$v*vqNXDc^&OTT|v7U z7;#T=Wv}ti9HEh*8l)7>;v|JVT9qQ5gZjgsb-aB8dzmw`OJcwM;Clnd_qI0Vu_0Lt z=#Lo4>SLmnS+KSZb^kJ%&v`5H$)W9hn`U==6YIk=By(;AYxv^SIqw3VF5J6vqc*J7 z7r7~<*}q8ch*i0ej|;qsR1ZKY4Ws17@>gKYP9HQvd)CbYN5zT`^xC5R{T5&N4KFSP zo7V}2wkZ>N(b>1FSy>gw_b@)kZqs>``XCGf>Q`&o9+Y3B7=q-qGgC`l&Rf5;f2gFq z8L_FV;amXub`hdiD=jiX6Fl;Z3i(rj$GjLcF8HiiE|l^xoi$62h_Yc!P(yo&_|y1z z10xy%TEGt7LdEpC%f)-o>x!zyLMunV_8GLd7`j>xMzh(PaTp~i_N%gjp*1D!9P;*w zo=?`5&E}Ua&F;6~G95ko=8FajDE67(#z02zJw7}bf9Eh&;*4lq5!)}`qd!&soSZW( zEQ`tOelTMEI}h-Tp|v_c%S9g=y*NL4kA`N&;+jkBRb&cxJ)DkP)iBt6O+VSsL{97@hjP`(SB%6TTtaDPFaoaZgk4}p!@@}&F9K|)V7JL_%YxZ_15Z<1O`Md4A-}W6G~Sj69zD!WaTI47vClxkUyGu6d zj3BWnG01~PhhMESTSiax7Rj8D;*#38h{_eu=9%ez8I<05_v40=aMGiPWsAR?bMKCM zo7AZ=Cpo?P-v9C5u+h@Sm|KmAKt-+4j2)2NiqwLsdHLsZApoPa|!ULGrFEX)iemiTwjht z+B(vfHY#`OEuF5H2|S#juGE>6T}I6(h=`Imy&XP{g!iXg&IP-+(^oKcfuGBT+`fE* znk6f^1SIS*@GLT>3a7D-8e$?k`;nx>!&FayrT*$DZgpzgo2yUXt1e-1&UJF%`ugq` zZErA1pfuwL6QG)6_38#5c9wL%P)k=~9y$9>oST!9h0#D;*10*ioSu;}LOP;*b|a## zOE)3gQOsF0WX;~uRK`^TBhF}5o6wXtxFApG>kUPo`6oKsRgE_n0Xo`dkZ)>@gcbM? zQ!-q1r_)z6!5Q^j*Efs)+=m`>F2TOtVstf(lLRX^3Vm_U&%2n_{pe7RSljIU`qF{# zx^Xn2`konFtd}X{C$Q>bGl<5;^@I9%o9;p~$}zdv4CrnK<0{feIvultR)WE6mTkMd z{94*tiBE%Nfh9Ygo>JkbWwPkJv%|LNtHOey)~wRUdi5qGc{{7r*O5M(IpsiDa%Gd> z#YFzCf3fH98@JdeeYapvvnG`;h`jSHn0dsLSl1x~fOQjWw<9_giWwKN5w-i~xv`g^p)|1b5c)|(Qs#D zTA7U(Nnl#DODNzg_1&rg8Ah1ESfKirMgdNGPn4=UDr=9kjOn z-m3fj;P4M~k(sBS0Iytvgn&8K&da*i$_KOd5NQv|n8C-MC{Gv(jqseoK6U9As{2|) z5lNcJrX*F4X4k^NmO4I{?wBVo@hSMKFS9(^Y*@l&B}8V48@Gr?Sbj;l`d!|qh!0uC2W-^* z%15doOI+_awKnJKHe2*-d_Wd(ow97^G~Ge zXU|f6wJ2^;wX(H&&wWfP6fLSvZ$FGn?tC<>R!QUYMKq@!>FrFLNBpy*9ZP`+7BEvo zU=bN!15Q5<91!We@x6?#9EW}*0AO6ZQU)ETT^feoeQ80Cf4)*}(rwoI?K$Vsa#eQ~ zc8h-TVA!tgeAkqF9=Vj+a=OhVJ zGHgK3tug~Y1;HlX-QA*L%9~G+JDtb<=3(_wUxt)LqjgTotJNO?3;Cxd6Y1f08{^HvyqR{H#; zbO%>0LswqQvnGae#Qd4z+4=9HvV+F^jp$fP)40!(!9vg|jR8`XbwA&2(h=3!n zmxlIkY)*U4P-zdi=L2V+B3oPFwgJ1h7m5l_zg-6FtOsaasc-j)(31D{2gC`JHof}XQzDmyc0v%q@B?>rEqEwBpi%eB_|i6FF9 zt5taA_9{7bO$Pc(PsqGDs0M*W?9W5F*cU^}Sc3atTcTHj&M@O3nfoc4#t>yR1Dzv~ zL~6z3?F}ZsvR4o2AI+-G0!IOX!`|sXjwjlx!)PyipdqZu&01BS?1<_CJ`73+Vq7fd z!+MiF1Z9BtUeV^ zPgXK)FsSDU_k(WpLiyh=y^TRed6@Te+H9B&&XF^kr$`|pRr%jg)^HdWe4nfikdqDg zy>C2V*;(0qkj}9L(=WzgO+4psEh)mCd5xyO9E!ioYXL~asz?0JX-LrB2%Z%Mvb5!fgi+Cy?dcj$d}o=V*bAGYqc1yV#p^G7D@qCh#98wL)(e12yeom{vtN z9E{F_?9xGmn(~$Wp4x1U=a#roqrJkjW?bqPQEnrx=iblGDh%rG?Robdmbb&XgOc&KW1-|KtG zY{W$dGI3xp12q*WGJR+!wp&(J9n!vfva{&oG_}2kJKUkB6AZ+787Jrmn@Gb?;X_sj z?2!Yk{0~1hRZLI_#U<`izHh`9+vmUTxr?H8;U@5#Q;xX+Esz&Ew>jJ;j{G=UrlMqK z`tpl^5&M~O+Hp6Vd;Sik8bQURqtkTr+pkOHjb{MsgSb=G@#v#b6=Qc^>T_yJUJNTdh7JnL=Z8Ft?fB>P z?T~zkzQX-o%^x=qr%m|7=E7f8- zpsKpJM(isv#O9{OU3Y(Iu_B9}H|UbK~ScQRumJ02V6g-P3ym34^UdTZQ~A4`xm`PkEr^XiHOEziyz> zS;sKma~zBG%$jlQT^3-=QM-ds%j!A%;g`kzg~PEp8Y9od+%51#(k@(KDO8<5=vUaQ zbWXnRJ#~e#DQglv?h0z4$ngl$soq@gu&8{diodcV{@{9E-uAaDKXdi&c6qw28)D>} z^abKzv|DvaFgoYP`IpC3H(v2rx|JT1>|uN=a;T3|77wr@j4l67;nS;h-pd`zQ&|GS zf=mhQyIW=r)6X+Z#?|<%4VTWpt6EK4R28>NXp1i2rrtE{32O_RHI$YO;lnIi#1&Apt_CI2D46%6AFx4sjOHULim-HsSdUY_f;|LwrAeV(}V zK2#3U)MnO)4>wF1e_X;Cgt9j%as)I4Z8F{p>o-z6snygu^#@COqPq&xH!*Mk8uYK}L zpYi7fhKMBfe0zSPoSN6+qaAEeF+#`@q;crUnvcKFUAdy-j5zvmw#`3_|NcCs`>&GI zlKu7BI++j2Y$Iu`X8BfMlbS+BNBvMQ-&Q00h2TuIh}kppslE4l+pyEl51<6ZfObdQ zLWQD4C#i!bRLQZW8lK>;P)x9S5xq3rb$_lW+Yq!eewaRDfS9+*Q_WvkTv+qC3fpMBkURuTHov(|?XM8mIfmF!vy?e`3H=ci&W1dBsO zhA|t;ZQn4(;SbJtC!5fp0VhA|g|@dS9BK|~F)}I~d8r}dkm!u2uk;#jU9l;$(1OnH z1}Asix6NLjWufKve(NJ$Jm-Q(Q|LGh?{#()2iqU0v&sD2>e{7}%TWKsT#&O5wG@rO z9x%h+yQM|Xv(P@@aX;ngF3+d+Q#Eso#ClcDVfO}7VHXto4%0?9Wp_diHkBe+B8lC5Uh!tVR$HO>OhY3(aey5;%Wj%Ni^#-+Q z4a6rXy|j=lc%t(p8*y6#eYJOkMmD4zg7qykigtNO(YeRrJ7a24k%;mrKXA<%oxox` zqE$hK`urArt&3p`8TmSt$jzSclQ_w}6NBX$v_E9gv;Ca)^}-S4&Tk@jHjm5 zg!$6`2o1p`%M3XP2h~%Lc9PE(3Yq?pC?qqT7qV^#7{AQ|wjActv5_RX1-<=pHjd_Ado;+-Z90E=CvnEugHsW0W`I%QFUb z8yyupZ~TDaO3&t;(i2F*e=K&8!-{R-=G&?=6BuhqGiOJ#D$Qr(^Jog;0pX&ZBGynl z@=*@%w%Icu$c%SozFCzc)W<(sNw+MGJ5`>HhS3Lb*6hiT-95t{xPagC9&-iR!R)A` zkk=#MsDDlAnQqZ(p+Ylm23_qZs>+G|EHc~05LOcuPQ_}O@p4;W<7&;r98?C%5|q9h zv5ZoACOGNj1C0s-mDznUMT@2kJ%_d0hPSXyhHUvuj?cZYP;t$8;Xi*7y>;@b<#rNP z!@*ttXxyKpDGPx{7Sw9aT$T~O>%2PPEcF3`M0_`52WW5R@Q=9pme6;Qh_!w_vm0UR zBfY}o4-h<=V;h#3HuYe*Xyi0JsW*0LO&-hRpFzo$Gj2ww8N&$8A-edTyNeQN zqMXm6g-tTsWKqKXF|ib^`rkvDFC)R@t=`TYFt%@X*!Z`6XenZbMZH!pdly_b3~+u! zJ!}Ll5sc5qg<74JZzU6`d!%iJz*e3CJLOttF;cM2 zblyS(IqFZG#xGwlfTVJ=-49JbD&wybARHv=c3bn2V-h?2y++#^_Hz|+?QaTlW@sn4 zevc}jQ^JOY`;`pHUqlvclNC_r1pR(cWH-)PWv1Z_rFNyc8YVxvE_q&rJ7vf_Dw5NJ zx!4NZ<@Av3?iPi_7_)v`!hCGL`ib?xq$-4dzu>e}&8Kxr`$xmTx0%631MQ8{{?-Lf z84_O!S9P(gz^xbTdPO?39+Rw%{KcJ?sOs^~2aVf`+CJ%;F{VOLeRRl1jTm9Q6tp|- zX7K)Tj8?fp4yB2PDr+>oP4Lc_xk(#KHF{(_T}e-6K2%)bh*HVzuz$6a?WdNQWVTJH1>(khgj_1eFL z#L&KIsvQq7STUC~#CzIzE}Nj@3fwQLc;gcKkpJ2z;B=c)Y=o~Z@BKFDE|OU zAM?J_fK6Lf0|DcKhqK13_<0rB&l_?7SSHu=s@xe1nykTSHK{0F-DToHrxU=T+t)#{ zl=1!kp1eppbVstof#R;nRAvnR$#mt4H7WwthkKGyITku{dd9no3<&P1x|w?NVy;e zdik@d3&sjfClsD&XFCIPHNV(G$Fpm*B0K^jf4cvAK^f%ySfW9s+WT`swZiQ(*K{mTdp+#o=Tp}JnvgkHJ zi63nnDv=cY@3xnN2nrN)KWa?|9-$u{x)xWVvF*sURr^M4(=C%9}2eOCvCaW%;5yMeOcR{ouU@5@_dgdfIlnM4Ok^wN8eGFvtrcp-RW%u(mj^ zsw|k{R&4Ja#A?bPr0%p(3%h{8!MSp~MeJ+WEfZ^dg1+=Cf1gjS)Ti*gy6|HfLIC!2R5$Sk3N z#sMeT`!#`30Gsj9KaYY#s?aSUKIS3^@|pO)2>FIAdSwFsMGN`QmHN3+9Rx9F;t5s~ z`{kx)$YIv+Md$*@$4W0)?(+HjddA-__QmW%z5J`6S${=%Okjpdug$f+8SY3B4;KAWf;# zd+)tB5m7)uI)o-wYUq&;0s_(r9g=|503r0yb3WJF|K2XwI+y1;=i=NYPs;4?oO8Tm zjBg1|!khmFwnG1lxV;L{k4ahq0TOs?}r3OJC@2y871@(x@TVo1Pyt3A)A)`8Vr0AqaVXY{~-BQvL}5t zgnqhesv+PKdG{;eM1A?bQd>YKz=AjNQ_$19UZJQ55z>zQ-=Xv8t80+;+ zaPROX=jhysY=IIDe{%k_IM4lG z-UHAUnp8@=yMG@cpwaHAQ3ceGu;ql~LY>Fpx-BIF?E#j1&X$Dxd%0Q6=i;c<=YRIn zfPTk*1wUK5DIB}2TRgay2 z08{JYZ3-b%iYULV_dgDaY*X8D~T$g_ne5g}6U~JIpVe$Ot zjdO7umn_KBP00cK=WRR#O=J2Ge(Obje7YC%S(44^;y!B6sX6b{@>#}?Z{)tFpbLJW zD-2m00?X|#eOKA$;ZaH|q{Sl}MTQL%5Voqzv<>RZymV7OA$Y>rJ+PREapCYq zDA|v!mLILw0&>YVU-Kyu@_$IdZsCz}nD@jQs-zE}I#)gp=U9IJ9A(BH$Ma3RC=;Yxa3R6&tvj?y>U$Qd>t@_erpQo07QagDAEXwO7))j*RnfGRb6HBGXRGA+B5{Uki z0y0TUhhC+#&j#l8SHjB^Ed9#RdmTIrp7!tmzEWbp6Y0wxv#DlQ5xmUX?OWcxCVvC( zd@fHlE84Q+{qTL`glDt;rR>|_RCDpy3$f?Jo0iu9)$)s zMCICeKDpZeCUJ}{AmqP%ohm_IJ$az(Ua&OZ<>6aWw8J9z|K8~T)Ng_C@>E#-r#t^8 z3jbH{0H{#SuL1bw)iwGX|N5yjKH=uq-WUI>zXM7If+Ddy`EUL3*V2?024IH;^M8n7 z{xhe5DzOlVhX0$wSDGmu2-nVhV}Dng{AbC=UInTYhnFGqk0AYLv{DHUHV~%$cSW2$ zF;J!6;K959Bg+3W*q7O)+j!Ei9{d&>{(5^q6sVH9^lI$Ce){HTR8o2C8{LE2{y$y- zzit8lm6XOk2CB!G?I!=PpMC{48B#Wx|55mUonO67K=uCrAqF<2|5u0jUkC30o5M=m z!JYkc%>!x~v+n5LGyyyF^OJ)rC#0l^|4{|p0RZh;oDfZF1xmE+MlDU|gXtpdzmbMyCpRLhZWa7RU??TKDL z7j`W0?bx_en6o<2Ngc2i5o8Ex{V$cSQhYgHg+CW1E5}M-YkRKr)i?K=F*ruYUTDt!jl$Z2`mq7X09F|Pf}ga z(1@lN88vTnj3?1|Le}@u0n#ez=d2HUVz{o+)TW3g^I(R zQL{uVQIWSR|1xMu`@;@$Jmt{2!QNwFvn+qi=j_gOXA!=FWW~ zXC*O4&~fgTt>*ggQ$qd#-+YRwxijnvbu9a0CbV|uDJILn&m09fKmo+u<@xpw*ZsBD zUF+6QyB7(MOh=hQ>sCj@?5vpmSwtXWXQswUWexId(`|K4G?$t=_~<-Qn6;ytrva= znzc(+bTkUHe!ep&`K?^Z2e*Y3r$<=`CSYuIOS26Dg=pR~_ue0BOhGNTHr|v>0SSre zCJ*pUu2`c88gBd3y(@;;wssj-Ywt0w)xku?D)aYrI_gk(wO5u#k%r_C=^y`*G`arh zQ~*?4@};W{cGx>k^M8=pHdJ|>l22D#Wod93O+@Xc_G*S_+~HqswMF~(rZP7sJTTM% z&bD-(2GIGVoy97fJ;7A!{?$q*w?m+GTafhDi9B_kApz4#`jI;@cOqe zmOs-{9khjndcO3mF4uDc1d0W{D5=Bd1jztj5~~yJx`$eYabRyUn^zQ^5x7LS`=Hdi zWOyhL(#=8+a2`OG^f5>Fio(d(Fw-Wfx#NZrJrBl)i@~6rwC;7kS3^SORo*+MdxoNS zSrn75-Y$h*H?!(bF~Lse$VSF8cNlt%YgW0fs+_)Ra4!OuJ6pL-HT~bm;g1jv7GQJw znp{1&oE$-1tba`=l(ikNQO9lN`)6%7Fl+VgrsgDGJ>B&*hf_({R`Y?lt`aX0-3ye| z63g+)Y?$Eu%DUT1KaYc+K=PJ>sC@KhAt>r(pOJ%c=mQ;-Stm$U9w1&WwIc`|qyQcp zv3B_*yl-+1Nan7{u)&*phqH`)oB1xg=Q;UX&YrrtCq$zqr*I%ul!kY3lDEK(w!mgK z*If*;81?qYNDAqM!z{+rJnB>~QLl|13i|2OJgp(^52EYzxA+R?LpM(gO0TWMzz%71 zFN}OW8uh*>a4ZxzqewtiYounZJvAssU%?sdG{AsTsD4J4r`|Z*>@lJE*vUO)U()wmuO4Nu~ELn}sa)fNwER$gc>NX5P ziMDeB)6&Zjn9~?}XI`bL+<$~P#0UOka8IWZVtLoK=?;IY9GvZo|y43!H5`!0;hg)#Y z8G0>)@|F`@DEO8-gY^@+m~q$nHazwh^X}V|Wu>W}GZ=V(Drh-I>Y}2(we&inC;adi zIpkk)KzNeq@>JI%Re#cfdz%CDf&fpi1-DSR+Y>)TJcgnqL!itOQP{16mooTO9X0Vd zK`(498Mv-S zKTgv*m|5~NPbOLaz#sBdn}2PhawC@yU`S~%E32kCSO)Fpa~YnLh)g~bTpZN1wmc!f z#R13#e?Y3*i!J(sXadfv{UG9pFOq|o70#a;P0F20ScOJk1Z*4x0@91Kx?NtFxNqrA z!BGA;9Dsm>mEwT*=Cw`IS_GR>fBaqEs6FZoe#aAEcfXp}s34RMYOE}#cWZWmdop#X zE>-g``GV^oQ<;o7#komyFlF(4+uXRWD|?> zORqd17>-SeE_*oUlMo8PlVVXckDN@@_kK7lI-b+K!*q@MF#ERs5kE` zZ2-}v6|fKX-$gv7mGI9_c<&e?4x|$Ke9xAG(f53E2A&HT)2_SRZ>X=_k|QuWX|R2p zZP^6(RAf#CfYz%({s2hNavZ)mIjL>M)YxqAAaUCuAnMt7^DyZ3{gl&9kJWqO?y|a7 zRPBQ>8GxKQ!jjSU*lt|6{*$bqTlr}|pn4c>Ty17uEN}8kA(G-ExpjUBXc+Q`f=vu7 z%58Et%mEDo-)Y9=Qj3a16Z%_|;JDz%M1-&Qc-@Bew@l6tbQ9Gc4P}BuA4=?JYI@6o z6IkjOt=e|z^F)rWdPCG`K3epuKN*z#Nj)g?679mW}4Ik{_K=?!kn5l5wESo6p=#4&3QT?XDDV`VL*MBCAfD! zML0B92j4Le*HiA=#844kYTO>GP6-YpVye1%yTI23JQv9h5m% z3XP+)eAchq1lYk|orb{dTvJ^$W}$J0ta@&5d$(XYkH>5v6xcKlXmfM@SW`g}O#&<% z0Z6KyCdUJ>0;F~J4MF>=_PN5-gWYJqkz-AZ7Lx6#_}4(W1l+Yl#u3j=qq&QZmG78r0AH9Xg%;$aK=P{1HLK1 zSiyi*PHjpwT1wFnrO6V_+y=mypk0Lkzv}{G`?Y{ip z5bK90q>Wu-hTFHPWZ?k9yf%ZGuj1lcyjkj=EIEF?K>%=50Z3yzB-FH2wI3yTs~Pr540$PgAE1ZWj!lx=zKIwnW47v z6Z1JCoinU@ilF71GfR9=GG%8)@beRJal=(%+#%qTaDz;RLE;2rJ+CI0>5=MiaZn!Z zxUvC(xq0F>nzdNw4>;Ny6Q`%FqpJ;)!_o5$$@=b@KhfoSM7o}c+rY){UKJGrW~4N~ zog{`i$ZVvQkkWSFpEye3v>lI)SjLY*)j{rLoPQ>8`!cC%gqaWFcR%0r&n2;08jp9v1pe`!Z=bT7SyT>Qx2$5|^?;!AGq+ls!Dh{ga-^}* zCFpMNi`xxhBm4^fCz-F8GUu#^9%;aVc~-Bn*ml{zxC&42PV~IT&-`0iv5V> z4}GbiavpEGWS{Mm0cE#|S+Ed!Il1?Rw);e$R?-74e&bX>wSInWPYd|T1Eeu{A0^ZH z1w9Dx-NiR?bEH*ftL4G)jSaAvw#saRu<_#?;=Lc4!w25L3MxOEPs_T>3#FBAf&XRZ zy`%L}P}FApp}V)z>kumEIO4;4l%H#BoBD9sz-F@8NBUYH5<9b8Fr7oraZPVz<-^73 zR`iOXci#Ezxiw!-;3H+upo;?`nETEbYetVn))or_CO|&EjO-r_C)Kd=OyOM40X6-k zNuGv^0nR8SrSw&gZ<8U8A`hjY9z z6I9*)93Epy;2C6}-2~dPXq>&$vI&hY(~~y?24kJ7uAY>aKck zh8UD=3XNzM%L&}PClX@Nn1df(ZexFcwu1b3Ota`Gh(-@F199i{M6PzA=>0$ zqtdU5sbZ1h-YXu8xpVe7=998;kim7Te%MMSo$-FMz{Z51GHb1rB$|S|D1r{vcG=PS zO>TE|YC#ojDPJ2=5k8DZQRjfPi9wwl!H9IKED!YK*7fgAs~(UR&sct*p@%i3>;Zl^ zifsj)fkw*`kn0j_q1qpnKyIxUgJQb?HIm#LvW_n#{06FaV<3S`M@`TP!S(ZiC;u$? z!%{^9FL(ZP*@;r&xPYpE^u+3G=@Ykw`|*49weJ_1;g}tj1-MRD+GExUpP^bCn=n@Z zTeb-@Nn@wcubH;m7^LX!P!(T1(18M1R@8`9MNNHUIKLgNi@#9WwKRJAju!8I`vqW7~922&vrm@;U?G=c}Gj@VpwWPRlMYpw_zTl=zHn<9%yZtqR&HS+(VC z;_xoie>rQkz>T6lUqPnVvT925WM6UQk8ElNi@I3}bo2Rr%V(g5xj@DZDGgslP%;CH z*FeJuzMTP-Am}C5sN|b0;*Z-i_Fp;}xXpLPKxGR9WsNClQH8|k%gS}L#Ho@SX2+7E z+Zqk#EK;>rj>X^=3T%34WDxl4+}8KwHBdMkl2H8NBwbr z0SCm=+lo^WFeyh8FB>udC*{{O0#W|>nrL_W;~VYULS*z#h#L()5XX5&CNsoq+1(nF zx+E?`4N=52-=JboJX=fxuajXGfJa`O9glu-Ig-Rb2-v`{{p49XU5pGoEjt(2EZ3iI zVk7`NhR?he{RdVU4}kHP4VB4w_P2bp)-D(7wxmN}P=hTCYi%{4L${5LkcXs`R&6i9 zaKX*|ojL?AYs<_Sc}ES<@Irh)(^@~S?&J?u4CUe({!wOZ$-}4YI9t;+ZL~aK7vuv& zoJs@Cv>)U3vL!e{ML7pX8=~B#Me&9Ec4;sh7s@R zaVFMFP$x`kmbGg`$i7M^tbNw@*(a&Um&p{~P2u#y{;w?4+ER#90qXzs{J_&oL*6!P z>W`ki_)uwz&q+7d`1~#TCU;C49*F;4o^t%b0x#XebUw#74E^m!S^9epPOUnK)JJ;c zjFKEdXSKRpYx~(=901fr6#xko$KHkvN~r^ybby->^(_c@u4*V0Ah>>7=L5TaAMpG6 z$!VV~)>NE;7&(Ez@fb8{*iJ-=L{vIX6>FBn&NhAzBc=K72}-N7gsw{h`KRAfs{(jw zozIRUCYyio)vxcu@ZMR;MUhRZfxTL`^`(mRclM2m9K&=4c$0b|;=H15Z(7G%_DxeI zZKC)~jf9xi@*c+ldW+tGjG3PPBC~wwPxu!DO~*gFuJoP8ch6kq)h-Gc0w{DEhPMyW z2h!aR9E+!o)w^4nJa}69pCzdeMDivGeG}*PHB`|2c>^eSsToLtUH3kht-dM{yx;3b zxjBFN!ov2Uzq1h_gm3P>K}!Ew2JDC!J@P#Uh)fVwaa^5(2(n6t*~dj*sz~FLeVgL} zz5Cd_kdfcr%?-+mb{*fG+*_)F7H_QECa4@7OGA+FbHh_NzrhARz+HvL+fYj`LzkMU z={}FO8Qqt?l=<1HwZ%Y_z*8}))Bet#*}(l*QfJ$trLQW#wg}zkw`KyyCB}g=>9-U$ zQ!EEjj_9?>27X)<&h#&e#DB;w1M_{^6mFn9!_=7hPs{}~{H=glqKY~!Xx8hDdA48X zlJ5hZ(swqa$;^X&u)yiMlo5FfdSUyMPEH0-@Fib~Wea&SejRHDU8F6xY(N0JHO;yF z<55+~VMoEV9b13f6+jXsFs0R3>i@n(LTGPGY*ne-FS` zif#9AQ3-Va1^lxG#Hf$#S(-*ldUZ)bJlcl{CID=GQT}U@Wg?1MQudO`@aE??)MVyA z-w-_7A^X;8g=^oigGGhA$_*NyCPNGwCj`YNEGs4r zyfUn_WBz$^qXH7AT1GzQw?jY=vO{HWa%w5n*lYr8s!8sPj4kCNknMO!d}>X6Tjjm) z-$PhP%{>4wEo&OpX9KL5nY7FG)gXov^=G?%7VriS!AO7vS2y=fojLxsA=fGvCH@|A|5Lp572sBy{MzuU%0C%RKiiD&*>ve& z+b;k3=ad+ZY%luPhDf(XV}H#o|F2f;^1#XZiti&~s^4eIpXC69c<0xy{BLvrpBluT z%a6_RyH(R;gFG?G&6mh~0MBQ@7!tR16vTFKHl_BTtq;+-7yC$^xHpX@eyRVX=I+XO z7NWh?q$paxytooY+T_P=JkgUsnK@N9vntbd0Z#OGRk!}zj{i;=fR>Tg=Kg~5QjG6T zT#oGS^WL1W`&NhcMU3-Y`hZEqt61$adrEiMRNti_CXkt5quP@Gm&C^73UktzP&T z)e=sgW}U9qN{I)Qt7C`XC9OQQ6!usj69gT62RdQXe~_M?ch`4&8+aDN!81zsXwmK; zvU`fXreXK}Cb$sCpnq22G82<4;I%tl<56foT{r6r7$GVbYdx^2LRS@tW&-pnDgm1O zfIQUV7oYTf2^HBP@O$o)IVxP!1d$ZQt06|`y`jcLcRGDc&{mcz6<(vnDMfEY>%{a0 zYM)H2$le9~a`lqQzi~{MUPyzsE)1bvtkO18UaQ(XoX4)Km-SKfL7sK^o^cvAFunk#O!iy|2QHz)tkD``UfnXr^^YWxK`=W{kuvS!=~N2J#(_EK#?uL5LH zw>O%eTLdVk{Kl{ObGp8JDs9HWV_jHD{rn7wM6%#AK>{V~ERUJ1NyY|eNO}8^e=Y}M zybop8KT8q)%*GN3tf#0+;DTdrwr^cAW5w`q3leaq<@r|C+o{ig_~gBQaBxQDOpQY! z6>P+~wprl+<_C0UBd;QXfA!MESpZGU#5%N6TVSu-A8{j02lA6`8k z^>1^;gb*A$#PX_+fy1$czi-RHkEBOQzNGXa&R;%C$(washpVSiOfiVNkF_-H8zlQ8 zm-K1>@uT}$!IAa!;`CIZ3N0WJujup@0#CV^b1S2=&1q|LzpW+*kzc(x{#w94ovQo0 zx{L=C_f!V=lz;{;;$%I%Q`pVEE%`GD3;v^&pJ%B*f1$Q4%eo_*e;jOZ!>>w=@CNtx zV3f${-cEZ}K)2L;1dZ4~!c~ZyYH%>T9-HR!t?1P}km}v-fA^>=;FO15?Tax-iy9BN z0O{sTvs{bbmsf(0h_p1`c)Pzl5)Y>QftDV+<*+Gcc-)QLJ@6v62(AAl;jx~eA%zpK zKgzLHt5x&q^(Ge>3h+I;mAW^bysMEFzEtq`z_ism=^OjQ<1*|rkk1|63mL@QxTv@N z;NNKcaj7p^)zNO^fw9hKw!IbJxK=I5$3EVYyEGX_0%IISbz8_FT)ZdtFb*)rypZz4 z;@y!QA$C3sBa{z&aSsolr1^5ryQ8A);GEa@sIPJ67`I0^W6P%=SJFtosKiEmW|F_o zey^EO{xb>t{TuUOuQ&t2UV|(23-T9x;@ty-gIj|ao|MZL5lts&9iv>2OtRn`DOjn* z($fBWtB{} zC8UK@NMGVr^7-t5|aB>u(t_}Bw zl1&IWr749*WZ-s+_Cb=fZR`)U-_VwtD5H!@jgOX6*_WB| z3xOiQ)~c?XFAX9jQi%^7o3W34+{WU&CQACnAoRW_^@u5tu?|~ji zYDEN2meaDo-*NF9OQKTft+KEyK$|-|m%MpCK5ZP+_I)AyJLp*n{lmz{PC%{6T@p8; zkTLUU_ghalDd9%?VzD-a6+7ozo9=y983zV)nn@mwO)u~Hp&M?didbcG=~b%LIU%>I zqp2-dmwMg;xlfQCSdWcuRVo$a^r;kC(F@og^0$5Nw(t91*Oj`W2R&G9PU4Bnw(ym7 z8Z~^QyQK^i`TeV`A}Kuv4)IwPW^ml#JZ{W{md|h-Rq9&yXR;|AyAk?g{q|TghAR6W zVo|&(-fvxs+C5D{)Cw~b?m^=b+(zIgM2&z>j`EOVM&GcJ?QC&aXKL4jKtZ>$52-KC zP8T-l{d`*oT@Y5SrkZhUu{K6UdDxQeM}r{n0oH#yuC8`_3x9S{?8nXZbdR9^6lPKy zUbl$n{xLNug@#U+v+;6I8q*yB`_9al2O!lqRMDbVph_UN43ViMyZIFhx6Mav z2gzoB6bbP|bX@3YafM9(1q3vj|p5QvBdF9p|o^dq8P@TbSqUon&Q$@*5W$L-Dq)w6VxGp(jp-B*sadrS;eKgJK~OF1)TwcTzJvqhQ#Rn26sXDk54P2z+@>r$En1T@-VchY6+mH z?mCUc=&*h<6wWV2KEO)sM%75IoBQDicp633Z@t&t+l zmjdpb_|2=sz(Yo(BK*89yO!z@$i5hc@APcnN;jy`dkDSZP4=9k;fxsNzFBnW6@G!R z@q5bav;Cfi1qAAwyju`%V5N64J4SoT^&;(?z%Skt{#aaBfQybCM$@DqJaG;~ z+y`(wcu>;Pd9&KkNKbm?O?u!k_kBU+OvJAo-K#ZDXnLU?(K|T{bztFKx!?c=9V^vL zW(X-~@)?xh3WRd$R_U8=j36c+-OC2x25w%M6)XJ?zr-{l~d2O{rETYmCacF zxd3#kxaMJrtf8}~6l(PN#ck_b$KSKcf#0W9 z8du^}uNru2_LjugfOkX4F#)EEX33zySkveZ6KR#V?A*y}Us^Q`?31k4fPr=@WM+En z&rfF-FAS1)+LZjpBDGTVq!&KP4e7uSbt^jpaz7fSyfm!i^-~pryUr z?CTk?uCi$}TZEQErz<)PDFsc=#qVxzHX#5!_ZgM2e~Av+5f`OK;6`(dx!)GK$XNxs z>qT(j4djEtG-~YNRzI9w2!8PHY`;%KtfJ24^>I(vNZY^{zZ67E(6=PQeXP2;?4k z_B3^IDhBYCHFnVFR$h|0#O_V_B(#v73d*9GTvjDC`L*AGMl1K*9psu6j_0F^(3}GS zBeR^iF-`twwfqQ)EaKR~`X##4WZcm+GV{+dvP_86ZPMHBeoI^6O{1v;X$D=d_TdeS zi?3a~uSt$1{D+he{hNK=DbK}s&Nv0Ly?&5%Q|q{Y)#6%+u~L>^fw|<$(%?Ifo?6Pz z>23LFAn354uF|@DejK;T@U=LeaFfK1g4JfPiezqo+ar1{6t5I)+9Lq%b*8L|EYs7z z0%_At+7R|&3%n?^#(cg5ztA@35h1&rA#p@^oBnKKUz7`d7O1BV+br5hWwN&!br3RS zZ%souR^tqrn2iFgVltI&%GJL_6Nkx(yB4mcN2a;=Y!8<>#v==CDjdJi+r5=;x%jcT zb(-INqcBZ9vK~B~5qZd`k}r{VbO@Mn9Vb?DZQy3@L_33IZd3lh$&@-@kMJ z-sZ@GeD!me0Es;56M*(>aR%n?`N;b4)r>nQ)bdp)DkCb`KE4;u?b@S&LW^H?HcFRxSy$c0=K}Tf`F)op{b=wj@aYnJcyHc(?dxEER)5e;%OEPgw65nM9dxAGaMUkb7hX6q!ZcqCMes_gW^$;ewVgFX z4HqTGR&EPeGjVD|5?fc!&1k0C)k5(9Hrwn%hKH}&SgmA&K|w|K0y28qkGfvQ~lbKb3)Xp;|_b`&(#&yhP$xPKu1(MHT}nZ4OU$+MZzy=A zpkWk93%@Hisvsjb9h;b@OIx?Lh}HLn4UT>k*uAyYTH!vM47{`(vjMrK&-AKrRdRNJ zR4gyx_O!`sy+~?avYzG4z#UD$%x+9$y>x{NX(Kp$rnZ2AE`j!7MhveWfhiD5D%c}V z*P8biP4`$*$M&jb#jtx+sHkR_MA#NH>U9Su?}rlyW{qgd4IE_pp~~*bq$!F9s$%V8 z_}PF0hduS(FH*Nq?qPc9=H-4NoWx!>!1|(``25mftLfMlHHW%ZonzOqb{6Xg z=Fct&F(7NPS-185>I3WWfRe@T239!4cnSC%TI4CAQgP|A)c4>?FA7nE@nDC4qp z1Q~T7bHFkkdslKhj8!*%N+BGsU0tiw^RX&PjPJ%C@L-HM5OiXdJwNEk)*0^)+@SpK zo|=AEe;=?`VxS>u#bMx5uGzxIrb{A$c-OzT*Kk9w-jN|$cw zNE_@I@rnMAtDIrsax*{ZL3yrDFy!7I{*|&cp*=qi^y5Tp-@CTFycyoDl~zaj$k2#s zm*rv7B{9&;R`NaZlTJKB$hf;8E3u1S#8D@&)%pks&S#?hY>coiFP8^I~GH zR@0+TB=*vuFDxvm)M+5A2Z}$Z=|XpLnF;GpB{|N~Y}ZOlp4bgFgH%wCV(XnEGVCFskWk^o|z4#N*(W+BV6}^I4DW+MQshm*=6o4S9p7 zRzpA%u>YD6gXyGj%TbRPC8)izfW@-B_7NV z>4(u2LryOtaez&zNTlwXGqTaK;aLNH;Jem>o9r<~sgoNv9dAF#9#Uy%cQ9r7966#u z^EZ6Dxwc1Ogyb<|1&S`=gk3GNipjH{Yd>zBu=A7|KwG}R^AS*Y9fW|{#tDUY@`-sE zhvdO_hNlU2;n9#zP#>H^%7NN+idD`_q@P^V>re#QP_@qOCB=$7vKBM)1kURa8>>wf}DqwAzX53HsG1Cb+vaYvf&HOc`l=B;(euJZ-E2Rjq{Tz>NA zucq-!y9~kue(6-Ec_V4L%^DlTEBCIcgWIEdDrEyVk^y_@h6AugM?u-)d>uDk#1~5BF)zE*g3pVu<;to~biq>x}9J>?c)O7U58q+uf1GE*WDD7NFyK7-X|?yGG~L zV(%qKFVA`;4LFRxVCblCP{s?D9pd^WRZ10UVc??)>$M^$f7Mp*5lOw<=L8$LGI9|V zbRU9k=53#w&0*r=E0QtYzPs%NE2$6-=na}@`8vyZf&gzIQ7(Najmt+uoz@_D3~nxOe*#n?YFagwXF8* z&6}SGZoD~dxVOq6L>`Z9IhI!8bAzDyL%K$sJ{#row})jJapVRWAGGv5biWXB{iOT)uI)NR>kPJUb!}a>GPo*W_Gcn?<^3mina2yH$ zKleC3`my_xnB;FYw6X-GdjqjQJbG_Oq${(bC&zhfX@-X^C54Po<6hrpO~M)W$TCDp zc6?;s6%L)vvb(klNbj67g5VBpxo)8WdmZbRPP$H(`J4W5;_$Nv4L?>3Zs*?pP^=>$qKZYtlqShvgDC#CbFEkkZ7X9eyw>%>}y z4fx@PiGaU9N`>|ec!>-1rigofWphjw%Eh$@!Ov8{rYmk^vV2zx6B3$&GoY7D7ewuU z&1Fl^6S=$8A97Q6UOkSF+IKiq&$p$rL5hwj&35#AgwRlI(vnOOO~5eIKX1u8VLSYd zxKGz}mzIUTMGTrO-$&yWJkI`!_}KLi4*;l44~Zp_f>VS2wvXT~?E38DvKR=HIB2t) zCtxYOwk>w2LZY#}`Rv|E{Q&;Q8K@0#rnZ=ga@7f|S-k7h9&(-8C_dF~r5^gwVTHba zd-wL{d1F{+zlXNIDb%rL)kVk9`r~WlMao^kru8kV>=gh;#L)&>{hGtT!BFCx1Rot615+5t zaR4Ya6`bTYSXTy~UC3>SRa#)GF83Ahui=d?df6dd?{Ty&Xble<*5upSa~i;;nuD^s zSWo>{_xE^{ry6`BWp1`OA`ud3sXzP=4=4_?NyKYSs3IDp_)A&-FL@d-RLs&)a28tH z)EqY@Ei#hpt8mU2A!L7Bq=on&ubUJI!MEzS_3rb>zN?@+-A_vwe~902LUXHIEn~U8 z(J3J|ws^N3RkS+=;_a*EZDuFg*%7QN3o7#PElhVcRrT5ZG#g0>{|W9tlQ;;zTE8M# zzu6$hGDv9Mkj6B057gtz_xv>3Hp>p;^ z4vw$oJ$@A=TI7tLl?<7iW0RQeiih=sgtCmjws{gOXcxxsA1yP#CMwvOr=g0V16=&s z9mzwYWl(IzjXiBL2J_cK>dVROuzaH|LzW4w<|kFimf(YfomJ%V47Z>?s~Yo?$H#|-H z;4XTm#}riRH!*iS+CzBt0B)36!(;3~>)A`Rvid+1;qCQ$&bwQNXs~2Z$N;3l)3QE+ zzufStLmwH4QV`kuYI2C3AOxA?hRGc)CZi2@!&4hy`mLhQ=lo_p)CxV@!#E2rL^vT< zCA3<%ZSX(c|8)jE2nsnOu8d2}FvIbTJ}5Ul77FIv`Y3JOV1gdW=R`i9*+929IwlCo zN%9!PY^Fmyt90(}Wyvbu?|&-6AlvXPz(( zX%2JmnG4VI=MLgM($2}fXE8NQx-9AD&lI1Dg}Lx5hS!zZUvtPLnL}MkIpshdNYd@a z>FDe^7nwF`l?%q3rPV{qAGQ6#z{ew_Mz;Mcl!CR621; z(|}OhzII^ip<#WfK4GYvL1D)$=PB7wh`xw*n4Xq|=%szVwDqN3|CFg!6T1|6o@vkP zwFYWjZs_YbB_8p7e3{I0=piPVoV(@d43BSNfM&C?IXT&dVp3$$&}YkJq>IPBG8Uo? znNV#xL6ol)A7(`{aG%iQr#gP|{hU#v61tuhbe`+CKI?Rg6-w6nwaLun=VKUs+%_-m zf>FIQ$HRxL(uWDh4D+XOH`R(useX?2z!T4Lz@BVMvDcV~>Fw@=3mI`!4h-qL*WgI3 zYQqtYxmkjENM`$odIVqB3))397PkgIs`mV`2T~VSz6faGrt@783G=G_@GB)?`N43p zP19PsXC8rEo0#=97>jtk;EJgeg0xG)C!AJuC61o#o4vix3U(5*NXdRz@VV!FcEfaZ ztZ7~yn@jF8t7S-QK;}8n;8AC#4|~H}m;BrYQr5v2-ybWO0W0&2(uw`LuQbZk-J;ZR z?WygI*6gXA-`M6xo}`T)49$^dQJ-|!uWRAk=rEUV>33JqaV%S-PLzwUi;#(~lffLf z{E2^ZY$3#MB0r_A>)HDfqang=x`dU7tsXV2lSIkn(2#VXpY}c4|J=Aqp6ZnAjx5!y#n-ft3$xH}NpQVO+dY zo>w5Ko2cQ8hSIskdP=#j8L$$wo|!t+FjR%a}+-j-tt(T@-xaU<`WJP=H)xo*imtJh~Fl2dQ$FpM@1(XX)T z4aUBj%sm6q#abt&RLIdxn0V09&&>YuQirF!8M&`+s1QINQKq-jXTw{(qD(nZXdiA% z(NlF80c*2h$_gbP8mE$wYZWjA@~Qm+qEXJOTTuI)dim;;6J80G@W386}Gc$8u z2JMxG*7yr9yO?R}fkOwy8o7lmOP-#^M#jaxCvGb^v%Z)MBety5j3*3-y54Ig)kCM} zN46REilzHb=D)Zy3cM_Gs>YQ@{E;6U8(%Rx% zV$SbM1xaV-EZ+Vw^k0~Ed26%d=R@%T&#~K{)`vRBC9IeO z1rfZzaUp%K*LfJ2CJqj< zE>7iQy(sG?f33F03%r#IGxA2h=9y39PwsmGqWMfP=lBzp^D97L-77eY#AzuGS_Js{*?$x$$Sm^VtIadKJn)d$-+6MRpdS=T%* z_x#lda6K7rfGv|v^khIvh2Qow>Ftld<|6+H3E<6sM59a>dN)x;I=ozK{U+6QONv1tDbLG~K95oY{xA04Ix5O8d>a*16cJGoP(n~jT3SHDqDyL| zk#3~BL_|Tl8=!-e(t?BH)Nmvs|c79f4te!W&o&xPd(1|ik~(>xhCb7#e3&-!z`(`!V&M0+*9 z;2QYY^1f#8vzY30Ai9k-jCVDr$Oek}10cC9pP%BMenc;k*mDUt0~*ngC&n5O$-Ctc zm4GYYGGk>PX4CuEaH37n^iJ9@P^cBm<1stSdyJmZW!)?l<)1S>t7%Q<@ zR*ylpK=${gDgN_Ut>B8xzsWS{4N*lR?GKz$Xznt3H;6aS)7jn3-O0^)_Z0?Uz~5ty zYN=1c&1xN8gsL_+=i2DT9m05_2EOHZeVaIA)y|n=9we;0madybbUO26hdq&I2HPWn zof&;W@54q)KLT-1d-`TD#59#OnC71-n_sURDW>CUnXhlrR`}_9d743URNv*@IeYK- z{tt~PumpCWuTDsd4)91w!l_yqvQg!CG$>Asz(ob`iUaoRlqBkt5Fc6{5`8wEl^}{D z>rE928M){W?dC0=!>pB~r59*$PcP1Thz*N*3eMkkqFZ$A-Q5aMSs7gX?pxiA9*j^g z3O#%Q^-i1x*dq5W^&fBS+#)_|P-w=e(i-WTs5RHk?`k93l8XJ%%a ziwxTIXImq~qN3@2jUe1l?3 z{O%2@-#q=AZM2mL#yR(7tJrKXUw;#s@uA&lN|f^V_6?$(biQ#aI=xgR%gts^4s&WpLtT8Ey+= zi=G@Yh+|bnyZSU;tLMpVh4S0W{9=zkGx|0+LncD$lRZHnHC|-7mWyN%-^QzQc#`2x z1K(~5%U6%FFj^`7o-_#_c`$eM`}e0NV-8^FHwf8cPPhErX0;f=606N z$XIsQ{IT7I;`0vpLRMaHl)CVS+;=yv942;j7#q4vP^k zS60M5V!;9I=)hW`APe_4i| zQV}hE6K(C;GXkD2E*WkIvjntdF70pR{PS$L(|u_2k015q%SiD72uZ^L3$oMQy0b?M zC6Zn4wr|7xBC}rIb$3OR=EeMy766gZx*u$X0%n2Hanfq_8kbV&=LihO872KiAgw=s zSF7MvrQ>M4u#Vq%SrBMmhE0jC3QGAoUTK zxCGE4Tb7mg*P8x)Oy!5qwVH`D3oaj4tJ;zrGTF|tY!yRheB^5HUC(9G;1i@)JIb+;$GvvYI)TD*J)K8mHyj;dX;FGmVqt#fo!B|}Th8zfa> zZobQF#Z>hf9qMCc2cN@ZV`o>oMxu~f-zEfCEL_L%_FANqg~oA~rlzNlz>waCA?xYJ z7C?g4!Fkv<9~;CscT+$~Ww$GVuh)p;(%)VD-_E7C`yG)l`8WI7357N)AwJsELVNA= zLROO&yF#pDTMx!P=-jq^tCo&}Ywg;!on|e%Q-*$+Sw=nyLa+d;+sKo|R~h7-LT|HPSFL%zZPeDXt9Rvqufxf_ z_Q}GEBS(!a|WsN!*ivV@sMB0zR!|G3p!mW`*N!G6fP|FY*sOX~>54K)BcNB}UL z?%S^nWca0bTgg^KO*oH>YIe4q@Gk|qZEMEv4fAdsE|kyA4mQOlxh;(=S`CLW4#HjP z4*8a>%9-V3H%z;F(xoiI<9V&LmJ{q@Kc4QcjHinQ-&O!*bamybphL~1FDq2jz@h)M zvd2>4**_e&zmD0(G!O$k1|4Tgk+vHo@0vT0<>)*%lv7FXuDTG%KR8+Tc(xOr{&Ret zvZcLlPGUD-tW}d&6M)K=e%{W1qEG2+SW=}q(jJ3L**2BlHjCX`w)9q$Igqrm(hoFS z6K=FO^QO6vs_eKJ?FdSI`}TSCXvB8S7Xr6g!eLa@LP|(V7Rktpi{C3m3{gZ z*?-(}yr`1g!x--W(bAX0uw-{QE0Z!ps0KDl?WbiI^lL_y9lkML0t%lA# z-u3b5v9d(_kro?`WZUJ34laVoXM0~qsmC`5gzu>&Oy&keMbwBTyWSPPRb%K}LeI(B zTb5FxWh+bQdAx5_b@Sa$2@|Ucn}7@~<~#;(`yK(7%h0}jP#y##yBN#sIb1}Pqu*Mh zYMF;d+9evESCE;yFkT5~_92Yf%V@YTe6l zmL)TnbY~wO4_QTB>r&G@&lZ&!6dpe^Id_+O^L1F~kD(rNMFk2?8I^F`m9WKP%F|*# zq>g^cb2&=ccjvde$Tnd0Tem(c*UvvqL_V|jXh*!A+6mH1!jh--RluVQXn`r{j zKAUr1r7wFks?X$ca`EWU&pkb@1H35Fm)0rS%k7m7UF_d{n-k<=n-JS(nUUvSp!%}( zlaXwNuB`Q!RsVMv9akrVg^jA5={e0hg=HP;Je!*4x8G@$nks)8nM%k3Di%wCo*qL$ zd5wf>oTFeqtcJ|9|4`AqWIHeRc+19VFbcQBUcDk;M?N2+~Bb*wK~n93$7u58D`i) z8q3FNb+H)gfKUEFzEcQgu3cq59KoPSLreP%I6lw1Brxo2s}yLTWJqHpcY+~RGi#Pu z@9(!pe5?P4=35KX5`;;Vnx^gyOp|ekY+l$3pHHrU4r!U9>Lo~$7P>$A)s)~Y118Jo z8HUDPux!Q5*2DeHXRecpufE}4zjp0;hOPW945CQp&6wm<<1ANr<5&Iwbd*~hIS7$X z;&un)zM(8_@(?nfo4miq*kSMlGl<-g|uvDieutay3889V)0pkwaYb>Q_T{+Gx2FW>T) zSM=+r|L5~b=r~wPo|hK1_Y7A!=cHw1N{Jel>Cc--k0AwK!#cSB8s+%_-%h>KC2wS9WmqYTSWge9ft1=?bBT@uZvo`;jfsM!5^>`f4CFhrGAio{V}Hf! zo0Et#4*|+RW0U0O5X~wFTC^V2w=CYzYW^8eZK!Y1_ANa}ChY+<5Py16+|B33`-?e~ zWyz;+PUYn#uHq0xlQ&9b7S5|4l$G_$q_?^dVL_rY>GWAC`?`^(k#@mYc)u zd`Gkl3`t|9)pp5>Bx@eO!g*7c_J~zuWqfCj>FF2CwgKtbI%D6&WLfQksQ365!Tf9& zo&T+)(82R-?2&29b^p?`vk{%Pj<0f_W@~a5>7<`^%c(|pOjFO|8F%s1zi^bViR-C+ zBK6w78dJwK!#TT4?1AaV($D5Ad(qi;aRbnkInNGs>%n70K!3i$MSr;XK<@Bc`LrY~; z=Ak$_wgdkxy!^guNA}D6LYml^fJQCxbns+GxnW_vFW5ElRhrbpG2G$Yj;shegWo;# zU2A9ML*)DdD5kWH7d{D7R6r62TU}LmPI~D0*MQG#(nQuy+|@&S9yCFgLg+HA0Acm?#Cr!qF^htq)qw+%@|c>%*!0EYPmkOlAU8S{VJ z<6q`^ITF~`nyG0W3<8C94x8zVB%b#+^SU{@wO7@axcS4d?aTyi8^G@x_Usn~fi_ox zY_-TellNXq|I(f4Ul}XF)ot9qjFwx0zGG+5DDqMvG-pRBwd&#So}dEAzT8FK!?5lu zC{2KVFNEsqeZeIhf+|&yWFE!qgi{7<&PVBbeq|g+E5aKC6$BPn*^XK-JEJ16U4JZB zW#huA@wYLwe|)!#LSO@Cx1>Gx_90}s0XdXRdfLvSq}(5hAws!N1Q7j4H~GwdL~e|h z-1@jJ%YR&9V@ec_i7n(J4;GyaQzre!z)xp= zgcS4)A|}828b*^m=_A{yH@^KL@FvQoO@Fm~RB6%xJ%-%VMI~AE6R(X$L#2x#h_OPS zr=p^J)gPmlJY;0ulZhBk`F7w{vwr;QkXz5loiK)8ca=}nh0ZNIE|RFnCSv;VL<+#` z<$E{~vAu*;2KGXIxj*4Z_DGkhnDCE%``dR(d3*`|7EN)Xl^t5N5AWs<I@TYulpnDmDOFe3;Qw!92tQdAoJ3TTz3L}Rx4IPoc13;IN^?G@fjv+OzjWXE{ zS{=iyeQaeM0;iKoK&H^?3LB@~3;j7p&?38rR$XYBT?1oPork%3;r0;5J6Qj^L$P3m zh17e{FaAs_s;y2cYQOs9in{*JGchrVDjpWMfz+#*UVRGiRWSJ(kzcz_cUE@E0=l&` z^j-+Mw^LEQaqKQfoF|7Nrns7w@JoIx3HWy}gFf}oA5d$Y&Pe2Xk=Y>YZYvawqi*zA z%N~brV@M{S!H$Gbm2ALW_TkRG-|03Er*~Vo%Z2X|2y!ck0#L+Axf$&yF>@F`8Lxz~ zan?iB!uZGk={o3-EQA73B9r+nSHnItj+12Cjm?>Cg*-KGzO9flj>Fben=y9H)Lz;UHWj;55Bh}AX2?=8Wlc?ZrfO%kH(~qE~)_GMIl;@L-J*E=1_=5 z4Ps0eesTVV-Q5?-mysB^F9K{I;lHsZ$1%HawRD zgh$7nqXbhqJw@S?61rMndKLAve11-)mBcF^Lyci49jJe6-TSBwHKN(ENFg224H)Mh z1po|;t(>!`CUF4AAmtiBAjn>(GbSbej-L&5f7ZMEC@0_<`RuWGbDh?_Kfm1g1F~>- zleM+HAAJ)=&JlkriMD%yyXx&}%cq15tk`wJL=vCubLG{8l3A@rVrVn&4z>CnV zdnlk$M?9S{YYR)jz&|*kR1LUW0i5|TJGVYei`HmU*ogy6jcs(SK%QcIjR86ZP91OPuk%9pdg zvzXi}qUElyan{uNc(+Sc>`SM6ABr~|c#gmq6t|IFJhr$9`6saImpy!~M3wLH;d@hq z%3i+?z-GF%B0g2)2UD^buiV?Fi@7PaQiX$>0<>Pl2+!F%}$JNEf`oQ4&12oEj}r? zPQN{nj4S|nfuF^A)z@eSuB(EV<1mS@7kV8A4G5m*zjgoZ3kF&H|AD+valIRPta$aF zqjy3AxoVktY_3M7LQhW*4HHva*`2$0AM5Js(%iZ82#mL)>(7C(?Uq?TI(4*w6Q%sH z9NFROpZ+T)3F=<3yopY+v#;+Q440ZgiuN58Mdx_pgZ(c%wlyy&ewkx2;aTY8Dr|u# zW?d&w3=O?gY}gU@EP%MBGf^O*W;!I~s>wL4)NZj)f-B@SXvw>kK|WnF@`3HVz=m+; zoVw4YP9um1+-*2*RCZ&oG$$ z3i8FU(FOxJDev=R6kAgshyD+~>k7&Yt-MF;^XBM|5q;E-#x0NjtCcJMUuQ?X&Mufr zMKOnhoa(ShBF;2Otfqq!fI_A$#b}OY6M%BrFR2Wdn1lm7LgUUCPAnr_eEi5HLD!Lk z3qO!ON~iYhBAT}F%%i9|u5iy}R-gQe?mSWo3Wi?KgQp;;J;tFO3vi#3CyyY+L#4)Z za)x{QrPv0uh3lY?qNUC1Lvi%tkNi?Mp~=#Xie)UvKt9{K<`R=$^D1uf!8~4#B___CBSpTa1zgIZ{eUaT_%qEZkaV;kprhB=A%`vRu`8X3mp1`he*CkX(yl-<%=j zhY!-j1=it5#u7LXQ| zx5szu%^eksZ&lZZY1Q*Vq5P)P%W|V}T0d{Z^H^E$Iy+_sln27y7t_1yCp@e)wf7Od znE{jccP>)U}cx1ltavznK0DLe8oD{31Xkf z;U1;opb6)mzTzJ#p3)^gusNJvQ|*RvKGa@@6iJd3k@}xL*(3Px^0{GaC2e&I*7vn zyI9Q!7mM7(DyM8gEfTexJTq=p?|kPH?M7>Y+oC6CS^BI`Cb9$vVMi=IseWIY;6+N<5)PUxW}Fof{15CvIqVm5z*JIBE*fUPM4X)?h_t&> z6)bchPRy!d?Bn^8w%z{449$&y6-Cg$HeCEspMNz7CmN7Bm82(oZhpG4LXjchk+_kW zE2(C-lwiFmu`@bIIkIWK7RlSjNv+LxHMbWAi(8`C_Mo4?TqzsY&R8nTUueWjC?fF9 z()7&Iwj8VGAC6~A#Wfi6QB{@A3I-XJoO(xT22J_A3!m?4Wl+J1y z(($ysW9tRr-;YRXTwiXDavXG2pJypy4ngFs(zob!Rhxs!(ptzzSzjx2tZE@DxsqX0 z`L!|`U59O;BAsaEvG**$PLPc%dz3akU25}nFqp%~=FrKH1w3-~bb5g02Q-j~*2eBdp>s+-8OpZmGOpu-f{f7U1lzHra!t08vvR#s zRv(BG+2neS!v|kx#Bs?x%}i2Q)RuU*^FGm-V7CBSpHDOS@%dpv6b?|kgRk`3Z4Hod znvu?>{F}_t_(FqoiMDlntpY1+>Y2<}C(7Zd7;#)4IiTY zQ4zO$4j^=D%ByGN6jv;$(h3S;^{<|1`MQ>A|6}aofZbN znaFZ@u>nALp-K0u{UR8_q#Mg>mv`02Q-mB_A8lvC3qR4;K0mwrf?{QV+DK1-Px(pv zJ~ZMQ0rK{hT)}mf`CUdkf{P*<0l54gCd6$&Q109NV$xI_Pj_?q_(o-h^)?o2%3Hwz zZa_5{6b2?_Rie1FAaN0~vAiTH&^!6Q!6*Ck$p!$s<{J=TZuUO{C(!s50m`_Mwq+VZ z{ayfFViZv7%jWTzA}Q)+)=EKuuDe9tz$=ldWU*z~~CE;H!pK8iDE_ z*Dy2IDsj6j0y3mr)w6Ih{MTO_FzJ7MXtSk93| z`R3+vlg(CWBF*5r#YWGq${GR9+|*YEEfW?eRr;~*F@lh}PVIYbGOsLW7fUmm#tyLi z9f^jnlRc>Fy{uj{7lm3rv;f43y#vOhFyG7k(4W`@zE`%ZZt1?1w7Vft`gzCvQoZ zmv4qh5~sYy^utqjTVC`tPrn$%LA~(lksbhd(*+Yl7etwnFu{^dx@s1qDCyW&#Zxw5WYp18Dkp@7U z5*4%w%$tr?Sbynkd5@tC19;CN?XL3WvMk-c)LaIJmbPX|o({sit8B&1?drP|o}CCf65hNc3#D5=nNegMNcgp7vYWNPcsy@}$a*`MBEQ{h6K;Zf_An8aVG_orZZ#B=FaJD;BtP+K1 zl+m##1u!q_>bXqmTCV%g9F&KaSSiihxUM?w)_2_y%GD^ZvJ>j7(UR?SKn%>2qdS20 zhwvSxP6G0oi(b4@cNW(F6T33<8+HYo;*w@*3?H_A=QnMS{RUQ^?$63hgipt7#6=6~ zF=e~uB|*BAR)oUYNo*Zw;$VBl0P>1u@_zO$3WK>^f&RRm@3zq9Zu*^Q>R{m~E1Y-| zA7oW9O$7?hS-9nzc+N5Q!$!{MaMDO4@ zTG@N99*u?CKR`)b*TdEihPxJ5vEAg*gxOuw9JUAW8xC7T_zy#EB zMGT4=;s68o=IvYmq;L$7jB{$h*fKmtV_!gN@<>BxBVA^9oJWJTZto%B+>dh=2XG_g z6k(T;OLSL~2DFCr5iy(Q%O%R5+PtmR!BIgKpQpQUhs3I*pW$rcqM{j9n($a7%9Uv_5o`qcEt-CNSXBW5hZen-sAnVd$H;%?4 z=ASqkJMWO66=&5>7ffft(AfOyw8~RsjegmL^sE0NY5wb6`V*P*R|)a|gGd^LY|Obc zX9Q^_o<36g4^;z>_aCU5wUeC{f!ct(o`-v6xPo>ejBnuPUav15JUcJ=RY0(f!O?%{ zQG<&=McXR#&=sDZ{NH0?%iFMR8mPQ5%I?zR`B@trWQltKsRwA39U^h zx1wUs;^N{i&EqxXa!_vP*iY-UjDgUV#2wCJq$z|y37G0qE|T;K@!kp9%~9mxCI1b> zY1;AfmJYL6&$<@^0KPb9K|OxZHtx==&47)Di$w0@aaOPPBF+xb=9Th(>O6lZ_;l%v zN?t?6XpE)x1yaVVXLW3QF^(>ftd9wv^?Ef@8E}!Cr`^R$R5kLkNAZ|Z`H}zd)uE8P zaN0m!_3`p}EUm{ts@T2y+o*%IJy43|Gdv*86_P*D*kGj+>EIcdbijk2bf9nu zoFM=9sF?F~Tk-Y>q|9Godz3Ume;uPgf6D!%xTV<_ej^zKNYjzw@$^joDn9*Im~lIj z^g4RXOfv1D$vABYJlL(A$k?TaXxz?3Io&BUgD!>rj@!}KV@qEuTqM_=J?kLghzx7z z9)?p`jed6SuiVr)jdYlDLX{FYAz;OE83la&v%25lOQue}Ht|#C=9G^YhhhQ( zzbZvoN6$aV0rMB;gT4{m9uSvc=t6y|JoWcyyd@cQ^jLXt6dBk!zLF?|*T>yS6TtBN zSmz|src}v|`~wD=^b*kp--vb~9FD=4KX>&W8n&abZhil>90E)Ko=7L%_q$7D->4&K z@=u_gm*^nnHkg@^4RC6oO24(yv1>c)=e%h!G1rl%y?S=$N9_fv8kqgv+hVjBt|!uf zhI72#W#MJhZWQKvp5(dB!f@|iNME1JwX+U#vL31{E;_n}*Muy0_#LIg8RQ;P<*Bcz z=LTW$^3QntgA#83hm}d*iH^`E%Nq44=%EZLPZk?UX1;ft^P>}@C%o;cf*wfqsS}k$ z-VKH3cD6?hX4cZ5pd6wScyvpu*AJ(AV9EjSj^r_dJqpk5z16%m28F<#Aa9dBeRE$# zGR!A9`%5R;d44a>S5Kx57T$iHmX_9>c40MwNi;Cx35uMG9-rcFI6a*HEQ<1Oj;A`4 zoQwyzuCUUnL^IltdI249Xy9XRB-#_Fbn1r>y14ywr= zj8|hicMjB7U68W39W-=MWfp1@+QoZMWjj^&SDUoYm0?D$=%N8TKB$5DFIs>u2N}}| z=-#fVmbwJ?<*N3&mFLS1>EYln%=fc&ytr0|PlWMV&OSw&;=H8Kyr( zn!s~o2U%^ABQ#amZ~rxB;aB!bp45BoT58W$YoRN1NTp$oqxmMC%-x;tY7;%EhsAT$ zyTELW_@(NgA2lS|pBng{YGe)g9Gwy*z- zC2?GqH0W=+EE7*__GB+b)8)X%YKBGqem!f8<(+yAkg?xS_~nTh{v8G-AkU800h8`z?1x?OEG^ef&EZ;ivG><%Pr6Bp zgv@-l7DiMLwgn5G3jTw&^w(!V3ljuX4sTYyr5OLQHLB5dZe>vuZOED)Vaj9e8-89NvA5@NX(DmRvAE9R`%I?Z9Mv_B*6A6e&u{`Jcn=5U*dvZPccSffRH+FqIJs<9 z9d`+(+CLY7SU9v1pX{k%5YFF(!c(qOEVKS^rvu&1TuN!H?%LX?G1^6m|9P zTT^H}BSTw6)A12_-b9Ik8Ik+B3iH#3OJ@~rcTH=)-E4;oD9$1(fvqOx&RQMs2#mL3 z?9!Z-JS|AxG}j8)qeZvz^zKV=Bid4$w3IXvfZf6b1HGhe37qA;LE`$*w^F#9x%CBvl7k01JJBky52 z<5%BFeE}>ib6~WS!ZO-@%jGY(VgcEC(0M24p%rnhii+j*6Jnk7$2IpxoO{D>t(oS}%Jau&!dl|g*$lB!ccUB{OLi(p~ z=lu%%po2d!eMlz}%^368Uppnv_FcTAe}neQN|iJIF{0KI$<{Dqj~lXh{`~n^Lo!J| z_04KG)0!DqKR$Y1ZDMBCNtyA!M`f17Pe;m3S?$dRF9rS&*n7)k9>Au8#3lruZ)a}K zhp(_-uKTVVq}wWUc*Viv{4?j`^{|!EnYq&TI_=yT3Qr6U};yfn5l4*OT$6k|AG5{y5Jo>7~O7CEPD<3|DGC+LNCc(=)T zq;{fM!iu@IH_DMAi2`;w80l2_(R3_!?9ubAlOC;Tlo{TwQ(4bx8 z9y7vcu*g|&m1O!37bfte2$dz)(F!?+_EEIwABL`tg^gW&dpa!xj*b`59MgP%MhJgD zg;Z$DDcbH1oqMN_7}{8Xb+`UociI5|w=e(6%Rda@|He5|2s37u;EkX~+}ij`->cu) z&R3+;rIyhW zT5rppecitek(UoKDNBtg=I(0C9LOPLl!|Ka`C!2DvS)K+byrBXIYgF}=V|^;`ye}M zv*eKDg>#r(wB_?4NEzr!u>ZfMs&8ObfgnrFvckRE^$52O&_?jz@))5c9H6AYWM3l^ zMO4o(b2W=HUUyo~W^iSBk(v2JBqhlQLuCSV5j?=EeF>rY#JRNMx|jV*7ME$zmdj); zCe{I$FrE4wW73y0i&1?eQ2~=-OP*JfpO4_i+XQ;AnHkZ^qI?| z=fN$%a!6A|csO;jQCAdt{`SiFL9EYOHb6D>0CW@p-UJ&{1^?@9BlzgYY|S3aR_XRzZ5%fx17_xoj8<(ql;Y!q@o-rWtpf4d<2ixVA6Jm0Eg_#S ztVr-$Q*N%$YfBvz&BCqE5(p@{IILcGq$DrcJ--$CD$zjglPCyoo$4+xOgOG4b4hd- zIIZjMPX(}5qakHrj57@iS)R-VPzLeJ<)8s!xZ1f`-@u>+J@ojNlkmx5G0B*~T#OzK z3k!=x{+pM7y2`&9i)9P62c;gC6k5=)GhL|)!$!`M1c%5{^i1(F7vy!mfGv5y*V4?1 zLPeQsH3A+V1#~-_`a!DkZ5#Eqn^D)^pif zn7sY?Uo%Tn!Y(bCLueFreR2^?Tbgb0mqR;Qaj@EqeSgmYm5HIVBF!DR&%h05x z+m{9qOL_DOuO7eG(Y*S!{w==2!jIIVlVg-?Kj4HZ)O{we*!p+#2~00ztO)Q~J7V~Z zpkqCafuybKY{GXs;<<50PvHEHLxc+6B7ArYOg)@J9#4)@fS2|=FS-4r_gzA$cEv%! zWF8klA8SS3d`*n}j-cyK#Ov4C`&+$ov}|l_Nvy{I!lBD!o&O9HOby&MI|GXtRltEN=FOOk*j=2Gc!A@yMeI6ADG9>>ZyHf1>WgoVr?wd0{#t>m$9 zU$_u6C9`6`;AGJ{QXrAxDIoB}aK1A!tk69D@cyb%OIugD-_!BM8i}qKtOs$0BWA*d zG=j2Ft7EnR{ppO3Lh`B7ZllMmP@!n`a^ie7QoC2Rs*`>2w!F%^3&D17F4`P}xp@5T zOryFh!w0$e@b!^knRG<&&dYKzy=r5O>r1)>iDSVu{_gUy9$`%)pKZoMIbx*Df{X_Z zW~&9$wl@}YpmCnTqi6l@>wG!0s;ruq%x$KkL-ub98pdFe$dk8;I0V;Y$8-{Rsyw?a zwSKIONkKUu`Ap4AZdLl^>ovDf>(@7$Ll*~Eg(I_9-R^Z&ufdqL;wB>zO{-l3OLqIH z7`$|qyf|9ek2C#mnv%mwv^@p-f;$gPCh`Np3_^AQUYwWOcK_Jq9-YA!eTP07xjj)~ zO&!aLOuRl%>Tyi8vGkEvYhOz4r1<-8NA134(8KkywvVILhZY~FADDhg-S{c|kqutx zpcyrG^z*6nZoOR1Y9>Mg;AN<52$pF{0uOV@?Z2neHWDiR)Dj!)q?pYkp>KT#f#2h- zMLlwe|I*$YJVxzv0e|g^a+Tk0S5YmOqlqH+s{0k06AVlJp8^u>zUlC-F4vt)e9ZM~ zXDw}h2I(lgli$l@+E?J6;Jo<$BQ!E2W1zSp6=~FK*epmO6G7Xk=JS+?q6c-R67b;9 zta-SbKQBbqouCRIZhVjU!I>I)ph51r9BOl<{3%^?bf96RPH0=Oay#Fzy6E}*i3}^b z_1g~=j=|E&eCjr~cxS^B`AecvE7e*DID__0b_Gii7 zOotRK_&Q-yL~z~S2wsq>s?R|-$I1cIOu>^YQCLmM`OiL{2+g}~=SNSBdKMKIKhyS8 zZu=C(&$x`ROU2a`ROtoeF=-~f8MGsd#rYF8@%;9mnE}NT-f=&AIK056f1hbTku7nl z*@{k%%1fRFok@1HRQVBB)GN$WKUp2PMVEhHzsA1it*s!kYAP=-+?xto-3 z^%lD>qCoDq`SC1Pscaxakx791gV0zK>4AX=VAg+`Ctwk!8m#k8AMNwjE5 zmoG=#t-pH?88<)7@Oy1P;vjLrvESCrUc{xRTJfYlYE>dxPlif9S#P^o!sxY@ZLe^< z{SBOp7S5^`(aa2&g_cYUj?dGLu`fI(^ufUo`xGq1fB%KiD395I^hk-xbjMANTDM9T z!++~JTy%PCA>%FHfcqLEnq#pwp3p9;$KG~!+tCR+ujMj1Y_p;*7Y+s%`0=~7wO98- z^vCTc>g8H7o2F%>RIYf!z#(lsGhqWwN_o*n8kU6`%6Z) zMZAhlTWNHQ9>LfhbQ&0pAA7vN+;+=*zJKru_ZI`GjM0ot?3PzG>VVTG2l2#c*u=|t ztDm>+GUpTKq>`a?>0Cb}*%jMZrRQhzEsx=Q{ zDUC4MqZ?1hbT~3Pfj;?o{=D;iY36KhUBV1^uLz*Zk}dFaQpZ;?O_4gAjEYeW%S?8A z4PdY}HuEx&HY`KKtPIeS*Y@uD;JMnxoNcyher3a>weZWNL)wDb)%G~rK$py{cs2^-j6zJxm>|SB1fWlsvNRqX7RhK<&5vaO z3XwSYwt@bmLiayQx}e!q`8aY&+|er8{WGAo4~iC((V0aJ3%|7>Wqq;x^7{EDyETg5 zkrvlEy^|fHfi~9L3`aqHE%7qxkK*~}$jk(*NO17CQ0Bpc+m9FG=;Ns%C$$%xQBCCb z6@awKs#2$Y8-DCc=|Kkx65{RS>5gSxsUSvwU-0w3#xviOQn|Vb`MAFA0Jjs<|+3ZuQPr`>zZ ziw~gmpMiVCgssCa`sQP437w*Y^8aqD0?x1Kez$A28JMijNEpF`a7tZoug zE<2eML8?>&?OZhL=8Zp-Aw!?+lyI33Va^$3|FM|N-r=LWdw*GX9?nxr&o$Xw{Q4mf zx37FgF2xs;xuKe7ahua!aGoUL^__jm=klsu*4@pMHq+eEZ)6=?DNlZ?LfMlH63Z-E z4jXlUFy$W?x*hM&!?0DDznf4QzCDM7T8RMEW@xf)ForlzJ<|UkW z;R?c5N2g8vD=Z*f-!PG4uciL&o7eVg10@nq_&$1StFyF0a<9zJA8)y)82p^{1OHd% z?U9M3EI%HPM>xze(EGEVD=hDk6K1Dme zJ|S`aLG|(>Ly{2p1DL73VHQdlpMt#AOcXlZT(5@`dRcguupm4>era3xY(^+dqu4M| z-|*OjVsc~vs{MdTrS25OGpib#A#%58=dhKld3{WeG*@73>FLQQZHR0CAjf#D0};cE zOcI;EYe$kPjn+g@QG|iMK1fgg@1h$U9NJQq}Cr!Y!!hYd&)58j_LnbC<{x zb-S;v`e3-*t`$Y7KjOSq_&;C;D^YDo-w`=5UxTI;HR!N#Z)v6M;r3T1u#`nsR7mpn<*8hs6vRHwEx zx9>Baj&aZGk<5uh)NU^%UcgbBbE3XV;>vRWc#if$MTrR!d5XD#R!6A>iR}-op8e!h zQtnG<#mf>ceUk5$KbvuT=ni`{(n&E&f+smrykPcq7Vw4T%;z0^hBh{Gw1N7}NWXrm z)#(#^@7CAs>Os=M$iU>ah{5A9Jkd8n#gD4mn&7O_!K4f|?ws!dVxVP*)ZBQ@+K_AS zWiKo@kS9LG;!(^kvC%RsyyZH*5f8)2ab~FH%3*ScS+%=li7iCeo=L#on8amvYsGr- zeQgjaL+J!mCT%fhaVvo*kle$coG+zr9DhvB4F6rC(1rrx48!4$>IK{J1B(ZVlUV`u zhcM%84SnJt7bQvc-4NNHE;(t7jwp{k1j#B4iA{WPu+-c=*xF=Ox6pBE*|c`t2n;Lq zx0Y8+RIS+a9VoRL%y?SBx)4*uY75naci~k-yv_PQtD4_jB0M}KUV8XV7Wfm@UgK}^ z$?oPKlcok-S5{T^J-M=+Q%FlH|3{(g_oSvg1IThds|X}5VY_kivokh3j|2Mhm2;kwsi)+$kE9iU_uz(FEk3@)gD7u~ z;^Rb^S6aHp-cp$GIPrmS9gXsF98?#NnHK+iT-(me)PO@W@V8gkp=}PkhMy2Qky4yd&b!^7Cvzu75A5QEF6>;I4QJ@h5hEFHs;2u4 z<(!|lie8;Nya%!YX+u{tWlr*kJXDb-7NaUEd1|z6(QLsc=Z9W2rn{SrfL1MC6Irr< z4K^~VPW(SWc4z5Jw1g^L`g^}nKZLgF;x?4-tc8-=SX|>q< zBNfNNMA4?(=CzZCz0+OQN>k*BTWzxu#iDJ#(G&MhDozToZ|@8_mF16{DNaP@;x`!G zGY#21Y|9U~71gHZ9rJ(=slMWW(oUzTQGb%)zOg@81{>N@E|&Zpbdpx5MI-lc*CtE0 zt|5!*09K4+*3X>at^(E4T;*1^$o0;5rA)ZNFUz^O-y9zP#NYy7*ORRKxEwHkDp&8|@a!2mh*XWaRo)Sg>=4!iuuwU){`r zuV2~Yt!8nCjK2H8+jovqJ`Vh-g~Q^md)+IetMe=CW(6)Q`Z6-{Tv6Tg$+U#Y&fR2F z=k*#}pHcfsL02>lGH`#^Y{SJ;Qwao)W8{T2L`fb#NZa*>^Y)i?Y(ixt<=S)N#VD+^ z%J0ANFq5srySZh$#BTUjt*R`qMnqHVn1|2eJ1xglyKp>}jucc_Q~pb5(MIw~%%`BM z_#=oZf{BheCZhz`N)4yZTkzIAwVe;Kv=!#hCV4kx1rg`EsURld@Il+ohYeMti86bc zlO!$5zylqJ*>M#QUxHYdPQ>*t9W=2Fx@0c*tnUsgW!9on$xX};C`Z(JVukE@rd=uB zZ$IGDoZX)8s@qsZXAX?zWg#d1M8*^QtMqWb(jwW!kqytYo}PlcFeX-xt`(V&#jA96 z8yO3`s33tI@s>l2z{4eB%I3Jzr5d3sM(7F>_@&`{#3IWm1ni;j?Z>zFaSdpLP z$SsLS7iQg8<&69mZ6m|O)#sjA4Hcx?EgP@w_oQ$1L$q58UeZs?TOlI81r+G4mv2#Z?V!%uVE-?Zo!N<0_xp{iA#=e$_) zkWKO3w)D27uConvF|o8KE+}_UfA@QMpuZmbmVf7~7{Pt-YEC`a2lf0KY1xO<*lZ$c z7jo4S8KBkcL5|BqIjlRx(t9qAyR-+U=EUkw1I(Y-=8Hw0p&u3{^LK?e?6MMBA9N49 z@meklP`+(77lqKfdrTc|M2G6N^Ztx#ZdVm{5>~e=2OlMj40_wbQL3f}*~nxdWtlI{ z?afxxSvTPZc`vjy2&E@nso^e>e~q)i^&kUJZf172Ei1>8)uA0hpZs$?K-brB{<|`yCQ6&o4$LC5zCAGqRi2uP`R;X*3;HM5phhLfXo7juMpO+>6) z(=MbvI-0@13cO^3ar=M|l`A}}{9HfwK!=n|bn4gMbw`a=_qwi{{G!&~r9n9mfX~&C z3YZNrBS0+iGrQ}-rugGMRg)|3pRj1v>YODkJ=7`XJs{* zP>aT|rXI~8O$zc)3NydEnakHZbyC>5sYZ;ukG{BNyUy41U6nz_2phpWz*=Se=HY%< zmH&C~yMfta<((YczVG3@h~*D=HNPqvx$M_T&990<_}o_tQ8RDu_&hzU8TUCJpdcLE zLpKi7P%=V?pT-IvxjV17T$qUvzHOzl#9T||jq}ltZgMG{l0AT(YbV(ucz63?rD_wW zt64Z)p~-!tV0s1_P3O${by>oWcpeP)TGIT)(wpZIy;8OW4Y^h?XCLrn|3>e`^zH=1 zpftD6Pnh2!ayBP{0WVs$s^2G8!QXLzv4D9B8Eu6bUm@ zDuYmFNDK*xB4Qy>MIZ@DK#O4tA!-p&&_ZTlHY;^JjN z(SKw$^+A-D@w+fuY0I-yz!$XCK#~QFZ?MX@U~vXq=_bR;g}8BD&=^F#9>8a(6&8|i z;&`TU%bXvteaV=CAR^cIf50|JeCJ&OA{klo%WNM)ED)G}ZWS)L{OE4>Y@6f9TO#7_^D4G23ND$UiPxM^)rpOAgx zrh0;0A6NdCT@=~d;MVbnd7GCbXe{f9 zym^rtA33dx*{43VsqPM{v@_`bU0;8H>$5y6{Jd1$^`gQ!Q~D;+U_}Aihbb90 zn+nCHk|}~oLtLke>Bq*(is#8YZMle~Cqgozd{=IWQMP<`i7O+HkHgEahg|Hjuu(_eBIR7z zGGOy8E8hhunFbwC_ffxvIoNy*Dxl>ZJE>yiW58>+d^!{fUK6P%!?q4v^zHjrpQWB6+|L zMEBnB+j8>$^YM=3=541#4(*n3TPc9z-+%f4<310$Z7%Wum38So?C-fvg32h^GJD{q zv;g1Wy$vjPd#p^C1Z4$46x_T*rNK~7kMU=+B*MI1sg6P(V`QmGJr>ETt zC?`&^$k9{PoeEpiOaC~4Q&IS)U(h+0{!_R0R#6U%K!?ACS05xne;kn9Z)p1m_zH_b ze_n)3e&J0I0ns?i6x~_pnDK*x0)S4I#%K3;OJwML+uC_!k6}fj6v04Al?RzW-D$$^ zWJmQ=M2B(HFJ?eG)$1!>=cSC#eDA*|WOrCDuXS(JGXM#?tOx;(rWdBhN&Q_ptkqpC zm^>qA`rGgDJD!VzgM;}%y&zTZJKyT9&h3+ZpR&e-l6BZ>zt#BLLj1LWi%wU5fvTzY zb$9P-$XtGkNh+kHA3v7WT1&)W<~IWNs-Y!Lqld*)sZ=I-u1_5Jr7IJ==Vs3{DvsW- z>VA^#h}LrMH2Hh)k2yeMw~C4q$#)@i!#5$5`-$I!TE#f9`Rp&%^3!i}sE{z_q^gMj*wo57+-Dbi4b+Ql4_J>_oM zValVw#3BJQ^vPBEbs*UJqZg1`NCdFE`P`W^XTDK>T5}Nyu@|`5>N-oNmVkv+W4XJz zHAI6Z5(yy?M!0fbA8lYj90{Q0tG|v5X?wx`9`W?&ye`t5kP*xoK^4bPO`?LGkTG1( z{Y#>2wMDUSq_M=`IIAoa**8Q8{zOpN;v09pxSuz&PVOnFc6;Y4eEI8shnCnO3Fq3T z^kr&<(V+X~JLj@B154&qP%{LuqUj>d@SH}Vi`>S zC#KY_M`|KooWi#Bsgyd-dPXFPDh%M230BM$_Tdi_O$sDw`&r49VYR!)B>Yj%W%4>R zk#fX6@?cWOPqNR{8!{O`UwCkE_;mq{%o*)0@(V1+_VkOs7fI*WnoO?{g?k5(=*}GL~ z#XAONy4h63U77I~+T;=I8C}*Grd11F&!XNIIx&C@t){*eydntRTYfJEjoW5@O0kStJAy)(p2?PZ7mn!T8#WNzHN&n% z#8wpRfB4myBod=r5>u0ww(w*=S+GGy+B1UAfz)=5dxO@?l`*T$!Sfxk4E|pjbixBt zIe38fe*$?PUX0<`hmEy+!4Q4nF9w3zZP1U@A1}!A!!*&>h`Wv3PO^BV0F zV7W=*52H^Sk`}`yeer67-9vwGH!|3y#Tuo;Dfbt|#)2g;-_X85=1vyT!hDYHa^*v! zlx68uz{7-gJ@oIk{j6;z?cj=PYLKY;e_0fbXoWT%5<)0W*0|jiBd%SVvDKYEUT(5o zAp(-aRE7+%^s$q{9<#!RzF;<$T4M5Hk2s^<%|yjth+kr?_eo&<~#R1$&!!_{qEB zdxkQMqe)^aXIb{c!k0dwJbniT9mUhhfsAwm811TpE$!2s*a0_%iyC7JNFrTce@iqg z%=^CZMD?pc2@-zWq}?4lwJt=}k0L8Yn0yW?I2I|2=wsUi{>cgqL}^&cspxN0flrx6 z15XAj-ejldS+TJ-Xf_q5TSXgTqK(kuzC=mJw8vAM;P2(8L849aJ&w%LtPLmTDSS4S zxG2alY+ff^(lTk-Q|AX&s21irjnb`2WB1~gyNX@=j;yd$2p?1>QPHZm-<7+1Oj~Tt ziR)WS*ywoW-Z;_1Mu)K=IoE2ptG-k4#DMvh!FnoWccEjgQg#t=?)+{?Va;oO!Q5`=wn&vLJz%(-s zDO%D}uA|-hhLA`ymN-SPp_1_(6}XnOx0{BiS(Qzd(Wu$4(`m&eFSH^uiCTST^VdF# zqIqRp=jia3jt7$Lh>;}yG98&Uuy4jIwKx?iTFiA~-L@c9#Q^(6P1#E|c=ZOLl41e`bc4*C0v(Bc9jhq}`UPdnmqC;63#4PF9Q9YkSMt;d|38>zH zz0T8RMQ2|eE50nDapxw+^LLdFcymkM!1SEA5r`ip^#FiWS$uW-4MW@vGKSZrHl)A%P%n>i`qEKPkERU4)c{{ zK)nZ)uxUnqqWInk6py|m?hm@j=$BCR?yzk}eiX+X*7oi6d$pmGc1T*9p^}+hi2C&! zS2!6;s9Ny?zx%Y2ebXuQli_QIw-<|+q`z4VWHm?vrB!RvZ_>lt>0AX0s6i*|DaDh%=0Y&Z}~f@o<`#B5fh)}<5NwlbK}pVc5Il< zNjz)!;q#>ZzcM*UnE1W*XAi8aRv7f>#T+kIB-Os*694yM!At@_Qle*7ihu+t7CD8f g){enK{Z=-1=G{%qD?On?*#=&xom?F2zVlE2FDSU{m;e9( diff --git a/docs/static/images/changelog/changelog-llm-as-a-judge-response-2.png b/docs/static/images/changelog/changelog-llm-as-a-judge-response-2.png deleted file mode 100644 index 2a7fa18e424ebcf87786ed3f7d09d59ecbccebdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 108020 zcmeEtRajij)+QkkELeafxVyW%hXi*G?(Xg$BzSNVG-z$VEN!6ZfapE}74tYeFV0ge>5}2W`OC)Wwj*!`8;mna@Lz>>us& zLGOPRGn0}0;}sWcK{Cw`iX`IpPNpPWOe{<+WJ1U!0!}7oe9971|N0pEBuHlA;^M%^ z%n@gNe|Y^@&3|71 z^)a8KrH84FmV~9Pshu-)2!do>oIL+%>_4@d|5eM%#>w@M>R(OqDVsXm+qnK3x|*FO zbkJ=7snYyUogf)22M06jf9kY;ulpqq5`lm8spf3zByMjD9gxdEO8+S)QJa4~{qxbr z@|SG*Yz*zp1<5=ZO-#)UU2R;*gj9^3OrbNeuy+w){v}Nt%YU`;kEj1>MBpES=^a1_!pJh-zLjlA5Z^S?!{*5_PJ1Eecpor?pYORNXLH=$jF0LpmE>5E8WN&6^ zV+sQ!6PcWfpqe=UD&Y93C_LnKgo~JckPEB~lHkR%O$dbu4Jlku1Zr{*jxGw>(|e`mq?-`s+nHf6a$EPLY92NK={O09X`#D&{*D%F+-IXPF-;uwlkNClj{_w%|p zIHP!EGzU;nM<>&)HPs9wF9)a6%9xhl3WHADC>hl_+QOfG&8NsypfM`=oc)lUa71nB z=ksw~;GC!Cv?i0{EHcg9t?Cnh7WEQ}fl> zhs%^3_L?pV>zeqXrYbJY$u5%=`xAlZ{vY8P0%6j~W(Wcp?Br*LNf&>@TZaI;^BM4- zds^z^C600FyszPCsD1WvdudcJy5ZVvgir;<@KohMSA5+AiLedL;kqa5e z;CKNH6zZZ$gQIWXs}cgh6?VOxLq`kOXpeMwk%V+!lcWkv7E0MZyUW(jGMpHa{Sa{Kext10xwd4y3 zgn?$?*vIk5K@&JTcrNVC_`X+bYa!x77KW7=nGs>y{bHSRoqSz~mX_9**1H6QHd{4Y^~;oO3B9toa*oUyg>eGE zC9kZ+f+(NV9?23lkTP-L&P2nBfj1futSi?l<0tk=^WigBqS-s=eC~G|@3`gA-?b%> z_U{a^n5YkK4kis|rZkbVGs(qIr@E$^4%o$)C)p+f>G^4;Ki9;gB*-O#m^7%})sC73 z$zuvRJOHSl9s>~%ZCXpw}C1Jvs1qH&YkZR@0tc8f>bcTEde62Ie zH;OeXSm%{w#gUvYVk;?`%9>J{;-4Ceo_fRj+VPF8&Qz6!!vq4Q8l?thg>s(qw@KTA zu|W<~g)Qu@r9sBQIJI%QUOG~`QZ=+j zVWY%aa7OD`Pc9@o#p_$;RWG$4PaGBY(OtH`(_YqaeXSFD)Wz*)i6_fig zBrTWkDpRUp{SG9;FXmn2_U+1;@GF`gwP=w))TEbbFPdIlzKDA{5XKaS8#W!*@Tp45J+ANdxPOo>eEHMnr`JzX_U~WX=FMte*J#MOq;@mS1M|%X zxSjFORnN({(HS=xeHgiPYIOiVIO9 zyE|4hXOpjkyMw{;@><~8$ra7T&Uw=Hr<1Z1pY?*n=5VWajr^qGaYK|bGi-qYP@-{C`09zf4mVO9bJpT)sm!rlflijqX&)>7{{R*5=D zoiLNm#jnN>#+&E8&f6FB?xZm+?cD2J4sQuPe(C?#@GWW7PE=B)E|C`gE$JZncb<0y zKngG4i_5pkZ}s0>Gca+nU0ED#vW=N~r^YZeXz5qok0J--@X}XQsA_#_K3k;wqE@C> zw87%2Xpeb5rHDe;c-b&{ebU3=)=QrTEmE_RdD8FcQWD%qy=Gb{IQPeFMX)Mbm zTOhk3;I0pU>{_jG*0G6BD@xm^T&VI*8UN>rrK6>-kEb7U8^0d8AhoZ^M=~0W2zCOt zE3@~A4nh5;lkMTA*K{KDWy|x}} zUjtKs_10N4%^&u|X4q$l7hr57R>td;%#9y=KMNcK(m!;6pv~}GvT1GdIB#K8(pfm4 zK0iMTZ##diXccJ|(R6Sx-Ixhp`PK&ZU1=Dx(z&g-56$efmfV&skTj^TX)u6vtcO<< z7rIj%?0_s=&fA#|m>w(mTlCy!v4?QLuhEZ@=6>pm(bF<`Hn>kOmln(vAY5Z!M_xrj z4>j!}*CF?0{}P$+je5K4^O^(|qexr{a&npn{(7qEh@ZH=>FUMV@c@kSy%v3T7 zeiF>|y_x^qPcA4t;s0ImyHoAX&aCmh@p}4r`jK|a!UA7ibJnE;u-Nb5=4|7ET0p** zD~*f5Dxd3h796!CwZT2X^?8gasc^ z^8z6#!;clRHK$cxlMKclI}}h5$np2P6FUkWKojlrFgF4_%zm#3fAfubc@J&(sAbcA zchIzQ=#BSV!8a8=dc5A#YgQo6ySkM9`q>O`Tdxeiy4$*vvHHr}u%(z8Kgi=%V8L@R zg5C>$d#^=+PqgR$-C;ZExjWg>VoS??#SIa7`5aPmZ6Mt5$8yPXjJd6}W<1|lpfHuK z`eb&Y^KDMu$FASF_}X&rXL#4aT56m5_}8tE&}Goe{0W;9&&iLRixGhbI#3m$;2 zhTm(@T!0TJG1w`BfrKQAL@DuO^|=HFT)ud7{^ukqZAqlh2??(-F_4t7;FptDuKW`84^(&0@b&D*ud6GL zGM^3i#MA0bZS%_n9RR-QH@oa!Cef-~DT-Ezq)Cqw{=E>Ok|9}#i!D{;&eaEcVYS_# z$whdo$JeM3bnqT~O8A2WKwf(FL0@KLo>g4ThC63uDOCAA++7sNrDhj)>Q2MNb(sB0 z)f5<}a8osQOZB!OeP(^Yjh0{p&e8#PpX;c1Y}X3cSUqV))F^~J7dM8P9}9`hhlu}p*cMsLj4Pzoc&{iK5gz}v$O27o>8ZO?7?>kqIe z7P;W6R_zVIT*Da}0|}H^_H$K+P21`8_##g>Vt>Ta#Jkm*a-E?>3Rz_pbgM4CZ@T%; z$d-n@^W#@Fpcs}SLh31vphRi~?vZrP^ZqJ=2wT~|Mb|IE@*&+rj@vGO7JlH@h(mH5 zrJXW-o98(gT2JWPh#OK#L#f>9R%`52V0nxWP(yoD%R*AdnyVqgX zoXlgM4B;SH%$-bxD_0V*h%J5#TAxfVO3C31`9BLK1xu!2Be>PHwtUqS9AAF5r3>qf6IllItaiiJ& z(S!)>r&yKSsP;-d*Ib_AZGdE5Q>TYE7>_+v9USxcA@s-mSqtKDxt5+kn>*PF#{6`B zB_w#xXG-+KOPrToMMmRckpBJ>As}0wYRo*(F8bKuS%L1#O9rb~oMj#CfX)yHl6!s zLfHuCSyACu<~0FD{nDu}?oM?>EU>)h!4n#2Hzp$TbW4{3AJU)p$Z5#uB~FznfBa?9 zo`|i^9D=ItpG2D+Hwrh$+}rZyvj*kpE20$O9{K!~d*7!s8TbNq?Un&s*HZ~pAIMy- z)e_BO_GM#jwC?M-q<&LJU<8&-Jc+3KAw0Tu17YQp3z1P~)?OK_Lg1+Le0|rHZHCIn zE~zoovYKt%4CgEya1~l7K?-YVs{YqGs_~J>GwUXw+iJ9tHbLF|_$0d1q$erX+RM!n z_G*jycLNE$iBo0j^S0Pm_seA}Sllv_2cZBr`p%13cgG4~<; zc7gZw%`iB-(4NV1BEPlPa#`jT)u#iFPubEaHlCNwA0~hFiTrG=ieiT+9;UAQju2U~ z&t;bq93GoK`xq>p?+}OpyU!6@7eqZj@Y9^K#VgFyEBxVl#sBQ~-hOkvXQ<*W!(a&0y&vByF&_%(F4+o1@TZ6M15B)Rn}{fhR3c=$cXSjOL?=*6Vc&T zxrTMfTsZyjmWWXZOf8##Q0ADZq(tude2KAm*_%_m*-H8*=TIM-daz>SYOq_%Pacs~ zXeME@;QHgO&$1EwLQxJKof=(AGQoV!;Dm9X>1<`&&?Jtv=|#wHwe^@io0Vdv=I5W* z(+j9(0MzATMxc<*hSXg$p4xQuUjAnPXU^p)XPz})1IC0O&~PT`^|4H8A?o5=5gEdT zYjLeQw{IMeEBkh!_4Ur`EdNr@~e+LjrO}3_;H@PNCOf>;A!>uRsrMX^wUy z#ant#udkSgQ`u-2UI){Skma566ET{{d<@^nG%l8LYi1;aZQ~q%65#8u)pEl=ki;5n z+e!OZT*iw`y$ZN872L`^UlX!&>A5znIvtE>M>Y%>K3*U%Zhc#MQg+f9Ht|@y@g@-g6--cUC|t$s z{&+0njx>Q*Hkv7Du20J4rS(kexe$?cJ)l%yeYu<0#^-(_++@8!4lb|4fKi+34fE}& zq(#MSt1v3020ZG`ySJpagRlWC>5SxD#OL>)r%&}Ecy8==6ioK)D$}0c|81crv=5Ye zol2EO{OoJ;Bh~}&+d;Jk-viA`gSNH$Wv00Yfxeji$j7q0w4ONO!zAEff5uv4;|goV zPxGTAu#I))c{06d)Aa#c?1l;MLfS-;ufck@Kz*>=lpG+9Muagh< zG8bj+V`q&*_YX;N^qlu!XcLRZ)H*haXD5(x`k8?(J|trT4#R;VxbJ@5>qS839RB1@drd|nY@cXkv_xAU9 zsJI)0eJLzTWQkoUL}n+WXB-jiHB;>7_eD98DoO8hgtE3JG@=-WUv;!+KGFVqP=rMn`Z*OBAJYBw0lB~n)z?^8|QXL;ovHM{iSG2T0>WqTYL?oQT! zS-I*zS3hk^z+-EY-5xa{FtWMwB8_~y+;@j>c-2FcYDcKu{|y}ndBQ(+S={9GY?{q? zQL)B$HR4n%vzlxodQ6!5V>ada5-iU2K9|!_XavA$Mx7ze&}o8@&qso;RafC`d%#qn z;J}T4>0-leSI{EqzV$dj&(tX;hGPeN~Jb6)Oq?W3gpB6Jf|a!XxZyc9HyJNe1=9f zITqL9%QqBV$uWMiYM<|V-kZQG-hW-w@YodlR~tZvL>CSaP5Ri=L zs(f~U??7P=$dX7Isc!84%I~md7|ux0-GQbn_mxybnd!99UO)p?Yt4wj){y z1pA+KdR2blo^SNbXd1=X!Vqdaf{dJP)DtSVS@M4ocFrUrPU9B0Yf(`BNEa`h12`R& zb5aM6&RFFkAhMq@=<=8cS^F>pHI+htqHGHVxNF0sI*uf<0(YV2SP9Vexc+Ds2-(a^ z$$Gz~fmn}Gvl>}{2-Z_B@_^@lqi=lT+I{fYIbpS2DyXEGK`v>fYYW@*y2TeK0i~4dTBoad=Y17vyfQ_9 zLrWqK@ui8paPH0EF~1HT*S-5SU|cAHPVHR{Drwx( z9U7-GEB*aF0x3Mw^=FS2yw01)BC>&$95)$_Lvx3m7JVIz3zE?S*~E83Ic(Y_*qY)I zwlUlp{k3D;viOH%v?Pc84lf|f=+J_D88zLd0THMqI9sX4}3 zn=WxL-?n?8*G-(Xz5ehB-lMMl@*Dv@gK8N75(CXP0aYqHq)uoar#9H^JcLs6cq%yA|0HRqwXsGTsOY7Yq^+M z`Ih??`Ud?tTeZ)+`takvYiUceLM^`0554ClvccRRBDSWq44Ugnk3r^meksBRM7U3m z{;53SFAyC=5kud7YpHAAai=@ef*3!EV$C3rF?vjDI--F{*DsHk_0Kj(Ei)jQLYFt+ z?A8hJ3N;i;m4dyH0U6s&npS6^c{D5aQX2t)BJCvYsG;|SboAR}st_9^fV}wSNc0=# zA@7)Fw@3vyet?gci;ZbWVvMj&O{w^Mg={|KZr+dvzZ3oE=g1TNeaMLE=A#d`^SKDh;Ufb`3Il)%50s`Q8m2wqmB-M2k_e4^2-w=mGY5rBGspdhLm-$1{vJOiKdiboO z#B+6(^J>qiD$O_ntC3|-{2J~1j+TDa&TxcGR8R;EE^fV?xR@cvBVY?|xSY(XNx6r7 zLy`23ro^zV^<{Q)zcs>-(VsTU5+1rzVI9#JWR#uS^ayQn0ubNIwG63ZiJfxp=!*SN z;~aFR^aN7zaY#SNLTlPL;(|Nzz-qDiHpx%A!d0i=V&F{rf%vS2$ zW)Kt#goQu5?S`)ej5GMR<_Z`rFOW*7_iNv7%>tFL=8KlnbDP2FIt@dWZO4himp<{u zMAJM^kKiCncYe*6U9jIfJI`MRDeg^`EZ!lRJo6RLgq)9Ks+fG!i|!3P*q)*CtTyQ- z$GDwkk>5mk8Fz^IdI?$RCSYs{1Si=8jKi%2#UYWgRUBGjrfagMpdOdks+$9QfE&ZGPUQ(+dM|V0+;W z@Z6^`zbZ4UGq)gcD*KT2P=KUW2gz%c{4A!8A<~0NA2ny&|M23NPc;=auKdZbbh|>% zM&LwI0;UIB@sy)1g;UOq*JG!+-I^5uyYwg=3`b$0=E_Q?YWN{rpd>e~XWDw!$4e^A z_1@6-X*M#KE{C@oUQgN{QY4qr3)-X63x8K&+u~;u6*c#~ql?+QoaC_4ZapMgxU6x1tMcpG#CEiB((A%nr%^M;K%yDY2s{alyTp*^y0QEW*~g) zcr$?#xc?*7KwOb@k2G9Qi`X#AeDsB28f5=`xanM|RB}O4&b+^(B&>;oQv z)O4z*qW!riWvN%FsL8J2M-#+ACWRwiGsL3N%u8+;!C8vY3D5x|5c?KamE1=q-=1$`B8K(J!+COgfv(Hg zfNqCjn3+DIjNTyT>RBARlU&S>*c<^$&tvrF7ukJ0d6L|n#%q_oVw4LSPqUpK+Fuj) zK$X)hO(>)uLDNwGN$R^us<1tgvloWk!~Bm8(XYI4w`m+9-^6moFZKMVwYu1(GJdjX zpgob9YTlXX{JwGn5JTty@Ke!T$Z;ZXDnoe;R-=TJ=WN7CgGHj)=lr6C3Y!*%a+sg) zA@*rk-kWQ=8;ATDn-iuGXZfc&f9}~VXP`4tQ!8LAQ9NVi8vgP@=5wyG;nF0|iXoYC z*ad4c=Sa|ouxh|%?aSSt$Z%=Kd%O`28s@GVWA)1^)~cl@XKSR?%S@Mk;*|4L#4gE_ z+S6PNq&3WHB{a^h+WABt`|zH9XIqzq%h6F^ZG{f(&yf92cP(bCqJ-2XOk%e&@4L7` z!dJv>hrW)kuCII8-%ZDntpuA*vpce8aOo(2i!vx5T`E7ZSS`u=(p;1o0CJ$&R1_F(jrQh1S;7f=q zv=s2JZtwgMtHI3~xn6+lT)O*^F;X9nfAwo4`Zqp(RgWa*rbp}CnjkwsSB5)_x3LJ z<}NicyS_GRy1PrM{Qi7j{%#JwBU&Hb8jXkIg(1$))N3Za5*%)EuN*@Gtilm4+cFF5 z*G8*teZl7kFyp|>ytZm_ues=$5hM3AKHrJ`FP-|QSo-leaHI;{O{X@4LOEV;8FHjq4iSPXOboeLe^bzfXm#W zHj#$mSH~eHZA7E0VQ4&cwjO~44n12Bk=nM%vni8|0IwW$^R0GoN+-SUkP73b49ABh z$16EG-haxt+_xcaGeH~>%rjd8CPAX`acoczCcHW2;m+F!c(Jaf(O8{YjnJi|(3cCA z38);%vJ2Z@3C!{huL5-3c4&+1_eQhS1m8tw^8H!%~?sv zVjrY^7u$s#dN+pfj5%sM+7C2Xe=4?VAXmMJan~Lw7;^71YGI7G99S2<9-EcZ*wuV= z1DOt8WwBXodNjC>!qa2>R}z`EcH5YH^T%CNmWf{n4z5S&*%t0t-=vi!a9F;woN3Gk z)u(BSTyB0(9vXL&g1`t{%65Yy#c9WJOhUlSkkr8Inr^Ky$v_eRGI=@8VqAT2lY zRQ8*CSUY@SUlZr$;s7eQJG8>{YmlY#;geRc1W{1t`00-B%{6>4nl{C;&7Ni@kr?~A z8W~nK*9#5nMqdea?>RAoVc6Uk-r~sdXRwT<+jAw`qze$kBP(`b-EMr`om@7LU1MxpiEqu?x|ny~ z7RX_MBs7hFVKuInz5dovE`1a z?Ek=RGDXpFso4%x*RJg8C9cO+@|;T>z?09MRNd$iT$GZmeC!fJ`e9%}??9X@(l^#} zCG&-@a`)jpWOHWG0>nAx7Uy-I%&gM67`=Sg;W3#nYr2s9{rf0$v8o9Rj*xhZ7SfTO{HZTCV{r6v9E{BvrUL;92WHke*4CG=(( zG|aa2TKn4ym;2nvZLO9uY^=1~9x-zL4)LNjoyx&CD$=s)P|H!C51O%HbCmE8ym7r9 zG*v0U&Z|Nk3U-*PQK843YN-JUX^K47(Hg(7{2WmfbJL&>Jltm1W}3BMj4v@pDAC+r z!AJE4+#5;WdWg7_zaP=7QdN8_I?Yz1T#|XJ+BKJX1&#Hv`Q9rI6y^mN_eP%U)H{`9 z%A3jGYa(xrfo+bs4DxJbaRnQTL?}Q%=xG++i;&rQ`eMs7-ii)~u{Iy>n6B|&650F& z+ljoHl4WX`<|fN$jl&yxU3Ma}x*`gC@_ zJJ)kqLPwRh25dN%q`La8KGT`!n-kBcP*@9Ae0^=ZFCT37WJL8v5obzgA4q<34CCe1 zKY5?y?9P@0EA>Vo+xxs&?;o4b{hy4l0$B(m)_mt%O^szMXT6V!5#X-*pswZ_+EZ}` zGz=j|vLqpgk2vn3&?yz0Lq*=}i&wa@E*NM;w~D-sVX4#|`g^Jw!ySnSxRiN5%6_JM ztm`%^e$d5McmzM)E%bg$u&6M}JU4(;SorjRwF+#W_+AakP49 z*2mK5fk249!)yOJQD7g3>erTCSANqJK~H;%BlEl+FF{39<%p0!%NYsNcp4#P#9Gz zjg=FP2rup7F7NB_F=Pt1mc|a@aXMh;EizoxVmD7%hTJ(M<*1VcX@#N z-rBoO<(uQyt7B+Hd~|wuFIlr2EoMvQtSxgpsbhZkgsl*FoSnE1Z{1GzQ{E;R4eYth^Los}?ES2v(Od6#xz4cb0!ZE1&#)l@9*C8q_E%;bGri9p z0@YO|cW4>iFmLwuO{sDyqBPr4u6w{aiBV&M^zhRvoN-~);tm3ZhPy3S2h$Fl<<&Nq zURc?&RWnL^lv*V|yK}B$1gL*K%e&cOv3J(z;kS0BUmb9n0 zpg-84O@l@agctt?O>dt;V?2&V^sgF=kbKG{Ur1S@v4<3HkEKc@<9M{IUQ15A+N6^! zAl>S^bLGs|y5=b6aX15y8pPvR&kR+otyrcVAz_hKq^d<{noMMdH*wgTc$Vnqy^kPB zbzGg|k@%|t9#I2FGu|^V*B6*5a`8$eZF3XXAZ;D-s-Fh$MCLQ>WVx^0Kq_~G-pKE% zC+K-8@)l7s7v(tv7c?J_A^I#qrD*isg3Gn+UF|g{(qCoyi|`-8hNC4B{Y zh>%F0Z64<-zv&;{Bv6&J zBIbls{g#0IpH7io00_V5U;kA?@&XAe0?``*jz5Y(F?U1o4>U;Vpjl-CVD(;hg7f~! z;|YKzJpSY0UPXvP*N72@5c-EdGC8n!Jkb9=;lJL9ynK}_@+SDi)8cOs@J|syk9+Tb z>-@JHqj(tj=C>hZ;)#EhG86*O|EQn;2M~zoc@a1A_x($d2hwX)4(vL=+3f5YaylL4 zY;xTwBoY+mhz$sjg1{Zgtmn&8d>GtJ;QBx}kic4=|NgVUQI7h{oWOIvolJMMA*hEt z&x>v^z+h};fB}QG=6X7roXD5_KTMJTLCJrl7eNK9QK6SA-X}k}%l_u;ZiK)vP=Z8m zyEcJagG=^{fEIxN@D~H@XJRwqeRILjCxS};j&nZNFjMPM&oU^T(^?vxFFN>lqvqMG zh;X>r93lEP;u|?Zs~{hnYExNi+{ps9_fUg|9+IhiQ4g$0MsafY2xz5TP0)GSEZk9mk6bACX6%Esn z3~vo(Wqs9sX^?p1<`LnF5SPs&MmWl7e^j>jA(eAElS*)BeWYMB@~`NELPF#MD*&8{ z111{I&~q{!BX|g2#oQ2l=3)4H)E~;K`66~j%x8GilrM`lk;wE(gGk^0Zz-vP9uhFT zV2l~$Z#-|}m~0q?a1h%YXFoPxJD^X(QbRWZ2X!0SzgYr~K`_W-Bw&J0?OonC)J$4{ zw-iU5i_ZVo?|x*rCzsCZPHTU5j+KI)kZoUfE`IyT`}6C6pAmnfc`PWOvE^egk^+^w zWeip-gFfo3Yl0)eG~kVC8siA7`P!)4I>K9f|%d^CA~)=n`q8cQQ4 z_-~lcfhU-`2zo3HE7wo7rL%s_eUE&!vlRx=)A>z&ACa#N5tFirGKB(>P_t6w>}g_A zfoIFnFRlgCM9FYXkLh-ZVYTP}m~y41PLt<(O0DM;;q5B$g%VG~+zO|}@Y(cAk~8=z z#avE+(wQnQVmesJ^{(tCA^X55R@c3x$*%)X_p#UQ9pcqqOH>*ei@eV7lHZLrrF5~^ zoVCO_ojAm|>)d@Q+9LH89w(C`DSK~U`erK$H>U+Hox_sN@?@L7!qlIA)HqKs`P~+! z`{===TV#^$FYCcGIwh?&fm)8~a;2Cs&+qyOqM{F3fjCJ&;dtLG-CiV;V(!K9|M&hH z64PGRWH1CJUcggldM~GL3(?-m7}B16Y<=rawZm;v#-QK&&dcRd&RAe$+CXppTz;3E zG?zV28NbE)uESR&Pun^*pH4bk^+4~I7N(c#`!+kFH0**ao#O#0>uE9U^!<;oW}fAK z79yF!<;r=Kttc)~1(8ByrtIJhPG_lsEm8@amD^}c@c73XeRD*&>jGKZg}&Y=Gan%X zXcWjO>}sZzlhXLbQ*tWR*o~MF#OlYsD%6pr#2HLUg4%D*Goh|#%U`#ueWv+pbaK!W zRdP}2MVJq}1~0D4L1zE{l@m-39|7l zUiVSXZD=r3QZtucclB8ZI=Wn|9{Gyx90N*fS$sM)QRq3)^j}`G`MXZ*`4H!# z^N$0G)QJrSrIdOg?o>FWufj=~C4sG)Rhn%O{e;TYiYVn%>%NK=-4)ldCtoGL2FpE( zHRnfY0z6-5CC|)z*E~vc_ia`%kK26`?(s$(PM!!D4l5dbY#DeI&Il@irVFL2PF-U# z@aCaWrulDcp9T5q61pReiRNFUTGp)-bf;b8wbI_6Ei=ansrF||K@D2d4Rp{ZG?M+x zrSQqp251RGc}r}L3+CHC>2%Rdwi!)XI=cx9L+4v^xMQ7R_CUPSo+h0JyMZ_7!2qN3Z%U4@g<|qeJtMT@0ADwx~vXvM0jr zDF`{9$hy}-4fZI0gym5B0%GLY8b18>e>hx$-Wl!umwST`O~+Cux)nhhN9<2jHhOmRtv~C;lfgMEm=&$oxa_}ZXg$tFY2&kLH#l9kEI&Rz>%pOl!+T5Z z(Jr@BhFr?xaSILdV79@{+TANL-Fvf2WI#2WFY+en-J-z{mo4{6pfP=$esWE#CdReG zl;aD}^UVZk{*!jUZN+LCe}%z-5lZEAtX=(YqgX^0tN3f)x1MaoTboJ11W5VeviGb9 z&~v6A%>8LYMm4Ud=IIwv(nGR{)OYesZGWqP4(EaB^@Kgj<@_fp*g%X%^Js|+TmkLJ zM+k8j4KU2e7a9lOUCs+ttx)U@IR_SZ&R#fnL0K?TOtH>y7%(?O&rJOdZf3E`=5B}d zHDn4Z>pjGCc)dpfnYQwoe2fl(q2TNyy;dhqj~tR`2%w)ua^O9LWet|iaq$%lFQz?& zAYp!=3ucw~ov#%FqN@ns^zLwg(}0e@oLcJ@`n%{TPJW_`Xs#Iy{ZHdY1V4YCh=$#( z{35=6H0lG}m_vY0TXs1bmD*BZ-3OzxlyERFFQiK#Hq>~1ZuEy&eN7)1IDfVWucG6~ z!2eA*B)qTw-bKTQ@mscko|>IrOUmd}-u55& z4x0`A5yR@U&V{OcHTZYg*1eXq@NB3_mL$HsppORzzP__vlTE1S{j!lpG={80FY;FX z359nrf!okOYR1qF4guxpC?KbTY)4mCYbO6dmyXq!|FN99S)Sde-MzbfGfd+*=XI^s z>#P}Z-*J6i4hqhGCeQ%U?saZ=^A+m4mbV|vokJf3c$hi7{%~=CK1RSJ)XMDbEp=*h zY|s0_p*bJ-nMW_F>kN*sel6r?{|)Gb5Z9~IzU}xLBCt1)?j*Xgc;YKm_JW9pFA|I> z0G>38Z5P}X4d-qesf53M$H{R<=kLb1IO#tUXu+F#S-V z8cTw5KAT-R*M9N_y|G!#L^^AHU7uD;njk22H&<4w-IVFWR#y3?;c2bQ@uWX;?*%}~ zL7&CL`f@5ieGJ=M_Z{%$yH=`@p7yP0*V3ZdC~KuAi#W2Cm|_tsF0W{z<)xA)lrHM* ziO<(}-ZpDJWSu@%>6$7e#Uct4%xFtiEm1a7cRP4ggYpAae~~mk_ylTv?Bd>U+^Q9m zudMJ5GP)fZy(1PNJPoqd6`I}r2035FEKy~rnR;V%Ft*>u3d(xw*s!?>!DV`!3$s2E z>tRdR$^G(AHX#1S^|Ghja=C$qcpkza6>3JS6QxY%a1L|aOC|UHalmRO;S-daIlHbR zN$fV_oe>z&=(Qv@TX>D%X4)HvZidJ^q+XFcYo+FFdz~rDin++2rVn&m+WbVLvuBCB z+!k)GQu7J2-DFkSGHO4Q9|zBiof#&azV#hQ>i^{2QI^f}wT+FzdYjZQ7VoKqK^HJd zFm{wGFQ{VRU(j5us4zZNqB1|d4zP<8zKS2t;g!F8NkA*7Ucv4kFu`)@Ynh*cyk9v} zwzn|G8!SBaT>5>yeM-3#k~5-;v|SC=9m>j(T8=xH`$+IwJLU$w;mHh}`5=vg@NMxd zq@x~*UZurAv(UL!*QEDogx_suu^k9*H*4?P;LJO+dV~Mzc@G5!(Q0Y?^P^zWZ&Y8# z(0v91*Z3~>Cbgrmy~Y_J5M}|Jp;S+)lW)j2eH8{VFhnW&fXCi0+y8d^Fv;VoZ1H|R zJHzzG%g*cDbv`gZf760wfo>5+C14G?6MEd59lB_8_)fcS0U6F_tnrz0HLEfJV0;dED@>{@t)UJfgB|D7ou$$C+3bi}d_9ARXk7s_vR9C92a^GP&4)`|wbASILy{ppv#Ys;ke_D4>Ju7+A_x|!YFOUPdTZpGnGAb!Wdr_s; zF-p3;(5Z8B{eZx6z}+AKxxiXZPjl^4$mUA#3?r_8=Oo~4$zibjs%O|4xzqS`aeJf& z6%9W|+eQ7y2|^;uanoc^f%@fO0-2}sxy|K69LR?VYeH#c!HqkXI+A2CnaOZYgh#LD z2%$u`wDolTeEu1?1X5tP(xf%R@kM}_s+M=YOr9c&@M_G#2nrhd;AMKf^VSdg%V8B| zXp)=~nzlTv3nN-Lyu7>2*hb~N{>dstZ&YmQb?xBBB9rwR!R$vnmTQ-~VFot-IoUmUdwv5G278Ah>IA2<{LhXmA|}A$V|i zm*5f{26qeY9^7pPcXt`w&SXE&*(bl9?6uwx@czS^VfB5tRCQH#^>x);8^Rp*rMaOx zenSY}EYB^QNP={|&LV>+#N|g?#g3mcZ8f>SGswL0Z|om~1DekX`V zQ2f0pL5SK?%P#eaVSnoQv7jC^V0BLSA>hoYwdk?)Z2ZJ#%iVM4tm^c3D48m@n8*lY zYyRXRVvmgoEBOU$e&%CVtX%gVqqbAu%VGUm$*z(XO}Un3oSg?`?M&jtmZn-8^IFm3 z=i0?uH#{QPUo_hE4PGh_z453HzCR9E-&iF__!8@y7V$Ci0}}s9xKPZgXccr@PE?zI~UF zC$22UI6{z=?N(qO-R^4eB|eHxChf`V)vDOkwQLninIg<{L0)w64$p24pXhdx*!oX& zt?b^o9-1i?2zz2DaV5XixL$x$R{8AUsvnWUJBS3V)q}=e&TY4qCzUl1n@}2-3-f1} zCf8)O=_}sf6Vs$yNTh^S$7AF8S`A=+=7h80vR$FCC8yqQQrQefa`>6cjK{9_WHH9x zf^}hO++BxmL=)w*Dd)L2j}Oz3imllW;A*l%3dHRz&4bS)MXVjsGWcQEmx_L4y+S#R zG&ps`+RS@()(R+ST&IqR&0pM-HF`24MeGG7R#~;4#kjM%#1DZD>?gsXu|eKgp3ML>-KtpOoN$lC583klt>QhfW>QAIL1AYI`Vl5_PA znY99ny}VVyo!vGP*KQ&ER-@=4v0F~?P$J8M!};3kX&Md@2mC4Gg<1o-0rfBvrkNS~ zOMQ#}1ZBWot;w8lhoqh2BqsPsuk;W z4EZ5;rbbJmla$F!Wfw&UF7u)WP?55ADCJ6%og2B6tNkp_gVT#3)Ep*}*%owcg;XgJ zT$vG)c(lJhj3!4oG}0{0N~%eo)@7-}l`LFS_z+Q|IYa`4-Ltd+RKF0o_9j$jwI2v$ zmYqN3b8^GpS;^ISy3m@)H`ktTDN$iPSyTfIvGv$(%;;@B_+a;S7gWC8Dqs-=MAx4{ZtQ)|G+L6@*)&T z84N3$b5_-eAmweVCI`kj1nhR1NafbQhCdv%T_1iAEFX4{L?q zWI@kRbOBj2zBTV6Qq>&+8I$Q=H^bA_#V7#afNkBSQ#Y{}IcjrE>YdS0lp4@sqm*xL zQ*M*j8rSILMG(7bj&ZY)Mh1(BiWbyBuYeA)wypU$V$GK7fiFSm8IdaOHkOhT3eE|d zVwRPCEcNj7Kh`$(o-bsozOJ+E$X2a-Ugpr3wmBTtR<6{bWSLSTNuWmljxFka4%lz~ z(L$0=ptJDMRrZZnQx6q`Xmh}PHOF?v8zz9EdQ)E84q_9>2BAF)i3U0|zh2-SWb+Mu zBPSp9i~;hlBYT;--d#h^_7Yr~8Iq0xUiknG%&HWhs=Bp)xUa0RHu3(1#3dBhm-=lR z?@iGG_EWvwQ;)fj+ME@54j-s5

qS0$n;4>AwwO}W!|fhT3w}|OfEYV7_2WDObLuvZtK%g&)-m4Jmq)GD+GRa2W*O7R8OKtKaG$FD|3rl+263Q&N&eQvDDZ!QnOZ z(Fd!cGZk)$xedW1G5fG;Z+6S{B%^bXu7`WRz3OC=M=wbg$nWk0VHM)KQ zQ2gbJS-9R$$32pGy?0jdKl#Ik>1r1mUfT=k5Fv~T!(ZGZpsrSlqX-tnWjH3OjTFg^ z5E=r(aN(2Y`Y1?>@_sZq9tt$_sZw)}4}A@@LtWX^bN!!~-!n*lFFa(gHa$$x9qU9} z-zZYkx^;i#k`R{2)ibsU=2C^ZGM48=ce`2WBsmvFILBiafjra4CoUe@`%$mna(Gwb zc_T6+q!5avJ^y9`Mnh~#^mnAOFa$F(Q}GQ(Z^`P-bJ5Yd+m|p_C8d0h{SX_mO+4lX z(235upX-+-5=i7dl_UrLnz{#P&GNF9c_XQLpMZ|?3jxxrIX8>Olx^IN4$nLdMT%+{ zH>0XM-u$LZcpi1Ei8X6Q!(hSwSiYbGPyxBSaqTF*vr*#tCV1<|#UAL=&3PYQsV^HQ z+b+LzZv$?=aMTfWY4VnO#?SLoECko8mSX1gOHfGBNf^XPGV=K@gPMfrR|6_bORTGP2=Jk4$zbg@zbD6}w&K|eLdh$L`f?ccs93F*^HEA4Zyuxauvzb`o9t;2&FDkJ#*fE| zfU{*2DE+BBIlRn>qgd5(*xL5=`&H1ZsGWHkh&gKXgp%IgV(`9u`SB|st6rG?;{d3gMcv+PH zh&^wSpqIuIj0)OjA9AM!(^<=Pl8q2oi&_vPVhqoZwkQM7#e0vQ88(DCbCzOAyfnsq zA4Z1KtqC&%9R2~jm}8xJe@g)oJ!cTmJttj{lSoJ22Nf|LYIDSI^vBF@>E>VZl?>w@ z)+oZOukm0DMNlLx$zEht!7bh%*WsqED#{^j?B7)gKwuf;VF%OLg8hsO}pZ&^ZsyEMQ z*H+935!eyVx9u((@4zjV6_1Wxvvywdnr+Hfx?jOoOD9c{ZQ&^F`-|7{7V^YiHzzs#NA;dAN27S6sV@dkMa@IcN3z5b<$)-- z7}X79{8x7p2weHImm>}%H{$qPdssUw-6l^MM=A+>(3`v6ua;XDR3HojUv*N3c^r6+ zTwVKucmBTkEdH$X@Ff~OZX46jk5jIY<5!1+2*dm0)IWNi-T=f5QcOgG`raXpfvS!m zv*(uUrw)Yf&ncBw5%q{qa$y???~{}%Ltf@RMr}(8dNSQ=A|HX#E7GG{Sp{t@@ z@$B}i?Sg2v?G_2Eq&I|cwO^M_Z3WAxQ_W+I-OFykXo>g7BOd(QBx5F=xLGq<6Hzi& zP?S`q#)|=mwYsAe4)-o1;;Sw*!w3YyR}8G8Y4{Wa1%;nz|3N+X}M)(_wzP;6lf~HuN>p|rAz??xumu13Xz0GUbL#AhYl zIhkg87h|F`hiY;m-%#S^y9lNe$Y}6wl{!8qX8s6jI=M^}vHaP3c~$2b@L2Tu5Xvd+ z_XegNWF?=FZRhizs;V{dxA4lnHJ09qWRCij0aVlSwsnM z?{_(7Qj^s@kL>-bdAzk%xz8%@^D6i!tv=f6fuh_9ikrsOjpEbCPR@txs^T#p&CI7v z+n7>MGq8*Ns-qGHQa<^hd8jCnEtn#SKYA>SfZzV`=Lz6ffsw$y zjg7eXFsYIgQf}6KKQs53>g&*0eZ+%cLGX{dKw0T9!0_{}5sMI_jKnvdCB0V`!FEyz z)d$52VjGz~v`-By550hGJgjkJ%jSZ zm0pVSXr2tCB-L3LHW%y$e*A$p!pBk6AoRwCKyth{1lI>`xh>kUKV|lKOPmDxRyTqZ zeV-lnYkE4N4K3yBFtMBL-Hm(e{rPd5CsADU;h=+}45|KNJ7`G} z)7grIb_B5LHA05xq$88fk*S<(x{#i(`fL6jt^IPIwiU&->Q=wrdYtt36w*LcWKt3p zpPzI73PN`Dy52v5?1JgWTtSIj4$gQZ;L53yPQ-H@MlLF#*a6lGiZetL%EJZ7`h2Rk zIbRIPtxZeg?auV;C~I|0X)}E*Cw;r}%th`sQoLVaq?0*Xv=QW|X_*7aaJGgV#VD1W zR3}>i)rH`r^!(h6zQ-3Rk6KsnM6~+yA|}oe#OO4``M@!N7A_UGW;0S}6A3xg0uGb2xO?d@k_VyC{ejeSoCXlzc||}%VR03V(b7wKsTZ&~pYcjU zT&Y}fR!%_JMIBlxZ09d{wrKCYFJ9*l+dvCb2&U~va<+aU{XRSA?lH=Dc+hapW-tP` z{kHA~nqEmu1w~nzx1O2K9RA~c_?`royM{FPn@j723F=J0vdHqH?N;C31>aulvmjYa z#L1ac5|9G$v4Q)XdVbiMxC?#nhu!vZUP{G$nlhGXmy^LpkQj%u)usM1860!#W--Wh zp75lx&QD;|$ec-VV_j^39IO6Z)w^TXW<1Ex-uK&N8^-;|BU4%kx06VefseA6@HN8$ z!Nmff%U;;Fr9_IgU^zDd|D#mYdXM^BT3uxtY^&#y?S}MV6co;htVrRt9X^!<-dCL} z>gDsa&{Q|uGe4<1v%@Qi07*f%U>uB+6PTA`z!LO%PWJQ5Hi-*8_T1Rj_!sE8T+616 z9L4HvR`31E$`-G)O+%z(8J9|}oI=Mwhd6UgtJ*vm|{T7gK?N3-9CezruwZW^eT z^}&k26(cq>(bM=sbM!GSfCgl_!0-?lhPfePa-v2wnnxDT=Q;L1x%S9R6h7aBQ{hn! zEsj1uIu`%-Di;L4PIGCK9FT_Pl@^jnHiwhgZtVu|l zCtMc<9=<5{7apXk3741F+=RBpvF#13iMC8K(VF`+GHafuEfSheN^7AI5?h}@-q%|~ zDdy6zJ=%;$Qa`Af&_Sr^L5&fr!-{dWnQHEUtOg84O~>lS-E;vbd3t66<_{MMZ6tLM z4E0=OTb%>uBdF$3TIA)Mx#~by7APF7hiW8KpojwTGZgqSh0+vR{G+j}8N6&Zp{84` z+3inVuKCn5qzx%8P8qRSvKuzY%WTcReMxKq8dd^7v_0LZ514yxKRs$2jRGk1Rddb_ zR}DiSNfVD|-M}dqg}uiiqpcCW7r$RkDaye3PJj-Za!1=S^FGtHzL~?K6@M2*2+WM` zz2m!}rPsUE1b;dSn=Y|%0$18?%}?*Y>$IrscX>FxH?_o+7b-GnZxup+LEd!IngriV=LaidV51MAF~BM6_tTNAbNLdV#XV#~tG5 zOmFth(1}0!UgAh9qK8Tpwggc6wV~;rGr7^kAtDv(_iuqOS+N zWZKB+dd93PFB8x3O;!<|#@IR0&dq+5BIjZ~vXLNCBskK!idnB_e| z2R7g;`V*r3(5dfv;cPIb$!Uraarqm}OuJ=LmfNS3dWX(8rTa6M(sIPEU?su;&|*_7 zo9}`(*?Z7mL>r9FQde%B>VG5rhQV*+0V;e0NW)i^z0%O#=o3;`1r$Eu@fVTi2`{ue zEHyg>^>YQGid4RPb_v>G*9dDPO#z>a zpuy3gh_47->ZVv)%IT`w>EYBB7bt_flsGAPoYsu3KL2FE05!iLtyWG3?f#7pb zSdcVf=+@4VV0V%xcajqKK9Bsuj{EofVOT0L)Ll5HWn z3oioJ!CXB7IejF$r&JK@eLer>NB!A+{e*d^V@xdiCLwi4W!a8zC*i!z#l17Hl%vX= zZ<4oy9vE&@8%_5em093MT`ZEh-YuJ^-MOSl;-~oC0-Ss$dZwJSzhO<1xY%v`Dt9^# zFFG!8GutcDq~1;BT-LfJf?7(^&;hF2?d;n9`RVqd&F5gu2!X>`GNmS6cJ>iXJ%%XT z(fBACb>63P!y=mozW|1@s`Ep|Y69z37b9W@0ky^hK7Q3}x>I4Ey`}~f< z^~nmcXP(xMM+9-_(|RSO^f*IN#6;Qy=bJ!|fxEk7(Cd5AXlCxDyt?%+ai_;2o8+pZ zu%cTLl97gY8>m3O?B`b~9i!SLtss%aWI@YBZDLsdU@?YdUZWp4SV$-WkXj-Bp}a^E zGZdMb!)sw5QmkMR7y{NGO;|{dy|s6w_O)i4wru7Tz0?(j0Fvp++j6iYQ72u@0~Dr^ zz#Z}&TI|fJ=A6?nq@x??JXEv0pX(GL>3`|T>lln2`X2i;?ns?&09&A8*85DacivEN z8qoeC(hV*vBcJSPM&+S;QLo#G8Xv-_3Y;t z-{a*7>Ou2(8U_)nK%VjSgu-`{REk2M;nlfo6%k=^5y?-v7;4Wq6MJnoK4p>uvAFUI~o_FhYF%)BBANycqAJK{GZJCJ1OMiwl^O_ z**$shWPB1;4o)S9uIyC;FWzzMx(hl~pPn?jZx)s4RCV0bdR+JEQ45U!jDE;!%TkX)qFTUBrFTR;huq&Lrct6iEf919-wALfB0z{){ zsx@Pc)XhCBG@&eYsxSwHI51;&(#l(w8UCc(m_rti2M-stEmq%FL1Fbqly!A4g+iP2 zVo!R%+@)k8@Z(~_9w(=P*urBTww?#Z)LZ)l+w?!|R#=MC6 zKX)3EX-Vd0&Yg-l*o_v&rDj!`pxDxrg6C@Xxu$9a(g!X0tL?g-Pl2sA=kbpN3El2Y z{Gq--aWYILX|3pIq`-d>X z&qM-_$lvg*tTLpnNEECx(UOt0KG(J+5(&bH=v;FG??2rhxRj+Ty48GV@8pv)WAoUc z)vkI(jHR7eD=0oHt4<{Iww`A+$HDhOqi8Re$D;bYrFBxkD)XuRL}3WV`pIi5TVNTH z)s4s5{{L|weqrK$Xvv{){E+q@kfoam1JB!K(po|?FKA`3w-o%(Lv_BSX>nnygyeJ+ z`UnHs3@rWm*^u8e`n23VPJO(XI@8%*^GyXRm<2HWo|pX>wdAMdFFM$|>V=(rUon@} z77eusO^h&*+huMVzabyRJZltMsVE=4%N@hfXB7&}6Ar^@-^L8-9s8ZV(75MM4B*CbE zGIX-igrW424N$mtUY0RiT7I;!{cB+kP23}KtxxbDXM7ydUrQBEOP_D`i`yI#w<0?4 z{cggbpHD-Di?I5KkOB41RGw+Lqtqi9KdyjKy1-Dofpx}zC-y|kky;dC#8? z&5|I)zi(f1yVQMYlJC}xiO}eF|9oZlcfk1n5Ho%Gn5==oOq`*QzZnv^^SwEC&fLOMn0!noJT7hcCT*;HIEM7%5H72Pg@AUE{?FK z@yU^jS2^46x9n+p8bc1Tw^8^p_${N-P+Nwe;hA{QVP=9-o-@)303D?IH^g}JhiAiB($kz?s=(wBP z_AM)~ER-e|l8*S7;@CcYR4|H^P%QHqlr-_7>>!CsaqYeBiglDXv@*PQynPHAK!%kMSHIbo`5O6ltybpM;~1 zO(3hnpG6U&TQ1lg4$7p6*4`5^KHR$?u-&*e<#9XGYYI_*75om47Re{o4#~VGUL#=j znsjWJ=gXvBX9{w;_U-NHY2$1D^4A6$wOgsZyrYJ7j*yHh?!VRd^>G%3>6FTPkGy$l zVS@bGKUVa{w*O(ad~o(0%BZkd1!II#=G!9=LG$ zEN_^fJZ{J0B3GxigBecd%>n{I-2aEY{kOI*G{KX87iP(X?nI0X?l+zTkc<~Xd`Ev{ z22}vPr%UQV*B!5TuJ&{#$=+XdoT^x*<)Bvu9#GeTkbkEH)HKpP+7kMg`KPI9EQ zq_7oe%J|6s2{1==^aHHRI?i-Ui;-}P0G!mn2+GG<4n|Zu>c^{cg;*L%IGYHVXXEP& zRO{Y<8kYarHfyh;?kxBt=F#=#iOZ?5L|4ug~6~u&671PV8M+5#_i;-U~wwdq{fAfIYKc{*4`bXQTGyFR69xssNWLP7@QAR)b-d2v~0 z?xG1kD%x)Z0p_*9jPU?~+x&FR4|7ROYy|PBB{BiBmMh>1&@L!xi1Ia3VeHWHz0mrC z+fhrt`|-}>d2>FgHw+xo?|<5*UXcoO#Vk>Kh`_+YV+jBA53VB|{7&E^eRjt`9{S&R zP<&|jpmfL+x|^5-QRi+>D`_fI)HsAs%uJD&GFm)nA{dhWXLm4gXv`}|tJF)n#B0af zj2*DBuZZz!k1Q?GmXB7agyHPdmeedf{5EjnrSugI^XXH`|9ry;O$H*v^qzu6HkIxx zB5a0=am=;j{dITJXJZC%_G$&Dvo|p*dS?B|$d?IGq+ZA|2cc; zM7RTlWn_w{xHvzk%B88OOiPO!mSPr6`lWm}=%k_(?4cukUKz`z7NAX{Z78$tzHhg0 z&;m>o1TSwzMwslfMp|;zTmc7n66OHhz2cn#kQG)eZGN47byG!bV?B8Rzlk4J7m+o0YqHeq!|01cl~W!S|g9GrVtK{+w2r073I z`7g=t=n6*`OjcoCnIX~o_xLo!H{j)N(F+Xt&hgMxcUFGPKgNiU&VVh>`TOZb){T{g+DPP{|q} zY5V@7?r5sQ|LTaw?qKudpf%_&jY${`RqX)rxSqR+@Ca+rm3$lKAJg*je8bYKWq%;v z`$>F;&nM zZn*$uAH-ml>rbsmA|3OB+e!@Oc|Pey6}b?T$|S6=4K`_(0|5ydgv?w6|4=b8noki8 zi3TiU;FwkAs!4<${1mOQy7m2G2x#@3Jva9c$+GCc53#gNbPg(xhf4_{8 z$7}037ogm5sYfgUwZiu17A~WltSlilV_wpqy?Warv-plQEOE7n|0#M5YZh2Id$dbY z6STkd1bq}E!>;T~&gxGo!om~Edc&mowO}Os{jrotq$@g5OPIRvF{%9*X~H5xCjihP zIuIBmr1t@V)ac$a#h;y_6qc36&q@xxQczGR=Hq_<=Wrz=pvw|{&J^UKFgG{1VWsc{ zSaS3FrUIVQxUDklw*3|oQk`n|7i9>egie!U*8=Ul_5Lo+X>(IukIR+SW#WAjuR|sL zbNG<-^ktS~ykW^d`)qteDhzHJjsncqS*34mZ+~#rN`A-5fq$6DcO`|4f&*gPBHJZ0uqkpW*$q~c2-kb*a_sD67rI|}cMnqF= zk3`4HbL-0UU2KTa25A3fwO5(jvB0ZOXhDYPzP`Sc{QR=~0s^^3MbZNUgSlCiN^|q` zL55x?@BZ9@W7@D1V;5=>+qaY_5WVzwKyXu?XT#Tq1-E6tO258v+}mMl9B_=8RbU*KPNi_)p+COfCUj8 zZdD63eR-aShApwyzf261S=ih1XWQE9cn7owkp{k2At;ODFG@Q?F6=#`xm4|UrL5pU z^|uDJ?bMiP5RkUEc5X#Q$3#|7PhP}&pljj9jx(UO)pKo+H)gGvDZbVAPc1>>jun>t z*_-5gnBl=l#}t$9rE6a2kan)XeTsE@byekdY&~ReXBXG*4bAlwMi&hftTF5eU=`!C2xtKyp=X7Nizl* zO_po8sidmP2n+kz@-{N)FHf$LLS@I@Ohse9K2rfDuT%ax$#y*W8Olz)B{q-0m|_fT z4XAE9xN-~+4=ZUZj?>lMO$PT#+1QM0<4673A2e3xwujo_w4sH?MZ?26j+prPxm{`S zWOf9v1bKE=qb@1ppYb%*@e`x4bl1H84&AJ&o;^u1-@h;Hh8z~6jSMI8fjMK@{~R^c zf}<%yJrmInKZISRky#On7k_zihC0+gHR|s(Nl99l5wKMLsok*fu2jO_M=)R0{OsYk z(Y(8<{?eC^F7l5LKSgGSL>j{&_8+7LPa5qJ=|Tqobi+`6v&Z}d4|o$(hR6G>)8ZFA ze{1FO%iTDMN&gWx{hCAu7&>ZKxn94&J&6PXc*6jeQR!g+GG7=U-4_@c`y52*e>ovQ zV)Y#Z z<>}@1_4N=Zj_>*Kp4V>`CTdmja6*|rMD>lg;v_t^0I8=phkI_;=4rIgcrZRJYf=@r z_Oa6%5fjl(ZUt&9pU5ohvg-NA1+3DG$ntIsY)SD{8L&5-{#-C!fq1iZk+~ zdTW`aJSJ}65pTP#=~fmmon^pw7!!^2a42T^=>m{VC-Z2Na(nETK=ID}g$2_CG@aTqfv}6K=?HaB@l=Ewxl~IhCuZ zC&5W<>n8Vp2k-vBO?>~RijNFhk;)W zQo#G_Pdn$`egf5P=y+K44>Xck?c(d3=38G8qeIVK8ZBIYo^jFo5*xyW>^gR9@4sCUNbb zn2|%z;z99_{R_%ert`%=dza;2QNzoKp84cc*3})xLI80}iSwkYxpHeqtK_W6kFQ=I z)2xmQIud9rjgi(&55L0Nv(Q;?%+q*}TEzd=(cfR(RYnX@{8dlyZMPX}OAcKJ+v*8M z(`F)jF?})>SCyLWWU=A0V03+s@Hh3t%NHjs%WR*PhpDJ0lx8cAvedu$*CQgJYIrw_%E-8z}JPLf+uE{CUa?IHrD68hDD=mqxSIo5i zssD|Q!$meSIa&MT+hWdx_>u*sr0-bty~Ak?BCv?diP>eXA8L>KqYwA8_tR1d>Dg(h zq;zC^14*`?zpJYrh+mxT*~L_Ty}3U<|K3W2)m%D@LFJ;MsbL@+ z&i3?6TICEDF`wtO*4FHc$FUoD`s>`Jd{{d)zboH+b8OtNF!QvPYa+^a#rIp|T$_udURXq*4pZ;l1vZzx8Z;x) zxoXa-1{;{xPu~c}7%yp=)rIi{(e&>tERa@=XT}?sphM=qgBPzVo9|}L_P>6>C1SD? zayW;5=AQ0eC1UAu?Lb#vROO;Jn%Z`rT(wzdKB_jk!{_;fDp%EZEiMxG>hU&2d6VSn zcH(S-@yRK9NJ*sWscD@@W@&l(Ry4lNA>AM0M*9qMyx*MinC19c4wL9)Kh=KmO4kc` zf3(LwQ`5+fw*9*|U|B=>`m{%Kx=7SYR8qoF_fC*&=bqyKqOcfh=VCg$pN?DD?#jLX(qXDLKaZAN zw611Oq>E+}effH6MYjB<8Vdm=|RTza2rdWxg3Ka?ac z&1@=FUUEb#K1sPp+F2^&puaD6nm{>>4MqBGBkI`VU7gcr*XI7lLhN>5J*#4mUAkwJ zV&zm{bqcYwT&vf%NcO?VQ~gsC`bH90JpOEuLKv4sJHqW9F2#x+Z<{sucanUt~hGtS}b7`Y-87UVE>Ie)rYm5GG$(cCT08k5i+F3(XP9p~tNYywG> z+6cdAJpX-4sNs+=K~7n#CRz%&M62tAuEB=3it`Uvwv)RWJ_|T|*Y``q>cF?)f((HB zS1{cJiRVQXL+gHemgLo}`JG>#4z1Juo_)5!cy#R(>iu=PleD%uSkVQ5ukidIzH}u6 z&Wzz%|2P853zQsac?*-RYSW-9KXaj0mM39!gEX!dPzsugX8uR)lZl(f(r3mV<@pK4 zI@WTPRNVRHH`fh=iuqw|mV<*ub*+oG?q>+dj(&Exfql3n92OwReS8hoFSFKDP_s6q zY;4f$0G|SN=V|uo^svm`}UE8tyNQVGmqtuM0Yo?c(CL|loJ%aC9%Mm!t;yX`AJW2Mhe0q3wOV^>XP4L?ggci?553V$t$?0V4 z#oS^h{v~Se{NpBRgf_T)+5J~XQmhCUrRm@fK1Dg9-GaiiJA#oK=m78JwOJN%hw~#H zFvfK!_NR7pz-cM(m_Xb>4xSFAk=x}?G<*L?0^ZDy~ z#`_%m&8jSNG%Ys%Jh@-XKc;9`6Zq7-&?NejZ<%jd5jm%nf~N0QL;3q<3*Q-E-8s#4 zBCMrEeWC+^{K{Q%CwXG(1dq;N83i5-R z=Dv^*;iiFkT<{rZ{#^?|O-qq}zc!CxRLAksmuWPay{zThS+j~xHN0~$MvcQ)1#zuJ z`O~M~=U9(#X9|_0;=Z+&(+O(;CO1y^H7-%oq5clzRc|`;JaSdK;ZdetsATT(GDs z)@XUDnCWNqvw^^InQ623+La(+zv70mRu*`ACg*Ri|INAs9Dp6sxBFqGL5EN`Aq%z#H>nI%* zSQ{ivU(B;@99wx5I+7+#FYpj? zXoO-Zm0@ZScqVVYlDmzi71Db8n6R0LsHb?`ww^X9IB7N1ye%F(7pGP(&c!;;>AklSG=E!z!pvC zdsX42K!ND$2cOFN#IlsYnWna;W^eC^=&y^525DAkFZY^RYVZQ?9+yKe1kxh4wRn9S zx6DBld;FJA@)r5SBqD}H`0o1H!+zB^2{FWmSit7g5Mi;1QhSKK(e##s=7b)AxfML`UQtnv0To-F2fo*`8 zDW)R7B*=4|t(66U|`X{$K*RO4w%pjyPr6aH%Bknbh+2>_6guXclm!L-%w` zj(|!?NI_2BK9O*j**K zP!EqW$8NFlbC3>+d?$ZFjfTe~9>YyNaC_!WL=(=6@Z15xjqS{^Rg2x~N6gaK&&IB0 zii^Ipv4?8f%VU(LJb4#42TNm-NeTkz=!IoPmEWDW)9OMa>mD2^8M$o!nPYqjt$5ri z@=OC+{7UxId~In=Y-}kA!!8c`;|kKw&hJNwH_G(sJ^^hLYr7sJ;_#Y^DGiGs%smcI zu1(2a5gH|%H9#IpT5-Vn+xyCsIU6Zf4C#R_dBXDPC`6q>`3fIc%&wFPXr1ewzOLz8 zPMG!%VYFN(F9ofCOAzEVo__%Ols-y7dM?@a*szDM0bSPZI0mdv@@5(f#{+eV^8#L~ zTP+yW$uKwH)l5yJk8EL(c;0_&emJ^|SE!l`4*F+$_hl~hD4#i1SHwqU82Ke91%#i% zViPlm+o^75Hs!nPD0qQ-=_FbGHljQqaU;3 z)9$50tu4)q8_mTrVOgG!Wgi7yDF9l@7JeeGd;6CchGz?6+Fu5fJJKR%NjwM(*jk@l z67Fv^!G`M*Lrkh08*zO?OIk)7qp7^0qp8*4AHogWjdo#fOLzA)4eQ^$J}6h2UMbo6 zAeh;w=|;*OW6Y=Tp3XBX?rXV98Ce$UINnvo=;@5-UR$_TZpI6>lA#MWT*B3gsNU+2^b>T6fOcRNfZ57t&h9B#nMFH&h~p8Vy2k+pGSd!J5W$;G8^vFUQ+_2#z4?%Ark z(4fq*2mLW4fI_*q_rrUWjGQl9GDRq#b{!5Z0nG*TY_4|tLpw+=Tj=ToRwN#SGee)Z z=9&yA>T&Uk^JZ}pR14cRGqDQj8hHWvoHje7kX~)(m{`x7V3p~q!4kXUIJ$ltHpS@{ z2Ze}VM;f|bS@yhV=snb!yw_jTbO22e_SLN;tiOjmq%335+}Lxi#52qtiT2*nf1M6) zM$g?S9o%wN;yDHzk3*-hjNg1CAt6;?=9tFq01dA{#CYsptQ+=3ubv$VT@`ri_>H6O z`%*}_42IoA$FKmyrH7cl*(g@-%;24M-7dzcYgoDufgxAlB<14nEP3^)sU1nC?49i5 zxE}5OQa#o%&w|=o=o^YsNx1A~A6|Z(RD6@prPrfbN?;6-)QeyiSU2H0JNeRZG0tTe zx7XE^rrFc0V`TE9Ew$FVQOCANaSK`mKydBHdzY(1?Ql%LSu^%vqC&Uu$nv@%ooW*x z|J91B?ZZPkLC1>w0g3r-sF1YteIMN#RpGsv7yN@gm>IGptjnoO& z34$gKFB%>>z3gE=qRUwq7nU-sEUz6K&j~1zpo`xg<(Ep-oEy#R6}Z#Ry^C_#f?no! z(@;~Rec|}NDCx6y|C~;Z0@UoTV<@YX0sZYDzcl=PZ7QLB`M1--t?eJtk;6|ENpMDv zj|8e~>IhO8v5GK_T~%fZ`Y^tc5HV$wqj?x4R2FGvm&BZzeN#`73sbJDU=}S>Gkj5*kH|i7Vj4q?{QJH zaGx?Cz6Y6_M^9*`kEM|#EZHmkJ#U|n$T67w`f0?* zehydABOu1v_&?OW^YA> z(mgUTG|YE<;{7_D&-n+wzsW<`d+j}I?sc#0dfxXEDr_lEnJeL7^hGsJv3kuSFUU%?>-NCvhg~VgL=dv}T@s|F}jHd~#6RKc)d8mfe_46gC`seQk z1c=@RyzWmhVe&s#w>vsnd9`Ze<80y9!&)vW6cMs7H|S_duYJ-RF*j88yuW{!|KmcP zps!G<>Y~@B+2lil7ny17^DUz1|FjhhZv$>RkqC2K`5*SB3-e>V=QNVT>CnrEde`ZS zU@5@(6a~r%u8r{T?S>_w2V6Y5wg&$_tmkim{}cVBIQgG{_&95s>-yUEmyi1OF*Y!= zw?4)H0f#&m28`AAzi)6|9}ZRuxUukfNtNSTRQUTAw>M56{&j=DhyLe}EFXZd5&Sr_ z_@8g*|5Dpm!aWZE_rpK$7f6N&m|Y)E)FQ55?yvjZDFXL<8gD z!-pzrdJLM&2y!(;3Wo-;Njo&MsG9^d{@F6m==BNb_?1mQ`C&k-(3f&NMxR!GWwX?4|Dn z|7TJ*35lr4O`9tIsOxWyMecW=*&9YjPoGdbp;lB{8r|J3k*d+u*eGLbTPl~vt?hHk ziy_6y zzjn3Y2`#NH9+x{o>J5;rYRjrGmEuFfXE&SA&b-LE3*fT|KkpK~gA#+t3j!&ef4zI# z+s_ltI$2$))f3cJw6qNG4L3D40dzsbSeEbktQlhQ<>}Gfs>BV44ISwrz`25BoMM7+ zR52YF0^L1ZJo>M1k{&<5gAtzP9G$eXxk`KQo(UzLSNw;M!J#1q1!L8K1y9RF7dtV5 zXJus-^Kcv1SJjjLiZ=jY!2*9WE9vN1Z;HKp7rpxB*0oLSjhpL#8Y{eS_ zSy_f-qz(?GGmVaI6H`+%;^IvoLL^<9k^m3mxgW;v?rx$)OY!E;^;{pg&{JSa%Po3} znMw+mHy5Y;-o7*d#!zf(K1QLqwH-Sb!M(evcvk6~PcrcepP>=j@52s=@?I4vU{~AZ zm)T{0h(acQhQ6hw8Qjjo+?k_}Ey$2v&|w;i*nPssBn-58Qzr_4*^1l>HSZfLO_)El z9$Woh#MwiQ^~oSWGRe!PL^m%lOTV>sfs&zLm$QO4v44pwnVU@x#UVU9cigu(gdf{f zB9qf|(*N_|_4kDZzaK@#rPAYW8ZzS2xq#p+^JNRM;o<*TnUon3NJ=@Yp-CNaikT?b ze185JR;Zll{Qm^1=MR7w^*(zZ$8}?~Y;9#t@K@k{XOIu6UY45O)UPRMOZY>godx1N z4A)};E|`7gfKU4(*Ca&k6@)&yZe|K|goCDYrdQ$W=NhIs7&`^MqQE{P%>2hSacKzhz-78AB= z@WN^NGGyTBp&9bhqA5}G@Bnss#WK(&*gH}l$CWut>kYLD56Bu4cg@+>znJTDnoj#Q zoOFh2HSc*ebI>EnU=~d598R&|8nWuc@@RIJS7LM%q$mBa@Gwkmh^}y3cYb$5Vv$!s zWar}Qno!9y7x1~0%kMTIgU8KyQ?W)|y@S^O?Qf z-ra95YrQgMG5+)I+7Yj8SluYob0l)QB2k*TdwSuxjx7!G1Lk!$R_;YjMe-TmmSGbb z90M0|CKRtmMOEsHaBZ!{;Cn*}$S&_>r>lzC7XHQAS*DQ-zc|XToTyJM0Fox^9rwa#^XcJ{UWki3eD9s?z17XUcqgYPvD zOcX)C8_0K(qoTexh4u7V5p)rK#B@)Q@ERaz%U`f|iEAT-x@o-V;PKrZC!+3siJw)Y za6aKPw!7W`eW2&6P}28uVcngmqij^lp=m9nC^q_~CFp}%0>^;Afoxr0jUa8mP4sg` zuG7vMXUbRKYnwhX9Vbw`^>&knt%fiktd!Ds7ez?g79^LScvJv1)$zN8?L7fG@bIG= zypn>ZD?}ap=~?H?(~}{WWF8@vi}$t#En6M>{wE<;ra!_@agi&#M8Cy#g{D{f!^-8n zm+}!?vJ3k8LH1^xH+j*o6q}SV;cMy7$p!sDXtu*l|Gk*sGW|Njv`?8jYVXYgiYsbl z7Hw}UnRbfh-P>95#qBLhjjEHZ?ChsxZnCm+44t9pucE=uY{gpK>nr(>(c8^eW}~jYl|a_{@gK8ctpg33k3Tps zhu;91h_-B4HRg7^+;+2>l&Z@|m~ZK>4h}rA4RYpH#DYHq#l9*@T>G3McwUCRmhNhn z12L60iV;k-YN;~0#c*mu?RNXQB=w!=H=Fy%%?Dh4A17@%_}IQFQy6YEEyc}(x)v0V z`Al_)`s%tPBq|yRNOn!sW0y%16BGI*N&V?`>+LxRJ$ENHdF)NBUR+OIz6Sixgp?K+ z@%O+I{hKlDWq{b78^D?SSRj^(kwcwIJ|pMZWdrfO$rc@tVxikUM-Dkzl99XfT8os? zE|t7)_B34fW2w?|885sl=XUjB+h651pvHU!g=6)qGG(`1Xy@p~u=ZSD*)QH>O(*3T zM)lUu^hgfi7_3N}%a|p3`;<=AL30ldbv$>iP3~{wIxbpcmnfjDo42zmh;1jfM>lVP zKBhe4Q|fcy(Ry*5s>|5rE$;fTT8HVVGf>~bel6|Px~-iP5U67Im*ZB04n`6VMg%0y z<%C*Et+Vc4U&jJ}(g4!UP55jYkED*yUM>XX`cdO;^(J95wz)8b0T##Beg}HmUU}-4yx}88*nYRFIp@7 zdwPZ2AnC4j!)06wD7C5ca&5cj+6t5A4MAms-^y!Dft$x}+F_x{_DT`P#|S#%^(G;j zf{Ns*)ScTKE(^M_#9l=>o{XpR`}ibIX;o3lR~2<#6Tws9NdE1!v*l^F*KfZm@@ET$ zZaKcd{rR&S@9103>!QNK=;~_hq9WFaoE=B9`F}AhEIGf^o|Emvqoc&Um3`}<=vUe; z=5_IjiBfKEiQ&XHd<&x;Ev&|=1ch#vliLlO#M1PWI*!|#&t8x&7zuGS+t0@QbUIpH z4!0^kdMtw^3g3Ch0jh22>p_gdbm#}UFgNDsK@_hxeyDmb@EQ9inm4-@Pfdcbb|wn> zIff?E^p1Ld4F(E2`vDvQzbk^ugg`UBKoWV1)hui2(Sx@)o+CjnTzDrn;>*g~8l3_2 z`QV1)du)gh`kO3rNw%V&3tk=#hdUQs!10qJzCgxt|Mih({KkBmCZiW$6yH;y7nZCXuQEl_w@T_HgZ(nzOh;5B{|t4*Q}p@c;=1MZau4J#Io(by$J$rzPTRt;db-ZLG9;eFYGMrdv=pb z*};t|Ba*WXQP>dpRjTgqG*m`H0^Ew6CAz&KF)Irey0#VqSo~3x!{*P0dCoixh##1v zYinB``Z}bRRn49}tH6n*Tw(y^2EFb1<0JWMc|l}!Q%|y+OIeX^B6FhY%)Eo(=Wy0G z=o^kMXBJhLaI5e75wi?$5huJ~ij?Ku7plSE3y}oYCgHh?k(*D_pvN1i^)#|{FQl{t zS6FChewE{osX6C6vS-)l#{*3OYI~T=1ygdsUfOs_Ar;9O9sQCT~$!@ma?u<P?qBw=@K>YT z&qdZ>RV*3iegTs8i6A5x09_}&Tqt@7Mv26}Q{H+05Oq{XdOx$qKOSVI99Mb;&YDx$ z6xQ(v??c~Ga1XMtx^13oUuqwU3?P4PRZ7jxxVmeBz9zk&0I!X!(4MXl4HTHq9lft_ zZ+95o9*_Cg)-p{up#1^ykIU^J^T-Ma!1}mJxw^V0(T1FQHdXJf*I9S?)ZJ*;arKw; z3A$CALS-4~AUK$)h)y6g>x$rf<@G@;R42^-APQWR_W9`!mOAywg9N~SCK`dD--cfN zQisU%oKeu%>14E;h$dY?Og%hX`aCr$Ux;5^fUBJHTBwF zBzm=%fYekvUPn=(g0_ImP-;P+i9;RCF=71VV&%knso8GxBP=2b87nc9s!Z=7)S{aR ziKET!ISVWCLZ8lX(4D|2?7JToEj$}5xnDydAv;@ct$nIp&Y&C1D8w+3^u3I$a|i}Q9YGn2fh?gBU@jnwl?eHqFlJ zq$X}=&3~7UqvYrRMwJ84nd)p8X;61<%eeE%@&#-{irk(74IAjmO%r zYd#ni|6xCHYQ|Sk@aH^hH?K=3np!o5aL(-5T(#{gaL;Lxxu;Khy@v=WfY4-rfc+fb zZdOi=%e5ym{f*!Zd$r?p_H?R$L5zoS0jPPpZIkUKzIEM9UDEB%>N7q6R3Y9Qh2&5F zsyMp1lxvaPTF#b&U)D|*x-FLWi7H=;Y}?w~b4mN+ghH|F z`nMb(3f^JOLoc^m>4(Ebr^Smh_^%#3Kt+$B&8ivLL?}D6?q6BoUK+hR zHWKFZ4*160y5clb?V5V~&Yk`)f(+(;lO>ARuipYXW5>;mrRzN1k4B8OlYy!OFpoXC z_vjmrLYb&y%Q^Iz>gwT{(pr>0%c55s%{JyrS=6N6_^ICdrs)c)*WuSg7WB(kH;F%O z+p*XOq=;1sJRT)qS&GA$NI!!%lXd`axwW_6~eT?pCgJ8x>swS9n+ zJnhU64=&CSOG0(;C^2E_25nl)G45FqkDkOeINu|E`wb; z(eEsqHI%76OISvB;O@jRe!}{-oqwPa1wbS}Z(~Ray@5flDO@4WbNS1v+hXM?HshaE zBOqf;KOME{`gfT6D{2{j0w(!GT>i;_O_4vPr!34bOS7BqS?ubC*_Prgei?zP+{N}v9YmypeljMjwT-G^;vVL|EQ*T74Ve|NIPq80f!I}cku98n*Qr~|G3W~P~iaL zvR(KMHM`!D;Jlu={vqxNR{+mW98l&Q9rKLxlzCO|vceml+~iuL=Zu4Mm|djHNlvJilpcV;hj@7e_TTWmVi z0Y+W1>GAkKuMhd1thA6({`ZTt0fd4!@UqC?ce9^amtjXTRE9WD~@Po|*^559t zKwTjJiJw*Z&a4s>z=%3c;eWoRoxs08hZ>)sn97e#B~4)>`>78?p3{^2S|pU7eKX}d z*pN!*vm{Wh{1&*gd6^Jc0oe?E|{xw_n^ z({8d)a=dCO8HwM&m&*X-eh$G{Rd;m)P&iIr7Aw8d`J(~x#NpDpL?-6;;|NE;wf!0~ zf2n^yHw!=T*gPlRB}GMv>h$eNzUN0T7zK_RpLUImR0Z96q_9yrWMgfcR7Z~;aU`FD zYy32ep3$~aFYMD1k7YnPce@CkEKwJ<2lOQNme<1i{2=d4b-fxPp{}=H@FVs$vnj%3 zYSL#6OiWAyUiB;m8nq(p6I0e_W$wd>ii7bSuP{46i!gYmfSyrwS|$*;)u#Kzyuz9q>hydnM}(K$UQ=R@uDdf1;C8=TqT0i2AZ-=U z+$+bi>^GcDTUqi&4wlueu;s`!+oxFreTWkQisEWuqQ}P({UcQY>JflSwsUK_hjYzl zVYRijF11enS>@#K$#07RW*qeZI+c8%j@$iMwFWCe4681~`(2pwI#l*qXi|f2)Rw6H2TluO&&^e7Zc!1IImT= ztH1u=0>4AO0QuKm_8f)ooE#kek-KvfE*e78axY=Z?l=WFZv0YsItrqo%O>*^33Gh^dd}ZiuOr4xc2h-wTIkFO`nrpcWD-PJ&I3)ctMBBp@pf+aq z(;2kq6X>wJkD@;`O67ia1C_eWVUoWnt?woM5+H zRoN&m;=f?$N6k&aD7H49zhUQr-2cJOTOHJJ+h|=~%Uio_jgLUp`=UG$wzaQmV5ZB~O+*IoNyFEY!g3;j|Z2zRGjwmdz5rms37 z;8)~!bnk%d(YLDzx#7S!<>CEnhO+xnqdEt1pxyTQ2(M8hKYFx+;_FBcZ_7Z{BelhM zXnH!~B=~Ad{g_)jWC?Get^lq@SHW~T>|`(K;lHVvBQ#~D=M!&xv2pJ|U;cEMy`5cx zo%LZ_$(gL@mJ#%2sVjSr^6hVur@dnZ5%uiU&hGiM(sc?LLG!D>PC@R$hllmVljeRb zuSR9fN#`8>6A67aRYqOK(og*^ce6PIE6ZDQ#N>PQ4b11?Ao7&0^D^u}F}dgdu%G{h zH`mtJ%J$dG{D$9|1)JrW+N^>YDxIqLLTX6h=s_%GI5y#?%gJ{SZEp8!+@0ARH-gCy zX5tUBe`3aq!BSWme)Elz8f9~gnhWPS4BnUe`|l9bkHxUcN&M_T?+-3F7|?~te`?@I zEbr2`^ZRG7@;6k)>U0$ZpkGL8b zW=>$_tD&qYoL9*d9JPaW@CX>&r))Kl?5}|(er`hi8h0$lL> z_1$z|tsHqG@^3-5!#vlF&OdAvLxdNsoVYvqaoIR<;AwFGvAb-p+T(_w#)h@Q1hMyta(ArLbW=Rg@M(qJ9d7b&>~}pTp;EO7(I%O! zibl63QW^MvAt!B5W@1~H4E0y)!Ry~Vc%5KZ>F94U=UgrquR6>n&-wrxRUN-cgJtoO zOSaxG#gv^|OrESx;dsF(aK7FnCI%n>g|Tm+Q>&48F#iFVmfL;3&Z18MmTtnPvKN++ zx*DsS8AEAMo-i;XC=sc8lg1gy6=O6>TJ$3!D}Edi9HJy==T8Qh@_^nD$P(`>T1L8e zUW>HOE1Co(7Jh>#N5=<=1?bA@*yBkYUB;WuLU2#*B&+O1BAj=l)AI~r2rvC%ez!z^ zZ?y5fM~5}@W$&GZ>}E}PU|0pDe4Unv`A&JiDFQ_6*#jOK6g@zy>z^BboOeVePbEbx zUo2ra9ao@PgPeN?GiWLV!4k5;ShMp5@8e0C3@qXbBr$ZN?#x&7_1qn`n#A+?`mfQd4C{fB- z@599<#K6tZR(Loyb&lc;Y}Np<(!pR@dH#g zgL>}WT}tBU(h`SfLI#SS%ZgmC0FrER*zAsjUVH5wb<8^=ZSWpkH^gStVCBdA-?_*-RN~RMdmff1X-xDtmx1N)G4Xi^ z<`1Rg{LT4tme5vsvE4pafGS3*s~Y>eqqLDLvYEZE$8E2EdWI@Vrz8>xzmG+wpy zhLi%>qxqR`^$H3smxZ@$%Mg|7^^ZJF*l^`{SAl5vl#E2W zgoFfZlTT8gKYw|d|J)_?X&QT)&aK0dTXjjn-OF4azNOGl!SRae- zl#)8^>On|+hN`92TEuZX1DC)_=$X)6m&QJb+j6sK#YZssnTc=x=g*+`S?&hz^`#m% z&6kyaQT$m`wb0Fd_|@C8-9*EeI8#FA@`ANzhZ8V@tAjdLajy14Ic(YDh;%DgZ{4}M zaC5F=KX(PMUEVhPGzBQtcUQj>1t@b`<;tby4qom{x6gGm; z3a_v2<#2XYg5n1Bj~8}Fj%wYvKMe!Xz4DNmfkG)Fyasz%yL_4i$WtF`Z1Nq}4p^Wo zwahb|qxQeb;nTF}M6?nOu7GDrpAye5)4?x(d7myUqZ8K>HI`qrL2X^WI8C>YiL2Y? z#wWVp-YLC5MI`+N`sK^r12XtO=S_fcJMgx*x%w9g)YQH(1FUfOtn;Ro6$rbyZ1?%i zTAOt@6jpxAhY!IrK0a@gy!(U=^X_j%khvgn4x>X1IP9uf$Is_UFAtDACnJ}VBXB0^ zd%0(7JDK`d-;oy*9K>!j#f(nV^J$D!HC@({vuZE$cN!+^T6CCalDJRcq zcgNW83aGaTtCGm%>D-!w_5;VC2aIgJ3AsMlOw4JDoRJTgn!>)mZKf6$uBY({2|9Fr z0|Ol8Nq7XhNN!=nllxjxYj5P0cgq#WSiwWM))VHqUPBTpBe!Fh_%NKh&I@dgcKXWQ zH^(dyzU5Qt2k(-AT2VV&e#c8+W?6fY9{-YNK@JxlQlF;o=R90IR<*6D_0;Cu>n;P; zIHN~-tYg;4xT@zhNSWYr@`-DO;^Z1xCQ@CWc5{~987SE0s1!5C?L-GA_sTzh1aDd` z$HsOIKvgd;wewnaS*Tlo?AeFbHPn8Y!X=W!IQYZ~vM4DwK*V|#(h5fevlZmDdYMqG zA9$`K3blmDOF)c?8nLifF;$`s-e#M}a9LFMP$E@urbp{-61Oj_^D&RYUqP?#?Dq90 zow9Ww7kkdBI~IH=uIpH=}%syAZDFuI9ZZ(5}PI@8Y4;(HKFRJVJ#lX@FZA2w=2__|K zK*4tfJg0mVLB4T%o;?TSqZ16o7O%@?L7NwD9TI>x1MILMgP8)@^O~{`GU-ez1pMk5nZV1Wb?uilQxn`pB>KahYri6;3zSX=N))>j&nHNfFP4kCwZ0#x{?aJgptKz zJb`B0N)^la29NVESE|U(cDB6owNc4CKpk;!EzeJhI8Fa_=90u!qTKx*lt|PlAEKCI zd3*$p-wzZ=%F5Q*DTyiiE)&WWT<&&{gphhmS->gB1{+pO2J`V&Se&?1%d?fpt74v3tI37lo^W1Gi+5Yp9sIdW-}%y8+a znq*PNwq*P*A|PYfd!-32luQ&tYo6d zFVWHPOQau2G#$X9%OU{EfJQ#lHnD7; zjt#cR7}wp^JI$)BqNhP=Wo@b@^4aS4vTIVBq3F?UEa2EDZEyhnVG#z`Rm6zsM$s^zFM(J5w+ajFG%BS#JYha^s z%u><6lp)9k3-A2W%CWnLbq%R|4y9a}{W}5{=>){<^zHs`<=BV$(i$>#@dR8lsb4U# z!Hzmp?qPafmmbNErj4y77@p$RjLj1YswV#j7B6x}o9c!hq=US=$DCGv_Bn0mI`kK# zTFi0@MZiL9c!WF51&nD-NokJpJOg))7cL2)&Cc!Gtx}Qcd^1i%XFo>VK~zEx(s2UG zr`=vI1`xNXs+bQbfi)A5LJ*#fM4T7uJ z0QX>}uV)B-z!sr${S!q-7kK!Vw-1pL-K z!$;D&cPhtQ=U?U?e*gacX6eTq+~+zsiEj^%6g_RbV_;Yq@jVPB9U(N1V;kW!hD5)m?RNao?mh7qZ4)UCZy)As6&xPfZ7 z`CNs&5c%9C-k?J?il54%{UBp3?bD}sr{*&%`$v0PqT|Z5(tQ8R93_GSc&?mwTCF;p zMXjweEhlfbeQLPcz@_nupgfR_qN~z(?(>=rMO8K1q3`1Pnx=-IKNqnG`vxE$4@7S| z7`RKPJ!gNQD_AQ()l#G&Ph#Q|Tu=F4^L)CIW-Jpz<<6z~yLZINVO?p5*VTF^Ot*Nv zrQu0h&AWf{NMEDxZ^YRDZ5KT6z{oc&p1&%QlhzXIJ(Tn&9`iT}{51IY-uZMX0|w^2#z ze0ruwxNIS<0_@jhaNFPgpuFDw=_-8v^z>6$82%;MKmCIM6X2SYtnm9~|JC8{-4h^( zRY2x!Qe8`J|5Vk1KLs-N!Q;0j|N9jF_pdZDpxP>~)w22X-(vqiWY8a909WAULE9+Z zf2sG>zxhcD<`nz?GHJNM0Poe~!(5B~w_yJd3iwZ9tuG9mdM$DMYXp(Bzhlq;pB^=! z#e_3VgNy}QK3$rQ$aXd@-kx3%3CAC>Sq=L`}Z4@;)1;(?)&VB*L}vAW@I(hE~HOM58Hh+4D9`q z-v}DTuM%S2)w+LCORJvYyM5o$)iI_4?N)=z7vW)|DT;O?D+cSAx1;fMw{hlu0_Ehw9qEFjAo|Sou)XNgizimLwj5`Er#zAa%)|V% zMKA9Z*E9$nCGNJr<9LyBaYKYvO#t6c*^=F5Jp3^LB3l`~U)2$-Rjj~p zVI10xZEhQYkS_7hBx z!{P(cv7<%y+SzSn)vCCv9+F>iD$P8 zMV4|z&2051q7Ql78|4!H8l;nX$2#kSiHYyg2;p@LFE5CZZm^!mfok_IW{JUL1Ja5NLEari z%kEf%>2jnTxfHV3jsWNU>ZLW-o)^-NoI5H0F{P%#yP%gmzFm|t8TA9%WP13M$9K_{Q7-F&+E>&kB%Bu_1)Ldf_$s@3!p$85at4cO>^Lj{+;BS z$jAQZ8!hs`P^hX@MIfjh>VM@L1=)KKvfDYj1YPt(<+T>RwxR95OPnxOm2 zEua5(^V4_k?g{(Q;PrOr&Zgakwy2mCK*Xg4T#N9Dy|Ie-cf~$nhF(2z>sL{&9kRL# z-v|1yVTpW&c7mP_xHaJjs!5*wn-{m8W)HY^Ex`8K!miyprMzj&1|Yp&%UY**uDN5o?Ly9?>mtODN>$V7+DKrheb^IEbbDLqA8=%YqU#WQ|l}+Y8 zQ9ig>8bHq@hA3Rns_tqs3OW&D-#uiPJ{W#}Zswn7Q@&*~aK($l({!fmFT)pFHBZL~ z;-}Or6S8DR)$dN3ZThVMP?o+#*~#rj>7D5tFW8ubTYc^3jQk|fOw}sI;n2b{zol&i z6RNH(7JQzeLvA#tAIr~?H^-Seuo;C|i+y5M#9k+!w2uFRY)^#Tc_ld`ab{|f>8G__ z%HEU$bo&M1Kow10z^QRNZpJ<)uwrSbuPcdBO@4-qAH*2gzoTB>khI$z^4TPZf}!_{ zkQ=B&?Ix5ay9tS{^(TL@P!IGV$t$gvFwX5p9;0gb6~-s)7yBs+jUvSouLsWc&OPM_~Lm|i1g?iWMpkIx1}&4 z&Wl5~=DJ=p(deHIgM)Kx=@wKqd=?60Zd3=;t$c~5XxdHczL}YrW{D7`h9fdO66-G| zQ}sEkn{48>LQFC5bEwB0HE!HK@lduPS&qF*+&_GxMVpYAZt+s#EL1~TBG;nfv#VNXUBfwO792bEWw-gF8t7{*h1MQWl|$&8PmG+ys3iRZxV z&P6y{(k5?KvEwiY$?Z0oCbjv*M}x__PrJo$oxsQ~D~2Xfo&|TQaPLJ*XI)lN_YQPM zG`nNxF1|$cLl@54q=kfCE0PD(R9$w&QV99ND+*Ri8lazRUlMvMb1hnYIOEf5+YI3K zj}155>4i{Wv)II@Wr(J7(@Lrk`g1w%bbV41f~>tQvN>0BaY0O zSXm%W_}NIuY=cy4>|MFvPsd>w>JNxZEw}P z>*kg2@jfyy12nrYKozUAf7vWZ5s47Dzz*x}X{nps7TKGTeDVOU4dSac0run_d>@zL zsYhdDA6#VC#JI~&1@Twb#YInitl%HeEZXFpjU-^q$LiCN%tNQ(=`lH|m1U*oPix}6oPZ(_fAv8pNxdl4G0ApWHy89 z+R@1w@kzRJVQa)W2hF1H_LH&p>)YjUN)*utA=J77&1ykrgmaXKAvR`sQ-7l`Sy`!8 zL#qDeiG|OA|pzo*#_twH$6mT8S0!Zg_;LNYr=3Yc3iDVOX0%ytASHn}Fx zEGp(@pH{rIRc69u_B|oLL>v#f;9FFJ*pH2xw~L}SLhyCvW_jPq)l}&FN0+~t8PU3$ zZ-JLQXp9{iPtoHmCiVSw(N3>^!k`Ta1h+$?owPO$nQ+Maq8IMWuOvEtzoXv;xPhVuq9*2yTwKABq+-lsEA)tiQ%-tAU?JE}{_# z@UEvUMzTM0T)^eDOMr3f5OcE4T*H1^{#bx5^UO8sH8nL>zG2*chCjml>?pm*a`jB$ zurF4<8>H8X-CO}@9}1?&9y`xC{oqE-Yu~G4V{50iZ{9U2$-LN!{YBrLL(J86i2vl< zdae0^NZ7vr`2}t9U{Do!9<}cCb6%2$oG%GDfSCSDpE)x)HSFn-_nv(Ui4C=FJbnzWUA zH0X>=W`1d*c3no_>`)} zdji9z>vhwQ#dl0Dj@vv=y}}R#FQyiTk|BP}Mua?#-i${<<*#mFM2O$OZti|<(BXH0 zbfTrE)}@SiXW_2tj)Q|V-i{8A)K%@kQ#Ql8jXe@SZll^zwW>qL!_$xlAvljS?U6GL zD+Cd-Z9rj1>f8h4aaW#8u9ic(B)q%%;r6|7aaq&Zcc1I znQfL}-;^~o%RQ=OKs-D~@m ztEnunBBEG3RO>+`##0RBs}?`W2W4|!pZ1nQy6_G7W0%{IBfQrQbWczk2`&T*6$}p$0duuRny9N^kZM;!i3IW-&*1TuLL}53RUGl~X7L7As{7O!U8##EGd(-@xiJz^j zl7Ula0>KLdKZ?qPu{DtTz*@=By3|`bc@Ik_U7GrQqa8ZhZjbh|1?ir+nvFKhc4*fn z@0fbh(Ns!F{;f>SOHW0U{P(6&__*R9QinqarVyb-q@H0{vy}#@n9@|yGnd!OG#4FH zVn`PLn79a^-q1wtQ4l(+{qxQ*otq_K5}`%tg$DKTxrKj%!pW&2RJ|K?=AVy4w#a^Y}#?r>{D*RsD1 z9GckNY|cbHns6~kSYGrk$G9XE#}2YK(EI}uj|L}R)CV&C+|GA^(e$sFpnpcmyPV0f zE(m!;s=3Q-G;NfNwp++X{ z=B`*52I?9uCAXo6>4yyHwz^l!vFIiv=B{eY%AJC8JQr6_|MEWHw~zE0zLu{gaTq&p z&GsjE1EGkwN`)z%@s+=BOPJVC0c8F9W)!nUiciDy4vwXM+^gZH%RW!K(}>o*7yezs z*!woJGoA(0NT!IZ6&seoGHu}dbb+x%>=#2m9@d^3ynmm>6PGG{kxq7;j|St4j&8Kbcl~ zdetq#sbS4~{GP8SqQ7asL2_(^h&Q#Fzh2emk7z!2y5}DSPdB2rQ zXZjy=3G^)|hw)Qb_czgU8Avey(ps~RFy>L!(lZ`cXSj))3TVNhq`_Rc>y|WelI&7g z5TU5w4C6+kHEfIPgY3JS1!Rgi5l#L?_%(_`?}#6^nANMO`PBZxk;Me|b{iw-=6+gG zJeQhNuew$JLit07dW!LQ=Ni&vDaCouIt}rWw2Z8H=Ow|jrE{TCR4#MzL_%>h?1A^f zxDXtYg;#HEL^~B=fi1h?n9<|RPa0E#@JzUxc_v!IMZ%&y+6RxMDHgxUh%*Ovx+Q-( zf{`);o24g3i@D1+POLa&m6s=@&yTg1?Nw*;XH(2#s{-xw%Nv<>>N6I-&fMb8JAC6L z#`7EaCB;iuE|5JO8t5q()uN-VX z;}vsce&(E0Y2Q;0thjMDdm)~x+B2~940=f5TjnMM?PKCkeA)B)laFln4S>*L9rA`|9lYSpDBueBh=WHU+sLQ7;Wd+u6q7NC zh|s**WA2)cDaJJUrTkaSnsiT3%g;m6!Qxl0yZMKEHF1;8SM3590=wU_`R?8g5Qb7H zou+45k!Z%H*Uc;W`uNarbAPzFxa^yp{B?_snmS~vTw5LhE98%+SsYa{kUtn%yH}to zCgq#?31RQ66c}m;#Y^EDm&TvyDfT`*IhSQNmWSmxSS3TQ)a>uI)RU-+!{!N%vR^)d z4YHPQSa(%TQw)EhNgEV5p8EWN5Z4kVnVO_Xj~_DJnrSOL--?(nQ>UnQwlT9ZFHehW z7U?|QJ;$$SY|4v}QZUDT*!Jc_xjAOKJ`c4kF0qd8uNY)=bcuZ=3*CC{em8t9Lr+h) z*yQ4z;QbTT5}Tb>2RPwylNS9}g7X{Z&jMWg`tS2Q`$~;xE8ldjxpa-IRHVfP8!3}? z+c0f5{+u1fQ5xf5#OB*|5GfVem8URI_--E?bt>Ba-4?o~mqvdj!muCCp2Vy>@a?T^ ziR$MrcJ%7`$t;yu6zGv=b&%YOqXy?E4pp@;q-{M+84t;f@yQe47|&2wr5uho=u$j1 z3u9{I@9vki>F2Q`XGKv(yvRvG_dmP8_S)WCu_`~HHy~qoQ(c2Dg?>G6Pl9CmXus1G z74+0m_5w%Od$WwHjr-A$_Hv@~NLK?}m2}dBv{Tv1FUDnsn#0oB-!r_^7cRuUZ>!4t*NMaq~m`7S$9#UXT!ftmbJW-W@~vIvjESpKj( zd@dY#5N2#j<*hJX&U9)oXed#6?e+r4w-Y{aL4u6}o*(hIhwb7Nk>ECI`Msz9cGHqM z(@&GpnY2{>JhX9$w$0MkC%6M}4qb+YdUZ4M%K}i(6Ne$4R_-b#Jj`~ETtl*Ili?seSbIag%GB>;1Ki zbn>k(#u*=`={q+c7m{KmDqf*#NW*pR-K6%bBw~5@3iHmL0|N0=(Wu*35m7=C=f}IN zL`dz~8Vjz7=R7h^ch%oe6f+xQB&SJ{A@VwmO~0~i??cXMqh8rj1AT&IE4aYz-F(AtUZdiNajvdRG%i}ort^Dvv1Pk#BSF+ z=T3ZJOMCo!q}Zm$Wrw2|X6@f@=>FO*Q=o4u4n-^2IgO-z{jKqSlmb|W%;aNw+3mCX zq6vZiFcVYlr;hLoU@6Im37vl2OXbKUxq?~Z%gUEJbQyow7JHtWVv8b+W z&k2kAC2lhNnsEupyDBW_JNU^lm$aTE?EBCEkG!vbtE$`DMkEv@L^_pjq`L&9ySt=I zxbe8A9k)eW~@2KeUE#NskUhG1%S6& zo^cN-!IRoX!2?f=sVUj*nWQmZAC=F%v_Fiq7Zpx4Dp)EV<+_*vmp4Qima1A|j2vy2 zavZ6Ls#;xKs!-aNU>H}@l0veA$-`c%e{_s)k9eMlrjQW3ID*k*Zgat;cRuS5bzo?e zSfN?Wc(UMz(pm<1abYObzC#|H^*%@42rjC&gX=4F0|MuUaOa2k#>=d&N8zgq1%9d$Kx>2 zAHL6N)t1h#>{J$s$OGuL%Ow3nLdnVUZxPFT!b(AIH$2)JZ)RPF2ch-0KZcWaWOZZf z1hi)7(q2egQG;Yfm?*M?6S!xCbRBI%e6xgw6IBa@>SoI$^y#d6H`WX*U-OPI@9rrt z@jAV}W@A6G*&nd=KyaN)8TRh;{=pSy7VdBu3v*9e!Do-$?-Hi4k8+}qYo?DG*0vY? zVe&Rucen>c&CL*|#Cod2M}wDpzNE<>_aTcFC*&blQl8G%LTmX;OWZGu*`8lt`|9`7 zmRQxE7HRLRYii>6qtkzHupo^#oW(j(qMCmxCakU)PGv!*v~1NUhC?O~ZEmbp!d8OL zxnnR%lV`HNO$lq6(-~#H!e-DNtAetqQmWe{LPV?mLNTk6b6X{J9ner}ZTXK!lDWmm zOF^yDcB90!?Y^a?7*UGkEIq5Hjtoy-Nc-2~G+ef3+Z`e-*0mCrwaS_A2zBODRLPHP z*EVxzSBMkX8|5Wr1fsk9m8pNN-%L04EU}iQvC>~ZbX5a8kAfKZX~HA$4L5nzntaaA z1P5(AJ3behlWMD>?n06gs^N>>VPD>zlj~(tqVe6(@c&x!!m?7Jn_|(A$15&A*=#P# z8D5-e?Z>-P4Zy0Ze)VM!+$ye0s5CI`vq{lv@i8-!cF4sf;g)@lqUjUNGq$&Wp_w@Z zZY*=@Hn56^xw{>HcV%z6#T%&(hiLmR%M_LM;i_2%eP1si^{p#&yG#}Y*txR8LFy(e^RF*gAql~5Tbj<#oT2l-IM>BWTgk*!TMQqyH@?Q6t7{W z$JaZnlSLVXimmtB1X6>RsLG(NjIyj6iVPc+4hKyd%A*#CvId{@!X%C*8RwkPi zhvZY%ueF)E6DXR#@vBNC&u=2}-f5)x%Nf0ZuQICSma=l8v5}B8f-j-G7qF>S4Kp#& zG>RZsb5o%NCX-zEWg4w8|QqE*{czamzjzo#vIxD~uA z&Qr|Ei`7UwefcuXBP5aY@W`G6L`p|MaPdei$v+ZHf{3aqel5Lc=jVA#OF&Xr^9MV@ zQYQs-6=PVrrtf!PVixQ8A-THm55%DKo!&77h&l24Aoul2xzlFPvxOS&bR+IYq2VJ! zlYD1-%BYRlXcY-#yRcf^)hhbYB#oT`J|e5)4$Bu`qT$Z#C{BM;O}@L2_bgkqTzS0- z4O$E>N3l6t5RnRyLFFxG@q-B5_0FkP*MR^M_r3n)kcQeS-HiLMRrl8`MQN|>>}%QqZLbYqSKD_BB%YRL5l(*MFfFVK3exR#z&(b^KU%o2Igz zGI6OpwfQtHUj={;DZfsqQ+E*Lut}?m&M}w|-V6?uW>}TA`FfW@5U-#~W+D;0yK2t8 zwYg0Q($oB1v$y0K`IK_14W@NqoYTqVAA2F+uEbSY;}-~`g1Dx;WsMQ0jvo$kW>Qhd zEhA&8n7qh2NX0!Rk)^u=?hiHmtZlb9cSTO8)%Q z24_cLdXPJRLTK37WN4kIIaWW8>k6+D=K?sONV=FmU)$Lf1@HeW_0`b;Mwe_^E)? zN=zSA39&~~UO(|R9rpqo)lh;5^KA?&*ZS=QsKU@)vlb)_k_dD=rJ4}Uu5uvl#8UNq zOUdLSpl`>k{mKH`(aF+6!EXZl+}rXTC&PwIQ+vR5KPD+NGL799X$gm8)y!@1J&GHb z2YuwX=1dv!Y{V9zi;zVJ_F~ifOpROy%{E0%xNx&8;uFz!-`PT^oMHc-m7iW$HjTDF4DheoqUB-q4ymrr{tNYP#TO6nj|M!RBqT6*2qjj zIQu*Vnu!@1q&ZG6S?f!C-fRI5pRwH_x8Y1&BWGXZ?6IPvI5lNZV%I;_JH40usow5-aWf8XU(hH9^kH(k01Mqx;cG~p z3PN+?Y`g^Jxb%C(DU>TG6_HR+vzYknYxwx)?}RIMH*YG-w^T`fY&fQ|YfJGQrm1Xr z&|%x8l(**ffZ6qQM|9U=rCcHGz?%-u=G@ z7MttK1Hp^-m%WtZ=Me0Y>k%Kf91q-RF1Ay2R`)*fyn6I1XLOGtzb+#YLPl+y{Twg5 z@XnE+q;gJKWA92xJndxseWpI>`Y%{j`A8nb&djjKaHWYm_EOG~gcFshsB zt(VZ_X|#J?_~eJjx4n10s`krSgvSpGU*k%_2@Os?to1C<=cRg~9^~|M=eelzdoo7PA5YbVm;Wb>a6N{IhM?FJS{-kI@l*KIblS~*KBqUDiMve+vG`G^#k@t}( zX3{*D=Nwv%cTk%akSRMO+7Iijl;Fux9dbW06*G~n>s1^QW&4HUMf1g!p?t6nlk~OY zQgstwLI_92)HOw|uXw`3!9u^sC)eV*1}8m}la*?vp;%M%AJhHS()!(A92HVkgSX@5 z)`O#OH6YIWR_#~GtX1}LClLKUdEuP3wNink`R=4bZuI*@d7mu*D}5h(SC5}yI+gd_ zAtGdhm;(}=H_OMg>6S%|k=pMZ-sc}$Eeh4>Eqv3wEE`DRx39I`e`BRuLhrxqhHxIP zwr}UvRQrx1@56klMAu*pLl0+%y*gV|fdZ$I0w?=O?+~Yk$^BKqof9d$b{!sfUh^_x zj6g_FpKcInbbn+lL5rbYZEhSQqr7Pwp(~OQLEo(BS}EPy^6f3oz}(Hy;9-dyX8(4z zy2Z}TS%f-@5wh=k4?lP9$6Y~`NQ!S`=&T>@!Y?*nN&74-{NR=(h1a+1J|3SnWyYiU zaSbpdGjqpYjZ1s{<$b*1fXLQnu_8?gYhi7sz5|1lBv^nOfCnD2QW!ZAHIVKZXKx>I z0V{mLtPDnyN_Bl-mA_mM1d>?O8pyUWUCG2t{yITjU5BLH6zpn63A$q#_>j;}&n5NT zes)z#N@PHQXN|Et;5cEX_FF&^mA~Yid8C>G8lV(wMNT zy08ARDK5dsQN@O*-cmCA<&jprJH8#}dtwA(QO=sn~xt2OiFREWzH<~V&$gZ}5 z_l<26hTYP>9DR81&3^SMB!-cH_l5V_if|pW;{7GDH7}W-&Gv-O4GuQW)Y`8&Btc0v z9c~i$6J(w)HPk0*{1@HZ$IM!vBnVSnGm^_bj@}$2L7hiNktPNSTt;~ID4v(;is9rS zA%e21=!I{XZ~3Zpd&RP@;o_Da_hFmWwEMufL1o5ikNsh?w2_}x ziEV6eO(2_A;+Hi9I_Ww75o)ZNA7{T0r=7aBI=ql?tqKOB0_|N^a=vd-yhqqnk338x zs(fxP5@QMu8qKx}DX)TA;lh+y4$NbI8~kn5V&+}lJ#=li7`{M?do~i8lMDD zyAscVNG`$6eD@%JWX;w1nf`d>T|a$6Y{&DDEBcl1;;#<%E@Zx8Kh1^#qVkH*xj6P# zsvODulndo1{O)e1`~6rG8k?JyEi7oTD!b&uUT*E}Nws=hu?nQ&@fzoAV&HPof91IE zMZ0O1pUiP*CYQ>1?V>Bo*EN8rgj~EWq`y$pt5Q{f$o}c3pJ+I@Zpp6qs(&{VYk(Yy zjA;~z@~^YhrnDfzO`VatiqHAgE#on^zX_M6IT-(lb^=X5!;-Ct6lmDjD@ z2f7pSehAJc57>>Rq)+nvnx;z>eq;XQ$2&BB$WQ|a1Q*y;<-bqZ%5IEU7j~I9t2gQt z_sUq}Kv2;?qPd2@u}R%4jL6f6Q-@7;oZAhNIQj~S*!G>6&eEj%H_ceJrI7g#pcSUXwx7N6~+V)vbZ{ftJ z{fzRDmC}eV1^FcMoZx80#ADypvN)v77AGdj7NS-eT>h_WRbo@VzFN`%62b%Na-}YO z`L&yh3!z9&|4^5DH|h#dn%>hs8zLL4U=+j&P94VzKKD`wG!~bl&B3Sjdcw$VdPdpy z{jq1%bE~O==ZD{(v`o!(M>V9%wp!7*UeB|ndd38O!y#@B0fObc@x|p3_Nij+8}!g; zXN8|PWH2CgQ)ZzS9Q|v>5=X7}nBE;ye2P5F=R1q!c+Y!#rS8?fK$u8mON%@s_5xR- zEg{Z*1Jyln>#{B7YW&WE6RkFN;l=3{6(g%X&h zVqz2<18US2rx!-@NB-rYG>h4%uXd6frOodmarZu6lQ~H~jBaeLzvI0&R5Oa>Kl=^` zbsEqo6k)1YYGH3Q6?JcnXQi`LDv_4^`I@ToqoeagiAM2;*lN4rOvAEDBLP}gw3<#@ z`c6ERUAAZDbrNc2s#W_5YmxpHb&SJ+L$RLd&(-T;npgDI^&FFNpd&Ewu(E`hk+k`t zl1QuiGF^>9!}U_G$eb-mWRASj`%a~^8$PRCT-iEz)zK5W&I=(uzm=nzd{hLaJRH&@ zonGbK1<5Ngni*$YjEi~zsXFGKxL7*BRi5erkgg;Y*R_ciDIx*r+;d^zK@*VCe_y(HqNuBZ<_Pol%gxXLUza&5V@tv z$t_muh-SUhWuuj#adTh^+xh0Fa}g~->UX%blcJqdbUYpks$@(>mrM7!8SlD614L&B zL{TH^BzPjQ-!JhG#FsTDG$5pm`I7x*+GQaQ%1mFZUz*GEX}saQ4k91!cC;S1Vhk2)M%BwUKAU{E&~XcnqFPz@>&oHZX^#34 zN3nb~vMIInQDDqf_E75n4Z@X$)s47j_sg%GzgSn^_N04SpW@;R6qU}Tk0txi*{oUH z8il+*+3|U}rHPm5(lW*q*AW-gY>A0H1Gdb8C8n0X$=k!+HDNNzb3%8!A9DNkwX%p| zdZmQpA{>tQAPk{x9?nQ0gD~*m#@b@VhVk*cs8cRpG(rk{lh~->aEc?T+@MR9MezNr z`U2;$j9l)_+jj)X@#N{~rF<{1BSC2Mu3UatPGu1_q&MVxDRLezs1O_W`#m~ky8;2- zE(_>Ba=g+RWM*?V*}GY6@;32!6LGzxFSBU=1#yc_t%u+F9+e<%?_tUaLg{y~4WGt@ zCz-Z_Za$`y>2jIv>_$q+c00|PV<`$_GQ~xV?;uO9u<~ZPV0oWLdt~gE=oe_T#`u8u z$HN4XK8s-IWzU-9ydkFYU&Mw7J4_Mlw!tG}&SJ*5LdclF_ai>L< zFflkh4?oj-r4(u~rzsurf;y8fzF$lyGlc#t#JlPdk)zLveH!ZC=%d}v*#6N(J7Gb* zR?;EJ`8x?|aSJ)xRUZmpo5v60WCRrOOCb$aj!9o5n#9b^e!J4pd9g^)$6wi<-Q5A8 zT|Ud(1fE?{f1Z@0qN4XaviMb3yk@X%y?-yMMQxFfR&-x0NrIj63an3TERaIvKFhAmonZ=Jhfe-0fZO@8@AHfBy( zs|DIw%{F&lwTZ=xhvkhe9BQlXk7^!Jx0&NR)%76aEzc$mwW|YM(<#^`Y|g4sfrPkn z`js9_rIMwO=>Z&~q9G7k^M3NFzOlX`)C_gEy3LJ|H6FF7W)x{3UtAfZIzj~(i zXudE)zW9l*4<5cKFCy3j{F7(q(8RTX!v`@*u}{CQ25_GLdCk3f($KK5CfOPHzf4F$11g&`*4P*A|NCG6x(#p- z>Yan1c?MFy+3_D{jRXO;9NE_SApz9lzZP`x^^xXxQ+iABTTB06WBP500G!>HW4V3a z|0=~r0tShPmyae1k+qmjj~5#{*O{RP@i)xtfl?c8gJkH zoQbkBzq`a$<+#hRD)x5Es`3H28YQhQ$-27%=_~x9D%d|ull=lizmqbU+{O3U>REWN z=9;#tA1N$v7WoX8Hz)fkM14H5NY?+?K>zGup;wR1^om6^QE$c=9 zwKsteQ=UBzn{OdqZ2zmv8D(H4FzgL8)CsYku|0>y({#toOcRboMV&6R`kZnW5+?)o zJ|vcs8lzn009kxoMV+qK&Kvn5IW(N>9rVgvUIsRO+qaeHi!zDr>g>?6DG7*_^Wg<( zp&<$*riPmL81r~6_FaIN`CLai#|;&^iGr33v3paq(sq$S3^T_8pG-4qkMfPnJzJ$X zo7m@TtLDl^dTQDN{vZra6ponV`j_lGUS+!N!FK#e z%Djn{IZ+;TPk*l#zGAaP7cW$@(rNY7r;m}YKQ08=c)>4zn2$X${mL)j_c*Ix|2|a9 zvq5RS32i6$i|^!=9Yaoy*;wkqNw%-9B$J?htmOQ>HCpvY9nsv6&VPCK*Y465En*-Z~$TkLKwtQqwZ#_a|MiZ|*ZRND{bmnn9rv zK^7a|kF;=NO&&F<8(+pOc1wsTH>=+u4KeBaerl>kt11$C^0$Hb$p)BO=S>6f(P8`$ z$bXJq;(CO}J03BDinRORBB5R8{}@QFz3H%*k{aSjmb90o(REziueC5Mqir`bN84M? zFV2I?*G5*jm)xwqk4qM;#6MJ19g zXrDbc9AcaDPC<+5aBRNstS_o-CRp~m=-ueYHn*xL&b#m;%;6d~8){N7T((?^*x;)0 zG(g3|$=nnG?d}jP zjhmzwpot|;yW<9F{z~g@%Qxstvol{~diYBH=3jEPnbg=<+TZi1P&2v{mhI!5SynCe zt5TYw84zl2{vaJ~?x}{M8)YiB!F;o{yw%wwrGxh?fad&UQ)aI-Asqd0^PgYIOJH8J zOdCf6rV8Tcm;fQOu}d<}-hDICJSJDb0UBCX7UQnBHwzzF5!7l>0yh1n6UEg*cBDP< z_Qx^><;RBa5+3E(kCXxp9Zx$rqt8S0+HF9pZut#M@cplK3(NV*=}B2kg{qTcO7CIL z7^NB-+To}KZ=Qg6unQ%7cvlg*g)17OBwlBTBM-J%{KE;pZw^>W6Sr4y^k}wKR-+12 zr^C7q%L@fmY|D^+G(@gs@bix?0ip4r&H0U)d~(A>D145_%ps3k8pmO1ewc^r+g(7P z^-lk;;eGtw{(>}?RV+eF)wTj3cU9MD%J3hOA-_J;8sZcQPCgcO8tTF}C<^G;Eln!+ z=RUa^4fmsHf}H_heUJwnl)%ku+4ueGG0m%KQtIiKQ!5@qJ;Y9cyG=7Jk@e}ejvJQkp41; z)q#Gm)j1(VTHU6;sG{h5J#dNq%*8p8P;>7QXxmWw;)56 zqU;K5lI{C!;Wn-K0hTz>ZCQw`6`E?A4Mk*HIK?2lB4%6tch?2?@3^||l6LsR{VT1? zxQ$%et~T*ceqg*9I|}Gpnsh5L-+B-A zv#bBy<#l%`SV;M{{m>Y7PhSP2a(+M^`8JU6xILOm z-BH3DANf*v+TH=3pTqsdW+r3S187dB!uvKX^x<-LTY5KxKs+Wh%9*Z}4ct*fu{$5w zWFTnws@J$#NaQ`_d^i@|q-1HDQN>6<$M&?WDkp(1hm8=(zXtuL5CSZP=GO=w7e6xR z){hBCJZ`j+;dPzdV2BLqcn;?*S>Zz})VuIZ*zFx1Db8e?wTS-akBk#Ox~znwERiqa z7+=E@X%thE&Pzl&=+y31A1)B*6gJbzX1q;n;aj#itwoE=+3SdY#Wq}6I+I=Fa7s-^ zH&z0BJU;PZhN_m{Fn~j-*+9grapOGJiRHLftWV3r^}B@o0mp#c0y+lvf%Fs?nT=MT zWXi+x>SMf%c1yz{#_3%w>RM;5|K-<)CMh6ylop@m-Ng`~#818KBgYM+3@4V*Z{g`} z&|Rf-Ty@Vpbr`}bn<|i{P{$FuT!gN0_We1Go>;+^Y$=<9s0JTh*sPb&ba=8@;S(t& zlFsIZ@)XEaN8TVv*?2kmR z>)hrMVtH(gJEICI8{_popUPGx==!jN zV5iZwz3^RnbDm1@{yxdWK{f$Nvh)ylUz#2gQkccMh+T<872K4x?LD5yZa0Z)G%#m6 zN1)EAYnQcO4vf{B$fs55Y}p?VN5#XAPA!X3q`p-JTCYv6e{G_FtInJc~J>-doV>3AQOp~dG@(4 zLiwz2$}KeCP<8$~mbaRr`n+t$`yMVv&@2}#|NBJG3-+>qlC2!2&2E=#1WC%YpzS>T zf78W|Xq1vbBGmE<`vV6Z6%}Ib(Y@C$3e=EKdRkz%5A!pLhD3j^N%%1UL`IP$xPb-q zmaMRLd>2gkL$l_}`np2)Cl*o`c_e&*J5`Ry{s~&eo}`z?RldLgLCv2@y`pm+8EG~G zblOglNCkLTQHA@ABw1LRx=;_#$wHHmbv# zr^*m3s00BoSZ#lxA5n%F2X`AmBf58qLPFpvg_bFo+14SVAEnjr#J89aVCF$AQr{Ukw|~B zcQXuF3b~}R-G|r=DRf6j2gA&kUow3zMjk31I0MDDKa5xt#V)q)rAi1CCY)m8>q+$I z+P8KlXJ<|9mt1E?lEBdVTtkr)mFP{^tae)!L+oQ+!Xu;TfB);wXm26k(HWJ%<`2wV zA|y1`{(@ww1_^Xn|6Gt0?(|?Z!CT6m#z%cYe93X}w$d#^;`!-lMoukcP>=J zWlBi?`|m&90p7;w{n$&>XaB&s3W#tlEUYv~-xS#==a|(Moo)i_^~wUeJJA1FF>zi4 z+?3uYDy*P?5-A3f9rN?a*j^_z>a|&sL?y=PXTjj%v{f7WRt%MWMt$(qD(f9^PXp0q7ldGnle0WLesitT)k8MuWq z`P)2s@fftf`-m`bd!aB!<1<*2;ZOfW?|)T={WY-Wqhw_x|K2G7L8VP&0ZhqWD0$!i zBoh7bA2}z4@p1S+=Y+rP2P^?LSBYW#_>;`}s~{ni$98#YzVkPp=r3EHANN}}k%rj+ zb=yB*09&&G#ljOdr~hqh6Dd⪻KyU|IP)V8{Nb8R}p`{Pogeq~Xnc*s;=92^7VL zD#>zh zU5M+f=VJss)@*;&p2}g3KeR&i>z0{WlJ1XJjU$|$>sWjTXrXIS4AV5U!CRv(xpIN6 z#tVpX(edK0t6VXF-=jfgys-E>)unfGd-VZSz@xbq_c{C z658DJsuBq$3an5fH4RV08mH@LS>+=&oo~SMBx_2HmPcYvbk^*?D_$RVwHwX~pMK}? z(b_O4i;jlYY$3ZjQI~edN4qs3lG5~RY;Ljtk&GepBOmz^ccERmm3k+V!ekr(hbkJw zgCjgKHl9)AA=B1PmKw)A_*FhZ+^$D_cQtok7bNQ=4L5#lD(<32v2q?R1)r)^WX1k~ z?Af;4;$k|*Pl`VMW29ICSk5v+iY$SGl6*W6y){us53ADyy`x+?lB|(e{0N^T=6&xQ6S-NS0S}!`nY5duO9gNR9{6`t(0t_)| z1|(~Z`#I_+3L-e2_g4a1T3ZJ?JN-t}d55H$WdvQppX@N&zB)ucXl_WnV#u4*i z72KHHaT4&y@E3}B4h&dhC+AgW#!>XBVF)A&L6$CjzMrZy1XOg^)qYXj5RP8`kE5^I{x|JH0XBZnU`YHB4(l$Mqzo0w=(acuN(E@ouW zjV~Uw0idA>USo^?dt*%Eb&;n{^@Tf9F@y&KxbnUEW(XiEpm@6av0jYwyV3a)|5h9Ww%!46e+FKI%3~aVde)Tbla%vtY zRS5n|V1wruvCWe!qaVt}HUv|+PWLMrn8bT_#ue1SE|=Nj{G5g^Bnw^k#f{syQ~SG{C-n zi;kD`!#L+L5vk&@U9n3}Qzdd;Z%5HqQ^wKN_Dmx^w}A$$OlijKon3kS`a|cR#{1k=l z?&3>atm{I~Bk8-6>zj@Gr1t@MDfQ-}0s!Dj1J!6)ZmcC7O$7*@Ht9lF7wq!m3h0r)TUivP)-|b*r zQi{hVg<+kr81fayxDmX5V3dMpx`WjzC7d{{X4OBR!r0P!F<+bbd)m;=7M+ zbIjKqP(6uk@%<+Z5|U%#s}n$PVs}$-GO<+XGJakU)mFW{{Ut_tcOn>I#1nqCntZi3 zsA=BYWOr5YLaXg3Za!6dc5}u-0TDui%y$rgQqGQjDm0uTP_q?9JibcPQE9bazqVO^ zB|g@AL)1%=H`y}pI=>O0mLLle{Ds}v#5L7Jhp3pPPY^%v1VqozJ0ZRbaz_@>oiELx z%XasJ-Z5#+@31-S>)O`Vt8RU!GSxTClD09dx~pV|o$# zcdcA!jGkq3k6zNzPIgsayc&VV0VI6ZY%UqBx%uiS<^Z-^j0Lb))wN1>E%Ir<45rho z_h@<3;e<1V=Lhc4M;HS&!Er6D6q>>z6@j{_IR8TX)8qIT3TgKD#=^!f(HfYe9UukU zt-d!KOKFORuin?q6z~R82tfm^;Jb4NQJ>cuhJk@$t;Xu;!coVv&;4yoqp&f45A6+K z%JR<~Uc4|L1CvDRodsy~BnT}bk$@lxm$q<7p){g@E}uF0%S;|&P(E_QXLBu zPo#S~5||@H{3CnYL$N<&^C%)G>*jo>H4Z&dei4#C@yXDSa(NwXdm>X17(V|>%95ht zbvPd*Nzzss1TL4%?M#GMOcW@1{*2f&pCsLwQ<79!{e2XN+PHai&t>d}bd_fzGVk#%n9??0h=y%lTSP z{Q1S${*IqEW1%fkmx}h;gA$9Jl3OJj!-G?x(pSklCw%c5&Bm)soc75all|Qj-hPF` zZw6=O=RjhUl(@S_A11n;7f08WSf&JpayXOI>i4B_j_NNCg)>`Oz5uZwd~HCP^+K29 zTJYdBMcD2qv%t;xMIVpNW_?|A_`fzLSI%VA#O^|pG;pvT3e;MM8|>2TYqBcmin5xo z3Wm(jDO$sw!D@YxJN?^~TB@G)%&Mki9&V)k0`B1@$VGvkRUq$d!r!|+%N@gfo)2yD zJ`y^N7a%r>7a2k^$W28>g@kM+PQ>H1-h+N`*;PzKQ)_#OKe|I3`dmVp>a~5f*Hhh@ zLgz(?D`oC-HAd7v9(7k`cu`$KSL9i%5UliX9BTfOW!#SQ)wJ=Wj#c7T;(OfRAT0|G z%f~S=uQt?JpUFdve*uXPtZFd`Q(*^B52DEPJD`F3pWIXkzX-e%ha=Okj z*5dq84a)2pedSW1=8eV7?qa)v6*5D~JPL^7l81Y;O{7&Bo&DMZjX5nk2uPTqv8S37 zGcmfFt4a2$iriF(T0bSRzQl(X_gSyrdNsPc#vW4L^UoHgP0^alHIB|b0J-?AdY!8= z^R=~18A-GWQXVpI06x7s;9Dh3(;3vzQvdkwic<}m+;p7qajCg*%PV&e@!VRzLauoR z*b@-X8Hx9;an*VoN)b{gFHSAxUFW0 zRKTh8Qu8_QH|DBxH#(2Xx&x!~+U}sr5Zh(*pI@`5D;{7}efOqmcs$O7wfcQuD-giq zL?Ee!5{Ep1%^VAQ6;kh|R80cjBiK)3EY>60+tg8dYuZzhsyR{gZX$IkyHhwFC}fln zbj{j>Hzt`8N?SkAd4Us*Bb51rwLg&8`5q|#pfVU%y{JrVB5+s~=w59je}^uX*%-_| z|9J@ID)i9PW|c|{9#3xrls7_ia8q8Y;#q=qw&Rmbsj4F;UHLGV{_NEu7Bfs zx8(ZHV@wPV<2j{GBzfLM)vLgLCt5n3M>)=K*T*+6i#(^&kVE925`_f>Jni4Hw5Sc= zpG|+d-<+g{@Fg&C;^fCkTynJVNm)-=6FkV@dAyR+JwA3jZ%MvUkG_cK**_WN%evc zFV5bml}f1#NQ(D~l&fXE=Skq~%+2cjSd4YX2j5WmPcyf74F7LX(&A>NpFi&;2F9mpk^5PgMNUQLI75QRAkQ#WCYi{*|6cb+F zAsewvdzXbUkHppO^+*D!w5~isd9xDYpCHSx;2#PHZYz++8}$y`a*$RXsi5FK+UouN z6DeV;qssLyS}Fc#&!z)AdUFHf{g1v!S`UuUuFF&e&Y2Mmm2H#q))!>CVA1up3@ zUDsYRx-O24(zDR+OvcDr$$sk|O4bT(x1n$VXVCA^&V!a!sq%5@zroJQPA9-Y>(o@c z=qc{jb&MSq=uWpdRUph+Gpd5ncv;RRMRA^&|FWQ=-N5X56|}* zh-8lKQv63lK->Ya^dQ6w?L13tNOw$&cG?!Y zugMS34;UztmK%UKTFT2!q^HU7e z`bH_FSuo7kG}jRhevWvHLuk@;1=7O9S$mJxB$HU45t z!QPF&T7g9eVYA}Wth{pei0G&!<>yJaHmvvQzh)qqfYGP0-w zCL%KYxNLGqm@ZMwd$l-t8Rqf3JNpFP{2A<9Ve@B$s&he1{asS>^0};La&*YtifKvG z!hUaD5*xrxjhay;CM$@GBLSZhdW6^ORMipmiACFeJi{krkal)Dbru>X(f6fe_!=72 z%Ad(e@`?EwCa8U+d0sfJqV>}X#!f%oznmUBU8R5=`y*IQ1Ps|5ZDw4^ z9BfqNosf~KzitSDyQt=feL@4T@BUhqF}#&Urch#jgRzja`~?KCK7@7%y*6mhE}8Ef+0S2uj-$dXtm?xnbsh#rvYJ zln2jsBg@t@ji_vA0Vmaf*%YAjp(dLyssekYBHCf^P2j^4Y$}Te1$Bkm?^~7QquOc{ z#k_5hZ1Gt|ogdc57%*|{Drd_^-Ckzjvt^mp?w$4(N*BN5T&@9Q&B&EyA=8Gfx$@EE z@+cR@f(bZmbL_Bvt!~LF7u3MB1E|dZ=K20b&IrWbH~X5JG#bq&srT@Sa>_@vGu$%4hy0ezWf~W) z(33HzI}tmr=@9nssjRk}8(g9%lF4j7gM_g^5>bs_Uahx~>~!1RSMF3W?JRVDpS|0S ziN|ytHefEQ=;!&9V`5`_(jU0;=F%)xs6x-%e5wE?E;Cd7p-fvu&t^e^Zd)xbF|klb zKfmyU0}AT%I{s&tDHML+K>t_uWr-;@(oX9$(X z{Cqa%`;t$G*85P1Roxw)*@Pe6d0PwwRn?mAK1V{m%6lb#M#G;kqc?UZ|t z;*_0AawHnI-TRgLm+5lsi1+dU)3=CIIc!CbNj#oplVC&6TYQ~kd@CRb?6EnZy|ZE_ zQ1J9%Yr*dJ#U6w|G*1UpFUDDQiPKCj*?`@%ltT^>QB6;F`t_}x(O&H0OZr(Wg5Bi_ zO(d*&jvfv~AhAHcp4k?;gvhk`i*B5wg2jTiia4W0d7o&W&xp!}MX%M$;uYw%XL_m3 zXyNrBsIz146^t*>yK*@v&W^psAFRW&^&m2Zwud6rq{EpQ4HXSn83M{OKx!O~wY1DO zU)Z}kdqE=k(Hy^5rZGK>XQyv9vgL5Sqj~O;5R1e%xqa4haL%mjpgd)NuqlRrHCVsL`X?XG6E5DnJBVpf%7$M$S zGXKORV!7%qY!F3!!IwhHcf0-92fGJnM?B{ThAw*SAH=1Zxn#&hDX^ zZQ;JAq$n`ek=TU}BD{b~`Wvj;MVq&2gVIcJJfT_@4(0wHnRofM#CN9Ko4 z!bV!S#`P?#3y?ApX^jJ0*DTJ*L!;V#qo}pcMyx%I@0EDaUH#?qHWe-`z6K@<2TJdSca z(OD_tVq&U=1PdG@u*ahd11H-KMK>@c9v@wLPx`?X(uLO*c82oBb*25BKO;^Kk=x?) z(;HAAoQLvFWrw9nBm*_Qnv{UOJIeV*&d$#)p8DI;(4o$L##rOjl{`(_dxipx(# zlX39ZvIS-i7tOALPC+mPr96?Bsr{Wm#4sr`%q#tFw0(3D6)nNNo2oIctTIni<&~DpUNOK14Nm*8*Gfi z`pvcdAQ70sEZ~qi%Of-)Fq8Sx91U)?I{vu6-BwNcB}6F%9_N8>!CP=Pyz(EVN{y z|FA)&N_SKYgokshi={dlATRR2W~FG#FU&2&Yqr=AsX6OpXTNd!(eoTx+?T4f934e#Bc}FQ(H_lkEEdUc-ZT z_cJS1>ZgCun?Q`2;t6?35rvS`-;@*o9;|)~k8vrE^KqQt>4Lw)E--MGk3wNBSW(x1 z|L5<;Jc{WpMbNzehv^<~0UyGYh^voev!iI17Ih^tH3kssc!jL0yeQ4l+-6%i!cpt_HQUYSS z|Frm{;bUZ24mbpQC~@-oY`c_=0N|9GjLub6EyLbl{uObOt&}FQdj+p+YUx)s$~2Az zsQmdOb_d{{SZW?OY|`Wem_?Z3)zGB^Af!mwX&*mIag~UvcHeobfI*6zYN_phk9BW8 z*VCJ<=htrr0LBm_P=k>)fr=5)atQDb?KDnnX`(SFPWp@cK{zYBq`rQg@uLIyG(&dyOT7M zJmFz1h9kSLN$7vcWwzh(XXrc|a06SjG7u1cg9{5yAtQLxrSjm(#Q|}d8e(Od?`XJ} z)1NcjYJ4gHgSU4Tb}lXe3q>w!^+iR?Do<|9#1*43jl_U_`Y@q)2?LwjrJpPjpx~hn zzjIo}O5nM)aant^Ik?-D_v4M*+@+;C7?1tDUC!H1>|md%BYo24uxG1`DX~ z=xi0EM7dcsYo5AsauS>c%T>31Y*tzZPv$2&S~_IPj`%zo^VouS2> z<_rg!n`bGvDjUhYAt6Vz*U!PRTGX`^reNe|AZd#Wo80C`Dah7 zSf6{>YM0`hlG$2ejTOshZvM9P^^KKZ&DZIROOe^cg>pyl{i?equE)qWTTy7{meWC( zeEEd;%ofp_iAX!(w?3tzV4OtSefC^mH=R!>`T8WIp+$g)5Qg|f;GbITU|O`1flV$9 z{_0^~g#DC-4{g(0QUX6aJ8PaFqcGW{+o!oHjBO{q6&i_6k~&qj;5;)PiU})XzOhl zyC230-pAUGvokv17}%cdRB>;0;{Y6P`xf(|X6A_uxA5)u%d)R5Y%gs)F}Y-GRV&)H zGU%l7&wpCAD9xsr80RX%Iy05ZN_biU&J9TI{HW&m%fgrvn=9Ze_*3+!7;Qo?@evJh zxdbnENlCt_!yjI>eK^@Q+A*?IeSW~XG!J`tJW{{T2({szIOu*k2= zCpZ<%3?v|ZAHu`+tFwCD#DbJOV;SoMg_B+;$qf^!11nbW3q zn>-s<0l`AGM+}Dc`q6+aeStOsn$01@`=)KpIh2=|!@hk>7{_5dl`Wl>Q%g9$(tuok z@;1p`ow&6%EjmFl<>DOHUcYtBuPzFO&P%MIv`XMTrq zZNqmnYGty?J5fn?+TGtO2qfQr0sMM7)qy#C!0+G6Guo&x;F^)PsCUT_pobje6oUtL zL`!kan5E3N1Ag$`5D?R&VdF-)u-D;1NdoSJgeu!) zR8Nyb0dx05&7K;=XNv|flcYC?WW60`m*5*j`{8JM;tdab`s86ZlGSyMdreIumNgAH zyFD*?21~c%l{NT&y#c&k_m!X~y$VCKqcXx=F z>9)UKtzBVoGT%2)%_PrQON)!2qB9GXaZy36dL91e@a{$Pd*M*J7Z0QMuTw)qU!;;b zxh5v&;sM4=YHZ-5nzqAaY#aDl4s=?+svhM2NkoFxn%TUVE)8`!2R;z}Je{2Mu6 zO>LmYK!@ct>2$z(zlvaH zi-~0dAd>oW(tTf*hksPDhPiKp%20ZcgWivSBIGF`3g^h$)MF%qcx0W-D8}KoKm0Ur zBc5vuI4+UGUcS|lQQAzypr~jws75oseoD~BiJ&@Jj;PL&kgB>~25er-9+uwJRDIGK z7b3$*tjGYR$k^Bi-~dhg`sykKE3nYtiimr93OJvx0*oU=!*TpEZ9o3Cu+f+)bXQt? zu5ee8NPWM3^Qf&q)I3RQb3)M(6Ov(INCFqpoZX<<^v1cjNN`hDGu)kwY>d;VsnM_&5=HyrqwKFy^RKT8EyS( z6Sp6vpH|+zD5mKk zOmPi}-JH5|dwGgt@dTxtSomRgsWzT`3$MZbCVsFF<7nMKCSmEz%mb#WjJe@QtW}$udvPUrtax+vYRe&)Q-py?8J~9KdlQlZd7n?jtgX&U`u*xIUB@MB z!u3Sx_zAc!M!$}iNG-ALwo8vSKtI8;8xEo4YHv}w0W5xrdlBjnxQB$-;CbAdF;+*- zP-E}h4YkwmW??wNh)Qw~&(y{y>ddzP173H9`KQEP8c1G^zUgrVj#*f2xr0MmhB-@2 zA(B zN6xRgBFOXKoK-F(V9Wm10ee`)r?rogZ-Iozc|}Diva+%z$g!eVuq#FGs+8Q7O0}ff zKJ=^*g&f$QHJ+MFZqcz!0=k`Q3z}n_=o~Y3Xy_NJEj?U2J;nWFL!?)U!(!DC)6d zbTzr9fE@#D)#+M1o4%WjoP)Qg#O$UWM*)o&T_=d3kwh!G#>wL*A%GD(4(9Jg;X z=|zf(9_yiWo94)HQ5Nio%6^&Tp;b zj}p>EwC`+ zq^-o+vARbb(7$p0BH-VB#QV5)zlOrV>dniuNH-~^emc~oqK#z$!s5NAR?9tahDZjv zOS{H(b`7=$cVX;{MyPwf#^_6 zjIa?J@@=r?m@|=Y`%G@Y>>}{Ivm|2q+|7QXCSJ^_y(X`-r#EgxK?&6Kd_i=509Mcb z*nX$7!iviznj<-89EU4e_!^RC4O9#IP zL^=worg57XukF@dxHK>@G>)(`0LffocCK! z>Q@vtHo_tyA(<;FD5Pr;B30!(@c4;!io)8ftntE)SF8l(78B_Er3OJTcG!WVVPDB3 z_T-7bKSOrv!;dPYuG*Kfm!xM(j@!Jzr_#R|8Pl?mH$3imndT-RL+FHgkwgEc2{HAV zcQ>(ulzB${Sim?w?MKLaRv4##BeDt=pvlWZ7mX zEe_%jmj*~J>T2c$z`CCzIGmpY4t|oL@kwRN5#E!=+RLEkY%G5ky;2Hv$a2-0-h(E# z@}?;Hkus}iih+IMS=LG_7n0t8O?J*p&;G>4eP0)7?7VUI(sOj>~T#=g|?E^o%lI)ME2}#v`sNj)S*3U<#G&`7>+*_qJ z3Mwj5NlB)fwyQj_Xv)KIaOX*!PCrb$Iz7Mt*>~X%)QJP8fl9;YE3Z@oVrZ-m6=`5l z{Z^3U>~}Tr>!8}A==pnnOFcaghNaO{+53raa#4OJXrOzJAN_^NqI?+%wIh?5S8n|a z1Y+u18hMDVzl|B4`(v?;@JfSmN;NkA$ffEVvqtO=Kq%#xuz)+e9S}NcgFzO$=dP}C zINP2cr@_G1aRh^Ep1jXqZ~yS8?Wv3Mdzp&erKS8Mv~|y5(_F?DRO_!0k5+52jdn1L zil`+;JRa8>g4tsK?9>~$aCLBueTiehUjKh;OiNF%(tYOt_S&~aBldkyL2S@h^_MO9 z=PM!_nEC_~nyZ{NIMqkpv7z^{QBFZor)ScE%xUvpIj0d%uiE^d_IIYef#PK&9UUDl zDlcIL)Oq6LV6f{ZvszVrW5|sW?l#nk>ALPkV?)Gh*l4}!((iZspOr{3g26i)pU!o4 zfh{%Ih)PH_tA*_xcL#)_V!X3l=)`*^9{=puJ|$Rx78?X8N;wGAq(V%jTeSvjT7zb6 zG?{O{_Pfj9s^nkIG9d~Yv(h)fk9oP=CQx{Qzx;rIU%!g`fTtr_){F6mI&k__{bzFi zoVWj!vs56dC!D^GYJ#ephok5GYs>vR##ff zzW2d$#FCA>4N0Bb#6Us00z-HzfrxPUTBhP7a!vmzPv< z6n>nOt>Wt%e9_5-eNTGts<{5^zY)281>A2pLZ~vk%2(VG9dtYygT$LQM#A(Fm$*}u{XoZneY$1Cg4FMS z6c%0j)MjdG**n^>60*68*4Ws%9!Eh%!c7fZV@!!OOW~mP*F0*WWC_nhW%*h&GSYT$ z^aN#1&LqhkAMOUip;MBwvcenmhqIV%A;Db*3SZ2z#$j@Guj++|2+vQ=L>>!a0Z0bq z&I^R+@f#r~Btzw(BrTjy%}S8H86CS@i>tQS1Xw2L#j@y_teHJlAw)HU0)!i^mfli~ z4KZ|F?gBsp@FP`=3Hs})?6f5o_O*s>%O44M%*%WCiOwE$<9I^@^3l;zoa+21ov1c< z=Ul~Jyp$Lzx)749#YPM?91L*DX?E9*XG4zLX`)O-6YPigYA{p;uHaAs0p^Ox$S$D{ z5K)`sFq_G$bKq zN-ZKH;w}j1{-33W=njs#iOf-bcYpu(q0JWJuTkX%C%wlQ{zUvqTRYGh)kZF%c)9?; zm&eTpzjGmv9NFnT~MqG#=1+A1O9eV8Dsl)Tf4iI_!!!H~-k&-ZotAbw|Tt2UkRKzbqD|9xH25^$SZC7<=|al;aax z%#nqc1FjAB7F;)BSt*`1-PaE#@lUVij5g}`@TvX0c7Q7wh3m{bNe`SSlt;bI?~m#n z6C?nfWay(lc2w7*My=Fq2(;`bFPLI}TS2ghRnS>7pAw?&Lbu4suu`P5xp`I&bjg&) z1dko`xA@W@Y_qbH*smY()5Qc%i#>kS_4e17qcsKV84VXR6Cvmi2!6IJ>9I&wPAOPD zUyn<#(R+ z2?)+}A1$O?bQ3_xct{AyS-;~k!RFKS9%rXu#hbs~sk>Q!pC@pZr6={$7ZzhXPZ0Rv z+h}StUj;n6zz}3B{4q$aW_udYWy7Y|SpCSSM zBmCh%c)HT(!Y{S6qau#Hh-RqRG^K+gq${5a&6GU>KNud$y#*lFymHX7b}rG4@49Da z!J72Tx2=rs4%>36F&%W=C4)MRTYZik88SL-GSO}gu~vpYJe!Xy+Qwj66%WN@fhXlk zfmfNTHR>hN>LIzN$C2W>9{CFl)+c4A^RB#eb zC>VLSM|U|pmX>B|0$o;Y+D-MslhfN|H|=4&LhyNg$HIJtrVM!VsBb<=2_r}^#@EK} zvKnuv`qeHxMc&J-m0vDI5*Nmza-)oXM35#=x}mR?z6=GN&i3Y|SAQX&g_1GF9*IFD z`mnT>N=!mUkEQO5=sIFtuB*rq;x zYPF#mgMw_?tgt8~yKB^_diStQx9z?Z1R{Y{Zx!*Z-5m?o_4(;4e+_#mp_1OOJiC5} z<60mV`rP*`At9++whe&i8`?Gby0|PXn%X2?dHz#$ zr&-(ZYPC(ngasE9m*q=BpVOZ<7^s{^3^JFgU@MtgsK~=H7t};;b-h6e(rh!?L~Bb+ zFl;92h(ba_o|xn0wjz*^;Iam32QabLfnX%%_Zq2t*EYuRVKvSqhzoFugT>PEVovJeV%tcd z`YTq2%t)_^R-;rlU<_SID#Tc_6dX|V=Uz_#hJC=iroAje*Az=wd)0g}8M$q?=_Zbi^$-SD>Eh|`%Wtjd}64)SnS zD0_x&;VmYF?(%cPqyc;fSPaA9Bp&K?tj^hWQwt{Q*h5%qNUe0|B`}FQk^qYB}7!46I$re9gaps0<_D0yx?6xcaXI}O7%c} zh>v%r#2?$4g~na0fz4-Cg)&|m;*ie3$R|tnX;h%Ec>;}LMMj*TO)alaT}2$8d6+Tu z;E*Nt_%bn2E!bg>upHBFJ+k0%&On;{`;(k{xr2DTb>dfr4??TiEAfYl{q?DFhP1i| znFpseep@&!4MrXvx?~Ni$uixINRBO^luAjq%o*dA{zQdRiP=#g0w##M+@H=^< zFE+GkhNSgdX=&jP^wwa#lmnhUXHWj*5@siq2Nv25unWf1f@s)`4f>j}vxLcuRGn5h ze%@U}%cPi^iJLm1D8DlF-oPQcf!Ai?lU5Yv=R_iKovg zDN$@{-`(BKDlNr8z+(Is6yziwLsCt^XyA_?t;StPYEewjI&CH&DxnJa;3BT}C~9fK zhXil;nPrsrV>4F1Cq&vh1*_XleQ)^jDxsk!k7rB>3+ReL6x-6Cs7oA2EN#-bJX!|Q zV&B?FR_9a(nh6S{eHyN(R!_T|bB8Khq4=$4H%Ha?DO4TQeC;PwkI>5=8^$ZFk*8W6 zHfjrtIR?>8toJF2dE#BP&N`7OTJz6lblXoqX#`0h0Ewe{822oOl-e|iKe*kd`nVsY zB0j!F6IxK|J7pW!2b|8&n-q2mL#BDD>mXY=@JmXz7!n zy0?7W5B*9W6Z?`#gHUA@H;x0zNNT&W)m6evYJb z4~ttc@luHyiV$vto5Y)AvLFhO^*M_HD0scj{P;bN8zPm{t5{iEQ8Bb*#a&2H?S*H~ z<_k~%jWk#O>&UGIs*5JpECCO|2qpHi;ap^>J|#h&3sokW^`6HZJKG| zJ$YDgXsB|sc34>G7erFE0TFP&_noWMC>>`bCkj;!r)Q;H*cIV0Q!RlhCR7|=65hv$ zb9%q>D}X}_YKFSUaxB`zvdAOC(F))FPp%TjrC5rainTnDiMVQj)&uXo#?D<6eioA! zi`Z;!eV}Uq*d+VcFot(MALmP9+ueh1%g`}XpkdhBR#2WOLbFj_Di4ov9YAM`acjh* z;iHYQD^fSIw!%yZg>g;*h>>_V+1zI}z17fg(3yjRf!WxAIUV>=m{E+@+WOqD;ksEB z^k5Gk`8RAS1=)P7aX`#qT`RM+x>`WLwe6Mf4k+8)U5(LF?9Yo0AybvoWrl?eYN7>D z#oVFqP)wzaH7pHBmxtafYR%?#)N)(UrYfcDe43wA$4Y-FCgslAa{u&sdWO!Ik$Ob7 zZx$E{v=*|;Rq25axpSllxb6lYyWs<#bdtna!-j-+C1Zz{wq2L~Y|6*b=}64~{_XL0 z%QCH%beY^Y;JlGFS-hKE$7%3B^t*)ROE!*sj?UhVr{s`xoR2{DPMpSRfp#LXal_>T=N%2rUpN>O~TxGO{bEObV zSax=$O}XA3TJ%YnAn8^CJ?_8?VoM7??$KJG(rs=bwFY+7Xln(=bEv%&%ZLZeB<75V zR{sZ#(2M5C2$NNvRp7uNnzHU!y7w02PD)e^=U3wD(bq1MCRPmS<>kl6&D;*2EQKSB zEvn=Pji$5Q9BCK}@b+cijHRP{Oq!*MFM`lvWAL9x`*@t5%+pmf;{!?yGg#hH=y)4g zsXeK4Ss3Twbu>3O>+Ef;zv`SYID}-@7FVlpYAjnaYV>AWVq!x{SrIjM$76l6Kz0iQ z1A_wgRNd@KZqN6D!=6%dUoT2#lMlfudV9NQalvA+^`RW7=9}k;IZNKuipui4o}X_+ zbnlFKrOSr=;tpm!WG`0xW(lYFU`h&Xgnm>tJ6k-319#JMwv@G}{MXuib9U zz&d>14tLw@iO;PW&3)Gm?x4sjX@%3^Q**JIN2OuCJq*J*{;F=Mtf^)g&TqhVm=QTW zdpWwZL;djV-jgj5J*AW=Xwy(r6Mj0s7i%m0mly-103iy(c1s!b{G=irbF;I0YpZRv z{b|n!2Hk1y4_4Fjj7d|{$0Sedeb1|gZB%s zPYS$r;&^MRdh4Yot)M&EwEk$U|2ns)wNJ^*Sw7hvh>Qp`0kJQxN0BR7*Zr7Y-S#pz zA$NUzYRu!!=8u*-nvw`$RvS~c95aO##<~C$l~HMj&-lN+7g4U0O4TS!_vdFra+_Fh z|9#gr(0TZncVR9yGq!SL&cVS|j}ZRhO1C#P#nF}{(r$KSoMq!mvn=SXGbY22$uW)< z4UL2!+&8uV8-xR)g}Sxq`%TH+rc`tC4^dI_!e$@fe*E|tlbDEL#;|vM%-}Uy>!%EO z@7XpcG6Vu5J{QOV3zckD(e?$weiKzrfWeYqKf%6)$^fDe-qWb|svv~j2wPdPh1wrx zW8(5rQK|{k_-0#A1m9UI>Xw#IH03g>nQ6&%?BCmUi&Ua24!WxZ z%jtBuU~E(o($ZP#MPgy*%QDMJ=^dAG2j?LTGlq2Cb5+~gsz9AIS|YT1r1`rR)s#2y z-&ZAZN>V|p1J_cUFU!M4KWL{9W~fIrWblVGt&NEkxvq_M?=-3Q7Sf?7#9geY(M?&Y zm47BY$%rI{PRBQ#3eumX;u7^U7--S@+T$88)#HJfEVao+V*I0rYXXjhlrSqXimh4^ zprD*rL+pr^nf#j_I_2Fw@$_)Sn&-fiuCnj7O28-mGv>=zLF6ADJUKx^STv3M z(pTI}X|?={-WK-lXUGPq$?LvR&rG7Q8Vpja+SaCtPvvb^XomXVvQ>98@j~z51YKYH zdbq1Qpm`3|o%ZJsH+J46A3L5D^SUT|wms9M8!caNTY6mTaklGZ<}OR*1B=?Fz|4p2 zftwl6xNd@<4MmYqy4QD^E2H21|4)92>^UnoYH+Yzh>k1<;`Vi*1zCs>Bb%?aB#2;g zXRqLD=W0(lslL$8xDI+ml1*{=X<-2j>%%2HDZgMhHM*R8GC%pnMuoJ9*PPPW)pqFKcJ^F{|h`0*9G58ci>E55cnrXLd^}z5c_}D+ z=nvJvKzH1MC>(fpMSg`*j%`gjv0$YBuVdNy?G3{|eYC2wR7~V%8U^;; zUzo$cuW8)B!H2PEvVVX1yC7Tv;xd68yw2o*wC5FT&3 zj1DpnzzyFODNHkVutUG!unwf~xc>?bf^iR@50@x$58c2F2}`;XY~V3ML9qZyoLrqg zI}Nhp+}}0xPsW?Mp-Ey0WRQ@NHv8&54_uG8h-Tr(_zXA`GSf^|NBJ^sy2kmC0S`+r zm1tY~B1l$b8=L(wtHKg;ER33*$n)BbhN@NvHu7FR7ax%fIWH3l@BH+RQx>6@p9hx? zy)-RxtUFFDMe&*bT3KJ^e89EFldmnJ9v}JN@g{u>4216CkBl6Hg*_Vx?CK>(!i&^V_vm;PXi1pQzUA#z3uj|kqj>VZ)<4_S7?b`G61qjGS;AcYt5O&3 zZG&TiCCc0D>u*5Z4LesuR=sjqPL9aT+$=~N2iQ6gERgo1e5KI*!AyMi9A-AilhcKZ zEw=H^0zI%Rv`Os$XX!A5jM1^Ufvgp!FLC(2O~JuZMhqr4HWhQ9m6Sr#`CNnnle(uT zTB?nv=UHX3=zkIE|E}XJhTnTfBcqM*SW3O*kj`L?f|BAr)7a2Q9UUQ=bY9uI#Kc6n zux4JTQKtW1kFQr>PbO$T1rGAX!I2okAt7OQH993Y^h+ZkA^8Dpt>UW=E-vkv^+^5p z@Gm#6_enl!FT+~N#_ILRUNamv7gWPZPCJHBT}8Z=y9o}zo>{y;TMSw#;P6 z9Y->vo}bwYBBHu!K%|2)ULCON)I7%BPD7;k#9{6dVd&>+r6T; zQdsUUiZe$Sal>Gt_Qh8GUldB9V>&qu^br>qm#W;BNRBYON>sebupQe-`u~Vc^OXtn3QSg`M zj!ppdXOP}C!AI6PMci!;Xn{PBnpiT!r{u7L5N|(AZym>GZqC0TtE&{@y%)(fpx#0i z$RO3wnEY9U5PAxVi!=bVn8t!@CExXZ?e5JO=`{JEyZv8z6zYjXGp;IpoQcAO{S@3I zNVgsQ=n&9^Iln@eO`3o-I`6$|;&57cCJAzk%2 zg1&Q2uC_3<>e*bYP>O25orzGbBA;wbZT5QF$~@a2&NQ&Jq;~eakIwd-wi#-CKCRH* zjpOF#T{Ku)b)2ogX^J}EVB<2Fu6Q`VXeL4&ufPCb;d^OWw6*a%Ka>>OoC{iIEp|o| zx#*bJ4ni1Du~D?~_^kFh2%pFmN$0V0pix~*J{X|~Kk5kzeJi56EJ+|!Qo?$p<+3efr3>ek03K6q3w2dc1-Pw$U!Hpq;L9N z#hF_^l@AsOC%P3$gg8`R=CFy%_D`&RiJRMMdbsJDT>*2DN5F94Az{czJAg4iyKc5* zXRqJbR5G}Z!XKM5KN;d^QP7%1l%(P#XMh(eQdNnlJmIS>!{}?ecHK@$Z+tpe>&_|f zxgTflr0xK9uUni;Fd~$ep`81a9P;j~{&0EEPbXr8)CQt{g9;DdAnUf;ahhYfAraMR zChJyn+|^%4CQ@BThkQCHbM6dxD{2nelXDv_I-lcwoXJ}T%_$cngib{dCq-oN@U!V> zxA#4Y_j8kcRQ_TSlLorUee(z7!{r|u__xOPZ3ijp z^3&uSS1~Folflw5(5Wxa-G%@uAdeazzLk&xfC>0Pml~D(<^2MOc9rXT9C3w}{zRIoX%Jq>074Kl|r$0C#L+3dP$QM)s4|4B7g7p9CVTu0olF*kn`Cy zwI*Ji-TEYxd^kWfa=B1A()-MWHiPEj$uFA#x?&B%Zg_TBNvjg2i}av5BPj`63d0FsW`9cU5@fDvpXTZ*mXaD z+;%7w%GQw?$-(v({glRzk(2Zdk$?++f4`&0^XU^XpyZ{ZrEhO1#--}Kj$@O}pvUp@ zuE56YOjgfmKk)jconY!-P6y8)aK;C0*tePe6_$7^k$TrK($&JtcPy~}`KR<$w{D{wH!SKIPW?+3`eyi^nVI##2?MXnLO`KLL&mgF z$I&edG-!@oR+Y|e?ds5T`dJwr zkz~fDS@p$fdWMs@p|TY~@^zUIpD?SJXF;JqVmeXnv(u^cs{DuGFS7xp8R=yggU2rh zD0|y%tXRt()=gpp+Qg;vf(wnKdBEF_eBG^tr;?NSpl3@`kp3UKE*HwkU2Hg&9+2mI z3jmI_$0?hionJPj1Up=~nWZ-zPEE@+OZeXvyP#uQAHHQQYr8(BC+}Zy;ond_LD zOz^-elhX0atfDzF!Ma~R>s*v-JULH)Z-_U!GzVaxZ<~vRd;>gWssGFJLL>(Jj2;e_ zmh?FpEjThnd~TB7G$4QZWQ9*b^~)QO(ZwF-3d)Y?`|(vS)Rl*4p(M8^HeVy-Ah?wb z46RaJcT!F7?qJvh^AvP_8#K!nZ!c?`(n>9lFKJDQl?5Xc5tY%ovDMob;`C3SmF7&R zG?lZ^tv_Dk+g=8wI#~@ko&*m}psH5XiMym{mer(i;O0eHFAqcp@CRzZA>UNlHpID~ zv3*2TP+PfO*7VO$wX;LFPFIhCQZkO8#WwS$46ASdpkrho$yp=9W*ujh+>}o$T3S47 zC}Cao^+-w%wHr&jqCAS=Wn^8oMK8ENLD?zYzV`~5KM*FI@q0lp-pcv+KP=?mAR#r# z8Rx=S{if;{ep+L@p?|c}f=|)cM@>urB}g;FGL_$`Bv^X82$I*t!~~RxDkP>nW)$6d zMuvh)rb!z>ok-T(w8n{v3)x(9LHTaA&_Q04y1KzxnLj|-1mJpq;NOOax+T}#{Lz9> z?IXOS9r^FL{GJwIqn6y*qac^b(Z;G?mGSCa=Iz^?0_eTB1K!X}__Ip{9oCq7u%+?q zTP&BQbx^;6%VMLk?1tO>%VjG8#5n6qPE#mc^W{P+3VH=6C3AB?x=>(4|vp8Z9o{$-qx01XD9JsJ(=Xx7jG4ndAR{SGPr2rdC0ssR8Y+;LW%1+ zLg7ldN-n`kj#Jbh`nyt-e!5kEtCvGl8@zL_4v%@kJ>ZRe+YHK+g|Is&h;7j(&%{xH zR97!UaIudq8(11KIOE|tcuv-CcIDOfyp#9r?ip^gK&_Cc+$8?XgkPc5y#g3CJe+lT zQLgWuM&WE+@QSOBii?TS+R4eOsbz=h>}-nomS^gK1rb3(8fRkAw>KPt0)^mlUE~t1 zg!Y%=_A;@IXCtO?!jx497HAh~wZf%{ys6?lg-m#KsIZk$Oua7AsTK_^da9AKs-*d@m z8)<0?e3W(~6%os0{JhSr#+{OMz-0=YwUetuu`nZAWRwu<`*KMziar$(?9shXH> z5q{_;Furf;k&PbWS87x)C~G;?F{3@9yj;AgS=GU7z+m#=0Xl3O8O9mhsC%}q?I|h5 zwb0u*Yivj8334>UGw=XhP6$)Ghg`v5Z*rwI`lPs>R6;Ah;O&mmTtMS;PW$+yd3&p9 zGSsfpp|%&+RMcVb#@VOoZ6t%-o=VgQ5-qJI|3>PX!ngAmGi>9cpL5YkcL+~fFF9lS z=8PJxgMhE$DM1Qmj~Dq4c^1Uuu1EsDK+&MLMfBy?)+eCUC!$4!-9steH=q+<552l7 zs5xZ^C({>8w*8diz!#*t8wy3@Qf^L22 z3JMgZe6Ro4G*0~E5(~m5KZi3)*^4v?GtGiCiI7mD-A)~A7AuY;pBB&6ugJnek zgoP0#l!i|Tm5fcEXQ(!Qp{n%0H;1kdO>)`IZz@L8;KbU@#&-c}4|-2g-IFUwjtLkY zEAfjB%bqNb#jlSCtBFV~>xf8Cj_30aI@tPYjHn@LIax*E4Ua%1w~)M{DsHziF{iSZ z--@7|_r=i3ef*`o=Rrq3)DbCS)OBlmRMqi7=#yG*&FakUUh-eU&K1QLhj92E0f3&7 zM&#oUulVfQl@$UcB-w7Oik@!WNB@xEuqYw|=0<6>eM;Y06^YFtB)5q#4eNa{H-JX_ z48aS@R0433bh~N459yJ$s;ZNochzGhLw%!!Aqd}iJ+;?xNSBqC)R)e;SKLwJ*Ws9K zs0{PXvdXNnNO^5RDKoZC4UeeqpkO8HA%N)Dj~uICFPPgs_@smV3C&mf0@vCu%vMc(|(ObJ^TCenOp`F!?6QfQGd3t1g& zJi|u-riS^lwS~pyUUg%4nUVgmdM1T`MeD3$iLs7%*b~6ijk*E;0L)+~p==L@(NT>G zIoGpwaeCie`D1vHLE`iD1hzvyN26U;jX{FG0b%Q(y8>tB_xAT+?TBiKLfczhTm(T} zaok)Ij|Ha(2LWm|d%E9H_EkYvhYZ}Hs=c?nF(VcevZQKwi!F&j;Kg@JJN7Kav=&+i zIXDhO^>iou;2*n-N<<4MB#=$1&+-19%qs6>XNle1o6z=_`QKHVk7CfP=?N(3wG@(B zyK%!|zp#LvXh(uFTe`cnFhO}?H1JV#@Y3bZy|QQhP$KINc6Atc%{82Y1|M`aWvEx( zFC<*u8grWRG5Nl;ou#6*qUDI5Eg&>>ugwove(p6$RujDQ{|e zd_j4BxW}WHdS~J!A!Os?as5Nf^(ykkH!}+6B8={QrGnj-3`l}FAQ;O_cS3M! zA<|qS^(iVTQEV0^;3hc#_#WHaVMQ3|uKh2<^SjV#N|hB>UY@J#VKvg%x_Y;IrylD0 zAEx41@i}B!TSG&Ke(cPaXc;jtwc#XjxbrbNcE+v9GiFmW0ybx5=I0w2Nh(m3a(7qu zZATvV_Ff4?KPWq_z9laOQqeo#B|N>*kuk^(85fCMA@9L5V)eq7j)XDJ@sZ zNxT3qbQo?ke8<5z_a^Z|sS8_N?segfMv=pl|;jf;L`GibmaZ?)ut zlhATCGEFmvYkKzWm2_}~%*)6BCzd<>Bly%;>%DOb^V3~80tpHf>lDcQkj@(dzS%)% zX9L?UotG*38W2;p# zVE2w{d)?}qTjTK-f)#=40Moa>U!u0I zu4iSXOyy}I1Tv7;SErppxZfAAt`!s`Nle|k#rAvv#GUxU38)9yq|N{s@=gG~pp#pA zv>*Lj4!ZKRBUsdVjoy`X7sy|{X<@mW7Wf=ATOTyMDCEXLuy}~oh0}PS5IOD$0+{F8 z9K~3WLu`B05_=JOp1s{KAmUWabwLSO{}33S|AWBL(9vo=KQQ*Vo8{~khuZq(tvRjI zW1ogj{t1K<)7f}yM_lx#471RXY(h#J-nJ8;odhzto|Hvki8APX4}N&Q=L@^fR#X51 zj4edoDhww29+1{e#!Y%=e8`5ILcn&FTRQo4IJe6GlI@*Rlpw?^bJZGT60@>KX4U55 z4D`MO0|S%#m@>O#d3<~<^I(JG* z-wjdNEUUaaAtPjSB)8hnJ@G6Lsu$`0dF3a_5BK!B@Ca zZ;9zD5F_6ffNOagu+4eaDII%EmY!pQYi;tlT9m)kKXbn=v;6qT0uY>a@1CE&*D(;xBzii(Q+y2v)8 zICJPzc-{wJwmtk>UU=7Ub&Vt&m`PFifBO3BsJ52w?UvF)ODRyaxU{$ycXuo9P+WpL z1W0j*A}yNWPH=}n@#608?(SdS_ul*3Z{06{WKGUm=VaE*p0j86-p}*&rFJTN`)8u_*DR|6B`FdMotc}C7W=%Wr=i9 zJ}o9BKi^_QT~${%K#$igy`cfW(e>aL0@>(4;gCWG=!wh&XtjLt$mgXoVg~dRMiDDb zuHgd_Wnqf?b=XiS5h#1q&hEwqEu*>or7W&ud$J(7w3H*&--{o0>c0_bz9VAr1RY89 zLBp)}r@@=+>#zHD4shsk_a+Hw__ed%{plP$l2*LF?r3DZhK`8|(A1KWqJICri=2T0 z+cYPqEwpyoZLUym+Z+V1?+sof>RijBR@@t)qJ1D7>)Yc|U#xfkAoroEs z=^Yin`~BaPFL-3|SMtS5czm0#)V7zDFf=cxtV~FgpsLVzgoTB2K_{S3(bhI9gHezP z36s(l5ucL{?sz3W_;%f@OS$z}JGftS8Sa~$@Z;@F+Ppci>HhLvCHOuIKOUQdoVouc zhJ!vlE44j|XDW{Rn@cyc@Vz0;-25v$K`IQzA)+-gU3R4I!mJnzEy9}F5wB2@zHt4$znCY+vt;Tah3e|1hZvs0GeL$djWWmJs_N4F6qFIy;&z? zy3%8s)AE`C$42Kmq1f@WGfSrk#lp(|F6s0?s*K+ZMB0Ramm?D$7OiZZ+`GI@b)+GT zkEuq}uirTNH|AZqr@*utQzZ>PHa5vIoFWR!i?fPk&ZA@Og$`5C0TX%%;pgCS z^bup2s$6c^s01MfuoS&~<6~_o>;3ZX;?PreSb}OoOu^EdF0SczhMKi#?w}wHPcrBj zn%VkF0@yMi*O21hxv{eC!uSw(cN$5@azO@-jzIpPowckOE^Zv3#hN9chOuiNEJw}D zfS4zY2+r=oA?QG#D<7i1l&~`+ZWWRkB2>qtgWFT-C`)SWcjMVWMkaCfgadp>+-V1l zO3%~|4?zdf>#<>DWIV^|v&5Fpt)a?W`#nOA^+P@U81Ef+RgrpAarM*=vnj`yE774`#M(%IXr-LO{4;_n(o-koN+Y?=v0mueKhjsi9(U_3=%r zNb%U~x6`y#6_@xn{65pke)DZ_RIRK!y&$VZH*4$Man?SYVZDW4)RkV~;+jYBK9&qj z)d;fdsEG$7wd#1mfjqy5LhmDX!|$86`)p6NoE$hFwCLk%u7PupoS%71v2^01NlWwJ zN6IrGUtURwI3wc>BtJ7h`C#&|4Mv&#wmH1le<%e%SetiG_))TvmnVM{r2mwyCP|!)L)poKk)W zc|f^mI8yEqTE`Dg`GAz*3z8Tp!vP$v#kP+g$~Zgc!g$c-&(%1+atq8gG|EhKsP3um zSG_wtE0}J{n$Ol>oSumm9&IDlb=xut=g?B3>B#7|orBI!^05*?MVvhiZ9eY;T*zCR zRaE`p>}LErAOP1B;No4lOW#KfCR?tF+(GXJ(W%SFSy5*od zb5c!{kL-+H-jRD77b<|uWQ~>NIV!`!SoCmDt&x=KJ)~ZYws|_Vob;xS&pW@0m`Y^w8FVD5+d)~EijKl1UdIXZC1?eC|<{w z{p}I;BX#`wpiaKj5a=QLKQ14+c`u+nhJ=pj}&W^UQ5a4k815NwD^RiHqVb{R>I=b$-_y*@f zxdW!f{IpzTz|cIk-hODS;Jy_)!R7LZ72aI@ukG+KhCOLWhH$!cM`2uNU!SUGesR8# zf&vFzj!2DC(^7isi`@gLF1Smf?#=sc5Q1_laQBeE&AW*RUKN|jA&9Kswo2;)!u|)d zLWKVWn9(Q=UhxU1W#hdaLJ0t+v&sZuFC3$JQH5>tIM~ z)aBVGtO@Or1OLIs=3CR{YLsYw(tmb!;NNdX%Q~Q-aEhz0OWa#1p`DXMsUcEQ&MU67 z5CJEWi=Z^@#QAVkK##n~Hqa|mT&R2H)ajKvLxq&3m)svp-r3wQ%Dx%x?D;f8CmCPH z?UCDpE(%jpnxszrQTQO}ddE2bFm4SeYP?@PqK*}!*Pe_itd-#d*=qzQ8$W)?@bmqG z6e;tiuHcvXc)(%_zn*3zKhW-TcHQpAT1deu7^#j_BgD05uCHuw>;1a9+ z!Cb{&+52V=J3;!6TBhS+XoB|J`T5bXA5(%sHo!5pxumvH>&bC6=dM>%%E57toDI_=<(y?HM=Y=q-ujq0hZuQ|$`w z1yZh)VoD$!yJ_$0EavJ%b4fUeqn`btcXx~#cr zjE|=+peF(BDcjkq9dc>0A6Bu9r4<7u1h)LQ0sGTJpgGA#|Lu4<>-Bf$=0l3uwDV<&VPfy-=Xo5+@~9pV5xZ*CvM3ksj&wI1s( zqk1SyLS>mEx-i`iCA5?%9vc}6w)C;>MhzOI9%+laO=4C@H`;c4o1{j%M}oqOs{F2n z`k6YpTS#$>wOPfHy>i>UBJE-K>+InCuL3nU4?w2uRyECAs1NB#hqxN_tP$a?d*WfD zR{rz&(S1Q(Hv+hdhwrGDiyp?+#XR_7OLC>B(Ty&=%T92h)08uUvr4*%ZSC$~ph9ys zM0RJ6#_fYyK_IJasM9H{)0wFBF^V%w1U1p?OKI`mr z%aRW;z%$;&B--8zeK6QA2BZcakEDH~Xhe+b=2@9SBNj?;IRuJpm*_i9gQxvCFxKzv5X=8aO^!SQ@qTN0xZ#_B4E{lB3me_2ZXP*DL!kTlakW*4 zvD_-jZHr{Et4;Pu)XbJf|884yo~(W{j|orE)`359 zegJoDdOo74bIW8SHp%_6xk#R#A&0Bh#V%Po0aNep2|AbEML<9hW@qq}a!V0)f*6>i zCN;!S-OO%F5&;*H!R33nDneBO-SzqS3te4<6I|%R0me=!i@Hy7_-kQK3d+T(FNj;d zAJ#`nJVh7Tp`o6Z-&C5+NQVY-q$$i>UBF$ypK+RY@VA|`&Y(Nnpp#pIz`a;{g(xBi z3pmd5i;9Z>aR?lH6A;2BPh=C>mia9ZF^KLt8j~bqKwV+`ympVGX2s52@d4A31;e|8 zkYBr(1KTI-ci@wl$saXc{aFwN;zAoE#LQFt9em#AVyZ?NbE$RDN3CmT%N^ zwJa_-S1zA_S)*)si7zv$#2(v+mmZQICNuiXn}Wb!$n9W(&=V^#Qbl zJwBh`>6weDxOLTB$O)Cv4K`7CWotE5Ln@T1tZi_C#ZXHYOOpN+3Z%k#wJhECnS{=y z5T^=8OC|JdO}f4=G$;CbkXri*6BRV1%-uXZK{BLHsR5*L7Pyh)v2!iw7UFmMfwz$* zXs96}#DEJ+RxBhfOalj-*rI~(!S}7LY(?WrSdNwUj!YsQ zeP)ELRt`=yO0Bxq;mz60f28^2{MPh9e3GGPqoeydS1oZ9-pbk7lV?ZPs>WbY7a<&j zstzf)Pu3iYARbWDRL&_*J-l73hu=vk)&Ry2x2QPWEK1P9-ZIg#HeqJeYZlU91+2WW z(p{f5@Ve4bb7{8EU?aI|8Z$TS(VY`ac@@`qB5*!6K+a+DJ&kkG>z)bzC~GvdOIC-8 z{qX8Ny)q`f@?RpUaZW)mZ^F2oF*i0sXm=!0P!HP^z>U9m7vWp=RxymH;U?$H%8r4F znPaEiQlrD9p&{U(M^Zc)_^}Jsx0Z~xu2&QX2?_jC5Bz&P`KKq|;Nal*NvDRIcmCaG zZ=Azkw?qgaXch2WsokJ>!V4!*o+&5FV4cia|9IeaY5Cp9Muw)B`WrQ;w#E{s(mFA-S9i6Km5cs7` z^GYClue^$!koFU=+E&}%x^otDnjyo!sh&P1E%$WK7oKBrz?S28ZrO6DFxG3Jt0 zuRKFie;R)bkoN>B^`AVGBU7b3F)Kmx20P@Qw|)gUdy%-lTNmP#!pw}LohqHrFw@5R z)U|Qnns`;n{v1+Sa3%W9eCQ)GHS^MxeER9B){XL5N$raGsnSG|l=u5f5lxie#Q8O9 zv``fFa`Ev7zJke|pKyp;db7ICU4D%+z$PcZ4og>P+-dll+`nQC- z2Dd`qpP)>6#+8lCRNUSA?-mQ!jy=^QO{UAlS!PnH{*QFqx zO_sG50bJK4J^%Rzr7+SesscYEo`s%k?YiP>vUqo}@HY_`sE% zu1KAYv!07$j(By4pt)XHHHK)G@$`$5pu$4!A*b(IHIzPp23sw?*`Kpa>U3otbeTd0 zb=&&#UQ@|29YBY9wX^=>Q#Y{KuTKi7^1c+VhqZz@c5LTs(Oww!D!IZmx#qd_eOte< z6f-m?cOW}$bA1Mi;sR|q>tWHBeI>d)>Y5@{VTvD~qr2Y+Ic+BR5t^#VOLoGXKF>hC z?UT1~DJm3!jtPJ=0jkK}vgE3buf^WE$BAC|Bu5w|ypVg?T?julQzG;Hv11jANv570 z6{{oGO3#)l{*}n1&CQ*J>U3W~!uE~Pk+DR!io3A}PH&o*lF`aaN*Hj3CKb{s5QmfI8siJ-UbNJ5ZQLS>QPb!TYwo{y;)-^qv$&WW@1 zn}E^KsyBHm1*RuGEwk~7PFO#M#64^5H;18a+gg zW2zJHuPyG^;;R%zscmf{`7L!VrM2iD+-j;5BR!*Lt%J;Af~Vr&o+ZU%UH}v?9W@N# z1HGr7M!3QuPB`Wo>sNkxcbF_K{t3Hk^?RFW+?4w78>y5e|_{a!7gyZM8@nk z;{McenPFqcWHmM*Cdm;Fo2b6P!rY+AoYLl-@?v1_LW^W=H0U9CV;Dz{O@ILq39cz>m^$6#Ki-E^D}vX?wYT~{N(iU zkoyI+ovm7Z;UnmR%wDI`p34ld;L;gJrfaj91VG$c{z{_9aN-?L@~}2lr6dM;SROm5 z^4ar=8Nx8&OUprVc{;odE_w#iI2oSW{KE zT2y2=>-WIICfca!=~dYTb6{XVTJ{^+#hxQJF#BHTQUv`l-7FV}S3Ah2z~m=9-!=bA zxS5&&Q@krdms;hG42lY4)#p=?HYBpqb@E%(^N$)fNtQfJknmC?*rPTL~uFDIUVfLcCu&9kjE)8b0QJ5%HDi$3*q{EfT>_v?5HL zu=8SQ2qs@^04&!T#?-GopRBn<$j+W`B5vJz)V(^K0i~@vQ+FR+AI;og2Epk-De?XOsk7}YiC{xBd{T;Rb4B1tb!U0GH6hDH00OeJFv3YsuA7XDsXtjor$tmHG4HAm$&WBH zZicWZ!hEa|&4;_Ptf-7Ws*t-WGcQ4Z+Fr%61~-&#G5zY5Pv#e0y9riroa@`}4pWb} zccma^4cuALa-0kt-F#6q@g-`k5?3Y~~xz+IrlLQEv_}ilIG`>&W zLqZkz6Ak=QG4;oTahYOdb0KX-v};j^7}RammcdDeBBU!F`f>G=3d6&>743)5*;QUP z>>t#3jL<w)xLilP80n4fyf%MSk{GBlI>?8z2ctr3klC;YucJ@gAPkdbB#X}YHtLv7)e(wo-~ znT{T_*H)WcJX3tc|4ourK?4 zIQ)@;Vd3Jy(ptBvJ8-%?+6ftmX?Xifc;Wf$OJoFX1d48|kj6z%JYVFBDNmo{{1QH6 zLe>daoUQ`g6x>w1o2rnoC=L`E*Mo;@Cuenqj12w`CFdcKQJz<2ixNnAKDZX*uR8q^ z=ZME@wyER`P>l0#Y12pZSs$J)=qTabZ~Xy`1qp_NCFGVtDxKxqtdVVfCyA~NsN@#N zdGQg{r>+{{8YhAe1f5UMJBu_L^>g+FY_gJgm)LO4BD3Syxr|rBA>MiNoWEKLa}Ahh z6VGO*o(|shlj62^7Z~^yUHG_StmH~Q20^(UWopeYqh6fvpQ(a-@bxZm!OYeUYHN2x*V}V;T}O8t3_sNPgfte9W^%q7W!O45F(|E4g2Jbm_e_?BD!p^Q;Cfh@{?`2=mk-rJ0a;+w_ z4`9_V0zVCLG13O}s(_*(-X;lT9a1S5+y`OK_{`L!T$eQ_E3Op7{?+p0J(iwVIilws z{&P^ZDm&l@*zpw1nX8-6v4n%mBQr|Cr~O>62XCeuJ#gG}Fgh7eaV<9)x4G2QQPBI+ z@NCIt=4ZE$$s&%{zWvayin+y&!6n|?jn(FPWs$TdW6w!lXL0;M$war8y`C2zn3_=A zDf*v$7JXO}G*xW|;(ubuuMKeu7B`=r8akM#-vuv$7m$09ipI^HC)xuMXtDR5wbVs zhRMp8=8uM1HMq3C=b{@^9J0B!Laz2tjA5`Mpv z+7}5&eauM=!D=RLlAao`)5t}(3^`M@E$j|=weJUz8ZKuWJR8gov$}vg#T$_3uwH!a zI(X%tbj%8w2%`L%=3@3p(yA<&cy&qb^J3{PEb&Pjmh`jbCRCEdF6hHp7lEMpB{}Sl za|i<{#v@HXdKt(UvvF+)o4ct`h+`q)`@=m%9^1#t?$h;@X_teI|}C;^Os^SF5Y!+7Bm^ zj6iKq9{J?>4O%C&{kjuXJw5r24byn=!^_T&j*xHYWD-fB)~JYxZ0R_v#tUJ%GouO^aC6WJml_IBho;4ZbW0A+T zsVy76X_MU8 zssB#>L;r1_`0#nr82@elD}hVkjmwAgES?;T)z zFR61zEic9nyP7;2&fDsQcQ+{f)+x_%4fb zz1U;D?!y7i`RyHnZbLIF>}k{ zDPZxMIc4#hTBo1D=sd+`M^*cDZm!Ds_+$)dclVC}`nsq~KS-BPLrf$pC&d1YqP_jh zHoK1rYc(x)R}4J@w*weuP+&yT&U&ljC}J@Cv5EbLznzBmb3AbpC~lPQdPYL_S(ZNk zTt6DwbxLcGoE#8L1KcaC=J{+?j;Wbkc)B?U zrTNljw!#CmJY7S}(r0aOiKC0^<8Vi)7S@L3$J)UfP%$oI>U*hu5(O0Efy^AxyfjjC zF*A9-qrtmpClB=$G;GDeP@(RFSE0uFS4H$O{?5~QYXrtuv>rayH|A* zdqICMpR%%JNVfOF1BbBWv!CX67d6-VAx#XVKrpu#VLsiE<-8T(QDx(Qvl0#_Pn?0C zNvMd$A3bR^)>HbwM-m&IbvrE(b0@B_cRftK*-+Js{9w`6&?PjvD99h{b1_v=wO_t) zz5$YXYZ%qZt&m~JZyALCmMIDR{D_QO%n^~ILwpy3W zwUhe@t6#@;9%|d@^5JG~1&ExlFi}m*0%k~j#7MlF$6PxWMTG9RRt<(>XfPqB=IW?v z%xD8L5zUs3M%)pfpRqL%5nCN5QJH zogh;6a9F$00@p>o_qv~|{Zh>p)r+t_f|=75-r0<_sV`b<04g^?Ad0?9Fc``h5gm&Z zLIu8CSCI$rh|dl$KV%OMZT6?X#rW}~QwyguI+UXJbT*I{E}feEdF&2D@~xRtxlnP6 zfz|EgRM+r>AVj=Boqg;v^@VNwP}A<4+RtzPutL(|_Xr~CaZlxz)g?b8ET<$dST)Y9 zfJJ%?MIMjIVSV>ToXCu4d2DN1s=5uI$;rL6tLo@$`Nw8ac_Vg)($;;r?SLiHoSaC3 z%ONG^oftFM?hae(c1tAgE^AnX{6>katgJ2*WxhgrRKL0@xM{6a)4t$yiMeOoe1OKl zh^NmE#C)npI5}t{Vgr$J@y$SDaT(>o3nvup&N52T7q^7McA`iM zI0T-33SrTdJ9Q&@9NdAY(>-~gHisf?1@9gZ@mUxL>w+2WK%XYk1OW5gTVjc4Zi|x^ zpoYWZW7T2~AJ_foNvZh0#v0J)4gQfmBWzk0wCE7i{Q-_I?zh>rO0qgm_2$Wl7#>fP zT?a_pvD_`3xqRg-G%MisrxpT*Ng#F0_e&JT*8Jc0$y~JUM<`E#Kyxe-ODJwDLT&`n zsvAh(80hHM_FE_AXURj^v3fslccS7?Nappl1wQqnxxaE8(Np7Kw@3@t(FB_V(}Kmy z>#31P!7}46N#IpzRuz@l7vxI?BGN-&!uVvV7qP4R4TuwubAf0hvP>;T5lQj!QbmI9 zie3)5IOE*Rc>fHGd;=lMJ>lQ3h10zV;OghSo(TMbIzA$qTNq5<08Qca zyZ0gRclQhcniI;e8rO-T;d2vr{X(UX3!$%62vd^p0_TEl_(Z2jkVK8>bDQ!{UO-)UxM$ezWwo~97evihZpEw9msCHVkc~NOLaG$w6eWII*oLcRCJ0{Hg zm-MBtg7os>U4pWW4gWx@K&Zq!`JB4ukuKAxq2`9Jx-2@H473G`!&?!A=jh0(-*)-k zuA&dIgoFFP;rT{KVEhq|X<+rOG#|oQv~Nc>(8SZcVB|cIwFH=JB-feikPp{KFT+0K zE^4Z%I5C=o2D7xZ;&6aRe%gQa;5qvCJYt*J@Hc<*+s#;P5}@^Z8x4i!$%|TL=kG7X z+~p8oAUk6sY(;*M?@KeFb)ECCpj6zaZ0zFrBV4MT^6eM`ykBnhCp7KJv8monRY2s? z4;eHbwACw=p+4&}%au#I&kd2|k<~48E7voh{^LF#VqbfVq>EdU6gQ<$c@nNoFO9@W zhg*@o8cyBFf3=a#HtaED|5X3-ko1b8aPN1&YA=r+GD4hlSsL!ovHI89d3hKu-~Szt z3W%G*)zZ>p3ORYpUu3CU;NYM%z^MlGzP3IROF!fB&xxfnVTn%L?VGSZoUa%3xZTM! zKtn|(TKP==hph}}ybcJs%noPl4o-XbAlkOdgXOyj^TZb+40n+FmxaI^;9wzd?}z4> zC^+*<+S*o;gH{vub%a&p+QQ`H;=QQHU4w8j#$w$N=c!8k4^?_!;a2#61Wy0Bg8c1U z$ZUhzU@G^Nz)1&YjuD<{_j|z@?_V{3Ie|P@5{Xvp0}h6ZYHH_N4edWu@xHL4 zJ&P0P@G115Y3|9J)nL1jO^gF9`~26m{OAAcr9BI`P{2g7NzJ~_ePgej%1gJ)J*3D! zWK&;!xxq(uQ?(f>BMYmluXfoL?#8SZdlrYXPSxV>Kj|>ep-TnZ;9{C9@*Z!pSe0=k zOnClR+~I%c(bra5XjjZtaWb_o%0SsY&W~`a;GzP|d=nUzWLk^zUM z-5WLb??H4+o5e6{8R<6??S-ja79@Nv+81%=4bk;oryOi!(d{M<~P4wZ#F|~un3{JdBO%pvF?tcVC|7RNEKb`pz3O7v_z|~QG zh4Mdh&rW-|vbR#jvVKmuT0!Qt`&>D=OC*83WPO8!FLri|IqXbJRQ8}A z)z+LObKF{U$?ErT@!Y?>_kT5p7m(eN4cFlX0@<7_P?abr$ViffRwSuYsW?Y4)$d=_1;~K}S%V$*;xexOui#!0} z5^HngM!89ro+aMxbs+@^UlRDM88psqw8vpXWa4qO%SN?vzg@99(n^x}@2mlcUN#o& z>{>LStsd%8q%wB{KHA6$1|w58Ztp@lKz)2Q)zM)2l6Gm>ERM(DxD($J(O(JnxkYY9 zW7SFpJG(9%2Hw=u2tLrKROc|DzNAuOUR|gD+M_J64Z9 z0JQR_hi=h;#>||y*W^p`;ge;a*1pgIX@YupJsmCc+E>-Yf8Q7RU$p63ZUBwaN*$ZY z3KwfF*I9)cH-6Y_=lT4c)x;75QVuGBgN;nj_NHoK%*(%VklP;-waLHDT#U@TDCZW= zCs^9X%@^wSpAP-X)2NbC>kt_TOwyQK`&%1;Ay0>WmB-uPb#cw(Al9CmWbuGc&H=J+1)Y(l^d0 zF1S|^EB@V-9LoPHBBdOjovRnK8|6OganaQ?! zi?g#6Gczu0d)5MXNGj=TFWoVFwY9eZx(i)~|5u^?FCRs@EK7Q(zBJ0y@*9XbQ2c`1 zX0Hfb>uKl0pch?D()(X)jF$>`VMHMn5Ny??5N&HZS~jQ~G`f273~#7acI`u#s!FX(Lm diff --git a/hosting/docker-compose/ee/docker-compose.dev.yml b/hosting/docker-compose/ee/docker-compose.dev.yml index 09861d7b20..8cccfdc2b9 100644 --- a/hosting/docker-compose/ee/docker-compose.dev.yml +++ b/hosting/docker-compose/ee/docker-compose.dev.yml @@ -122,7 +122,6 @@ services: volumes: - ../../../api/ee/src/crons/meters.sh:/meters.sh - - ../../../api/oss/src/crons/queries.sh:/queries.sh env_file: - ${ENV_FILE:-./.env.ee.dev} diff --git a/hosting/docker-compose/oss/docker-compose.dev.yml b/hosting/docker-compose/oss/docker-compose.dev.yml index 53c0eaef52..faf444c24a 100644 --- a/hosting/docker-compose/oss/docker-compose.dev.yml +++ b/hosting/docker-compose/oss/docker-compose.dev.yml @@ -103,30 +103,6 @@ services: command: > watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- celery -A entrypoint.celery_app worker --concurrency=1 --max-tasks-per-child=1 --prefetch-multiplier=1 - cron: - image: agenta-oss-dev-api:latest - - volumes: - # - - ../../../api/oss/src/crons/queries.sh:/queries.sh - - env_file: - - ${ENV_FILE:-./.env.oss.dev} - - depends_on: - - postgres - - api - - extra_hosts: - - "host.docker.internal:host-gateway" - - restart: always - - networks: - - agenta-network - - command: cron -f - alembic: build: context: ../../../api diff --git a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml index c7dbd5c730..ab0590b990 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml @@ -104,26 +104,6 @@ services: condition: service_healthy restart: always - cron: - image: agenta-oss-dev-api:latest - - env_file: - - ${ENV_FILE:-./.env.oss.dev} - - depends_on: - - postgres - - api - - extra_hosts: - - "host.docker.internal:host-gateway" - - restart: always - - networks: - - agenta-network - - command: cron -f - alembic: build: context: ../../../api diff --git a/hosting/docker-compose/oss/docker-compose.gh.yml b/hosting/docker-compose/oss/docker-compose.gh.yml index e079346a68..cc2e6612a3 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.yml @@ -80,26 +80,6 @@ services: - redis restart: always - cron: - image: agenta-oss-dev-api:latest - - env_file: - - ${ENV_FILE:-./.env.oss.dev} - - depends_on: - - postgres - - api - - extra_hosts: - - "host.docker.internal:host-gateway" - - restart: always - - networks: - - agenta-network - - command: cron -f - alembic: build: context: ../../../api diff --git a/sdk/agenta/sdk/tracing/exporters.py b/sdk/agenta/sdk/tracing/exporters.py index a121bd857a..c156d7d906 100644 --- a/sdk/agenta/sdk/tracing/exporters.py +++ b/sdk/agenta/sdk/tracing/exporters.py @@ -24,7 +24,7 @@ log = get_module_logger(__name__) -_ASYNC_EXPORT = environ.get("AGENTA_OTLP_ASYNC_EXPORT", "true").lower() in TRUTHY +_ASYNC_EXPORT = environ.get("AGENTA_OTLP_ASYNC_EXPORT", "false").lower() in TRUTHY class InlineTraceExporter(SpanExporter): diff --git a/sdk/agenta/sdk/workflows/handlers.py b/sdk/agenta/sdk/workflows/handlers.py index 7216761897..738392f345 100644 --- a/sdk/agenta/sdk/workflows/handlers.py +++ b/sdk/agenta/sdk/workflows/handlers.py @@ -511,24 +511,20 @@ def field_match_test_v0( correct_answer = inputs[correct_answer_key] if not isinstance(outputs, str) and not isinstance(outputs, dict): - # raise InvalidOutputsV0Error(expected=["dict", "str"], got=outputs) - return {"success": False} + raise InvalidOutputsV0Error(expected=["dict", "str"], got=outputs) outputs_dict = outputs if isinstance(outputs, str): try: outputs_dict = loads(outputs) except json.JSONDecodeError as e: - # raise InvalidOutputsV0Error(expected="dict", got=outputs) from e - return {"success": False} + raise InvalidOutputsV0Error(expected="dict", got=outputs) from e if not isinstance(outputs_dict, dict): - # raise InvalidOutputsV0Error(expected=["dict", "str"], got=outputs) - return {"success": False} + raise InvalidOutputsV0Error(expected=["dict", "str"], got=outputs) if not json_field in outputs_dict: - # raise MissingOutputV0Error(path=json_field) - return {"success": False} + raise MissingOutputV0Error(path=json_field) # -------------------------------------------------------------------------- success = outputs_dict[json_field] == correct_answer diff --git a/web/oss/src/components/DeleteEvaluationModal/DeleteEvaluationModal.tsx b/web/ee/src/components/DeleteEvaluationModal/DeleteEvaluationModal.tsx similarity index 100% rename from web/oss/src/components/DeleteEvaluationModal/DeleteEvaluationModal.tsx rename to web/ee/src/components/DeleteEvaluationModal/DeleteEvaluationModal.tsx diff --git a/web/oss/src/components/DeleteEvaluationModal/types.ts b/web/ee/src/components/DeleteEvaluationModal/types.ts similarity index 100% rename from web/oss/src/components/DeleteEvaluationModal/types.ts rename to web/ee/src/components/DeleteEvaluationModal/types.ts diff --git a/web/ee/src/components/DeploymentHistory/DeploymentHistory.tsx b/web/ee/src/components/DeploymentHistory/DeploymentHistory.tsx index 3b6f9cdb32..d596e2bc42 100644 --- a/web/ee/src/components/DeploymentHistory/DeploymentHistory.tsx +++ b/web/ee/src/components/DeploymentHistory/DeploymentHistory.tsx @@ -15,7 +15,7 @@ import { fetchAllDeploymentRevisions, } from "@/oss/services/deploymentVersioning/api" -import {DeploymentRevisionConfig, DeploymentRevisions} from "@agenta/oss/src/lib/types_ee" +import {DeploymentRevisionConfig, DeploymentRevisions} from "../../lib/types_ee" dayjs.extend(relativeTime) dayjs.extend(duration) diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/AutoEvalRunSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/AutoEvalRunSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/AutoEvalRunSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/AutoEvalRunSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/EvalNameTag.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/EvalNameTag.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/EvalNameTag.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/EvalNameTag.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/TagWithLink.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/TagWithLink.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/TagWithLink.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/TagWithLink.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/VariantTag.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/VariantTag.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/VariantTag.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/VariantTag.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/types.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/types.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/types.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/utils.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/utils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/utils.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/utils.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/variantUtils.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/variantUtils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/assets/variantUtils.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/assets/variantUtils.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunCompareMenu/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunCompareMenu/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunCompareMenu/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunCompareMenu/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunOutput.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunOutput.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunOutput.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunOutput.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunTraceHeader.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunTraceHeader.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunTraceHeader.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/assets/RunTraceHeader.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/lib/helpers.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/lib/helpers.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/lib/helpers.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/lib/helpers.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerHeader/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerHeader/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerHeader/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerHeader/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerSidePanel/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerSidePanel/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerSidePanel/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerSidePanel/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerContentSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerContentSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerContentSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerContentSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerHeaderSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerHeaderSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerHeaderSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerHeaderSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerSidePanelSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerSidePanelSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerSidePanelSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/Skeletons/FocusDrawerSidePanelSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/assets/EvalRunHeaderSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/assets/EvalRunHeaderSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/assets/EvalRunHeaderSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/assets/EvalRunHeaderSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunHeader/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/EvalRunPromptConfigViewerSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/EvalRunPromptConfigViewerSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/EvalRunPromptConfigViewerSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/EvalRunPromptConfigViewerSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/PromptConfigCard.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/PromptConfigCard.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/PromptConfigCard.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/assets/PromptConfigCard.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunPromptConfigViewer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/EvalRunScoreTableSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/EvalRunScoreTableSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/EvalRunScoreTableSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/EvalRunScoreTableSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/TraceMetrics.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/TraceMetrics.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/TraceMetrics.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/TraceMetrics.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/constants.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/constants.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/constants.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/assets/constants.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunScoreTable/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunSelectedEvaluations/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunSelectedEvaluations/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunSelectedEvaluations/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunSelectedEvaluations/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewUtilityOptions/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewUtilityOptions/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewUtilityOptions/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewUtilityOptions/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/assets/EvalRunTestcaseViewerSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/assets/EvalRunTestcaseViewerSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/assets/EvalRunTestcaseViewerSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/assets/EvalRunTestcaseViewerSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunTestcaseViewer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/types.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/types.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetircsSpiderChart/types.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/TimeSeriesChart.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/TimeSeriesChart.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/TimeSeriesChart.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/TimeSeriesChart.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/BarChart.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/BarChart.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/BarChart.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/BarChart.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/EvaluatorMetricsChartSkeleton.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/EvaluatorMetricsChartSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/EvaluatorMetricsChartSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/EvaluatorMetricsChartSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/HistogramChart.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/HistogramChart.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/HistogramChart.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/HistogramChart.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/LowerBand.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/LowerBand.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/LowerBand.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/LowerBand.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/UpperBand.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/UpperBand.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/UpperBand.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/UpperBand.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/helpers.ts b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/helpers.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/helpers.ts rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/assets/helpers.ts diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvaluatorMetricsChart/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/shared/BarChartPlaceholder.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/shared/BarChartPlaceholder.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/shared/BarChartPlaceholder.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/shared/BarChartPlaceholder.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/shared/PlaceholderOverlay.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/shared/PlaceholderOverlay.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/shared/PlaceholderOverlay.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/shared/PlaceholderOverlay.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/components/shared/SpiderChartPlaceholder.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/shared/SpiderChartPlaceholder.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/components/shared/SpiderChartPlaceholder.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/components/shared/SpiderChartPlaceholder.tsx diff --git a/web/oss/src/components/EvalRunDetails/AutoEvalRun/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/AutoEvalRun/index.tsx rename to web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/annotationUtils.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/annotationUtils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/annotationUtils.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/annotationUtils.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/helpers.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/helpers.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/helpers.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/helpers.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/optimisticUtils.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/optimisticUtils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/optimisticUtils.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/optimisticUtils.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/runnableSelectors.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/runnableSelectors.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/runnableSelectors.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/runnableSelectors.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/stepsMetricsUtils.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/stepsMetricsUtils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/stepsMetricsUtils.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/stepsMetricsUtils.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/assets/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/assets/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/AnnotateScenarioButton/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalResultsView/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalResultsView/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalResultsView/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalResultsView/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunBatchActions.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunBatchActions.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunBatchActions.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunBatchActions.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunName/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunName/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunName/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunName/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenario/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/EvalRunScenarioCardBody.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/EvalRunScenarioCardBody.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/EvalRunScenarioCardBody.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/EvalRunScenarioCardBody.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationInputs.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationInputs.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationInputs.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationInputs.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationResponse.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationResponse.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationResponse.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationResponse.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationRun.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationRun.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationRun.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/InvocationRun.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/KeyValue.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/KeyValue.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/KeyValue.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/KeyValue.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/utils.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/utils.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/utils.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/assets/utils.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCard/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCardTitle/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/EvalRunScenarioCards.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/EvalRunScenarioCards.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/EvalRunScenarioCards.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/EvalRunScenarioCards.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/assets/constants.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/assets/constants.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/assets/constants.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioCards/assets/constants.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioFilters.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioFilters.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioFilters.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/EvalRunScenarioFilters.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/assets/InstructionButton.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/assets/InstructionButton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/assets/InstructionButton.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/assets/InstructionButton.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/InstructionModal/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalButton.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalButton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalButton.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalButton.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalModalContent.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalModalContent.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalModalContent.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/assets/RenameEvalModalContent.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/RenameEvalModal/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/types.d.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/types.d.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/Modals/types.d.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/Modals/types.d.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/RunEvalScenarioButton/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioAnnotationPanel/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/ScenarioLoadingIndicator.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/ScenarioLoadingIndicator.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/ScenarioLoadingIndicator.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/ScenarioLoadingIndicator.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/assets/constants.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/assets/constants.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/assets/constants.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/ScenarioLoadingIndicator/assets/constants.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/types.ts b/web/ee/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/types.ts rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/components/SingleScenarioViewer/types.ts diff --git a/web/oss/src/components/EvalRunDetails/HumanEvalRun/index.tsx b/web/ee/src/components/EvalRunDetails/HumanEvalRun/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/HumanEvalRun/index.tsx rename to web/ee/src/components/EvalRunDetails/HumanEvalRun/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/OnlineEvalRun/OnlineUrlSync.tsx b/web/ee/src/components/EvalRunDetails/OnlineEvalRun/OnlineUrlSync.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/OnlineEvalRun/OnlineUrlSync.tsx rename to web/ee/src/components/EvalRunDetails/OnlineEvalRun/OnlineUrlSync.tsx diff --git a/web/oss/src/components/EvalRunDetails/OnlineEvalRun/components/ConfigurationViewer/index.tsx b/web/ee/src/components/EvalRunDetails/OnlineEvalRun/components/ConfigurationViewer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/OnlineEvalRun/components/ConfigurationViewer/index.tsx rename to web/ee/src/components/EvalRunDetails/OnlineEvalRun/components/ConfigurationViewer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/OnlineEvalRun/components/TracesViewer/index.tsx b/web/ee/src/components/EvalRunDetails/OnlineEvalRun/components/TracesViewer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/OnlineEvalRun/components/TracesViewer/index.tsx rename to web/ee/src/components/EvalRunDetails/OnlineEvalRun/components/TracesViewer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/OnlineEvalRun/index.tsx b/web/ee/src/components/EvalRunDetails/OnlineEvalRun/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/OnlineEvalRun/index.tsx rename to web/ee/src/components/EvalRunDetails/OnlineEvalRun/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/UrlSync.tsx b/web/ee/src/components/EvalRunDetails/UrlSync.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/UrlSync.tsx rename to web/ee/src/components/EvalRunDetails/UrlSync.tsx diff --git a/web/oss/src/components/EvalRunDetails/assets/renderChatMessages.tsx b/web/ee/src/components/EvalRunDetails/assets/renderChatMessages.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/assets/renderChatMessages.tsx rename to web/ee/src/components/EvalRunDetails/assets/renderChatMessages.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/ComparisonDataFetcher.tsx b/web/ee/src/components/EvalRunDetails/components/ComparisonDataFetcher.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/ComparisonDataFetcher.tsx rename to web/ee/src/components/EvalRunDetails/components/ComparisonDataFetcher.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunOverviewViewer/assets/EvalRunOverviewViewerSkeleton.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunOverviewViewer/assets/EvalRunOverviewViewerSkeleton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunOverviewViewer/assets/EvalRunOverviewViewerSkeleton.tsx rename to web/ee/src/components/EvalRunDetails/components/EvalRunOverviewViewer/assets/EvalRunOverviewViewerSkeleton.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunOverviewViewer/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunOverviewViewer/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunOverviewViewer/index.tsx rename to web/ee/src/components/EvalRunDetails/components/EvalRunOverviewViewer/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx rename to web/ee/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/assets/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/assets/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/assets/index.tsx rename to web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/assets/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx rename to web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/assets/constants.ts b/web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/assets/constants.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/assets/constants.ts rename to web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/assets/constants.ts diff --git a/web/oss/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx rename to web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataButton.tsx b/web/ee/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataButton.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataButton.tsx rename to web/ee/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataButton.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataModalContent.tsx b/web/ee/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataModalContent.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataModalContent.tsx rename to web/ee/src/components/EvalRunDetails/components/SaveDataModal/assets/SaveDataModalContent.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/SaveDataModal/assets/types.ts b/web/ee/src/components/EvalRunDetails/components/SaveDataModal/assets/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/SaveDataModal/assets/types.ts rename to web/ee/src/components/EvalRunDetails/components/SaveDataModal/assets/types.ts diff --git a/web/oss/src/components/EvalRunDetails/components/SaveDataModal/index.tsx b/web/ee/src/components/EvalRunDetails/components/SaveDataModal/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/SaveDataModal/index.tsx rename to web/ee/src/components/EvalRunDetails/components/SaveDataModal/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx similarity index 99% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx index fa0aef82a1..a5f4612b58 100644 --- a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx +++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx @@ -10,7 +10,7 @@ import TooltipButton from "@/oss/components/Playground/assets/EnhancedButton" import {Expandable} from "@/oss/components/Tables/ExpandableCell" import {useOptionalRunId, useRunId} from "@/oss/contexts/RunIdContext" import {useInvocationResult} from "@/oss/lib/hooks/useInvocationResult" -import {resolvePath} from "@/oss/lib/evalRunner/pureEnrichment" +import {resolvePath} from "@/oss/lib/workers/evalRunner/pureEnrichment" import {useAppNavigation, useAppState} from "@/oss/state/appState" import { diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ComparisonModeToggle.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ComparisonModeToggle.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ComparisonModeToggle.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ComparisonModeToggle.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedAnnotationValueCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedAnnotationValueCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedAnnotationValueCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedAnnotationValueCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricValueCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricValueCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricValueCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricValueCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricsCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricsCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricsCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/CollapsedMetricsCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/helpers.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/helpers.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/helpers.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/helpers.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/types.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/types.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/types.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ScenarioTraceSummary.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ScenarioTraceSummary.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ScenarioTraceSummary.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ScenarioTraceSummary.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/StatusCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/StatusCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/StatusCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/StatusCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/TimestampCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/TimestampCell.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/TimestampCell.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/TimestampCell.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/VirtualizedScenarioTableAnnotateDrawer.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/VirtualizedScenarioTableAnnotateDrawer.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/VirtualizedScenarioTableAnnotateDrawer.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/VirtualizedScenarioTableAnnotateDrawer.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/atoms/evaluatorFailures.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/atoms/evaluatorFailures.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/atoms/evaluatorFailures.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/atoms/evaluatorFailures.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/constants.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/constants.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/constants.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/constants.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorNameUtils.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorNameUtils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorNameUtils.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorNameUtils.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorSchemaUtils.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorSchemaUtils.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorSchemaUtils.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/evaluatorSchemaUtils.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/flatDataSourceBuilder.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/flatDataSourceBuilder.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/flatDataSourceBuilder.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/flatDataSourceBuilder.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/types.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/types.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/types.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useScrollToScenario.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useScrollToScenario.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useScrollToScenario.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useScrollToScenario.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/index.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/index.tsx rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/types.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/types.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/components/VirtualizedScenarioTable/types.ts rename to web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/types.ts diff --git a/web/oss/src/components/EvalRunDetails/hooks/useCachedScenarioSteps.ts b/web/ee/src/components/EvalRunDetails/hooks/useCachedScenarioSteps.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/hooks/useCachedScenarioSteps.ts rename to web/ee/src/components/EvalRunDetails/hooks/useCachedScenarioSteps.ts diff --git a/web/oss/src/components/EvalRunDetails/hooks/useMetricStepError.ts b/web/ee/src/components/EvalRunDetails/hooks/useMetricStepError.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/hooks/useMetricStepError.ts rename to web/ee/src/components/EvalRunDetails/hooks/useMetricStepError.ts diff --git a/web/oss/src/components/EvalRunDetails/index.tsx b/web/ee/src/components/EvalRunDetails/index.tsx similarity index 100% rename from web/oss/src/components/EvalRunDetails/index.tsx rename to web/ee/src/components/EvalRunDetails/index.tsx diff --git a/web/oss/src/components/EvalRunDetails/state/evalType.ts b/web/ee/src/components/EvalRunDetails/state/evalType.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/state/evalType.ts rename to web/ee/src/components/EvalRunDetails/state/evalType.ts diff --git a/web/oss/src/components/EvalRunDetails/state/focusScenarioAtom.ts b/web/ee/src/components/EvalRunDetails/state/focusScenarioAtom.ts similarity index 100% rename from web/oss/src/components/EvalRunDetails/state/focusScenarioAtom.ts rename to web/ee/src/components/EvalRunDetails/state/focusScenarioAtom.ts diff --git a/web/oss/src/components/EvalRunDetails/state/urlState.ts b/web/ee/src/components/EvalRunDetails/state/urlState.ts similarity index 97% rename from web/oss/src/components/EvalRunDetails/state/urlState.ts rename to web/ee/src/components/EvalRunDetails/state/urlState.ts index aa4697994a..29a988ca50 100644 --- a/web/oss/src/components/EvalRunDetails/state/urlState.ts +++ b/web/ee/src/components/EvalRunDetails/state/urlState.ts @@ -1,7 +1,7 @@ import {atom} from "jotai" import {atomWithImmer} from "jotai-immer" -import {evalTypeAtom} from "./evalType" +import {evalTypeAtom} from "../state/evalType" export interface EvalRunUrlState { view?: diff --git a/web/oss/src/components/EvaluationTable/ABTestingEvaluationTable.tsx b/web/ee/src/components/EvaluationTable/ABTestingEvaluationTable.tsx similarity index 100% rename from web/oss/src/components/EvaluationTable/ABTestingEvaluationTable.tsx rename to web/ee/src/components/EvaluationTable/ABTestingEvaluationTable.tsx diff --git a/web/oss/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx similarity index 99% rename from web/oss/src/components/EvaluationTable/SingleModelEvaluationTable.tsx rename to web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx index 2bac4883a9..ad8e49c7cb 100644 --- a/web/oss/src/components/EvaluationTable/SingleModelEvaluationTable.tsx +++ b/web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx @@ -19,8 +19,8 @@ import {getDefaultStore, useAtomValue} from "jotai" import debounce from "lodash/debounce" import {useRouter} from "next/router" -import SaveTestsetModal from "@/oss/components/SaveTestsetModal/SaveTestsetModal" import SecondaryButton from "@/oss/components/SecondaryButton/SecondaryButton" +import {useQueryParamState} from "@/oss/state/appState" import {EvaluationFlow} from "@/oss/lib/enums" import {exportSingleModelEvaluationData} from "@/oss/lib/helpers/evaluate" import {isBaseResponse, isFuncResponse} from "@/oss/lib/helpers/playgroundResp" @@ -41,7 +41,6 @@ import {transformToRequestBody} from "@/oss/lib/shared/variant/transformer/trans import type {BaseResponse, EvaluationScenario, KeyValuePair, Variant} from "@/oss/lib/Types" import {callVariant} from "@/oss/services/api" import {updateEvaluation, updateEvaluationScenario} from "@/oss/services/human-evaluations/api" -import {useQueryParamState} from "@/oss/state/appState" import {customPropertiesByRevisionAtomFamily} from "@/oss/state/newPlayground/core/customProperties" import { stablePromptVariablesAtomFamily, @@ -52,6 +51,7 @@ import {appUriInfoAtom, appSchemaAtom} from "@/oss/state/variant/atoms/fetcher" import EvaluationCardView from "../Evaluations/EvaluationCardView" import EvaluationVotePanel from "../Evaluations/EvaluationCardView/EvaluationVotePanel" +import SaveTestsetModal from "../SaveTestsetModal/SaveTestsetModal" import {useSingleModelEvaluationTableStyles} from "./assets/styles" import ParamsFormWithRun from "./components/ParamsFormWithRun" diff --git a/web/oss/src/components/EvaluationTable/assets/styles.ts b/web/ee/src/components/EvaluationTable/assets/styles.ts similarity index 100% rename from web/oss/src/components/EvaluationTable/assets/styles.ts rename to web/ee/src/components/EvaluationTable/assets/styles.ts diff --git a/web/oss/src/components/EvaluationTable/components/ParamsFormWithRun.tsx b/web/ee/src/components/EvaluationTable/components/ParamsFormWithRun.tsx similarity index 100% rename from web/oss/src/components/EvaluationTable/components/ParamsFormWithRun.tsx rename to web/ee/src/components/EvaluationTable/components/ParamsFormWithRun.tsx diff --git a/web/oss/src/components/EvaluationTable/types.d.ts b/web/ee/src/components/EvaluationTable/types.d.ts similarity index 100% rename from web/oss/src/components/EvaluationTable/types.d.ts rename to web/ee/src/components/EvaluationTable/types.d.ts diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/EvaluationCard.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/EvaluationCard.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/EvaluationCard.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/EvaluationCard.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/EvaluationChatResponse.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/EvaluationChatResponse.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/EvaluationChatResponse.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/EvaluationChatResponse.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/EvaluationInputs.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/EvaluationInputs.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/EvaluationInputs.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/EvaluationInputs.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/EvaluationVariantCard.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/EvaluationVariantCard.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/EvaluationVariantCard.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/EvaluationVariantCard.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/VariantAlphabet.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/VariantAlphabet.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/VariantAlphabet.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/VariantAlphabet.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/assets/styles.ts b/web/ee/src/components/Evaluations/EvaluationCardView/assets/styles.ts similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/assets/styles.ts rename to web/ee/src/components/Evaluations/EvaluationCardView/assets/styles.ts diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/index.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/index.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/index.tsx rename to web/ee/src/components/Evaluations/EvaluationCardView/index.tsx diff --git a/web/oss/src/components/Evaluations/EvaluationCardView/types.d.ts b/web/ee/src/components/Evaluations/EvaluationCardView/types.d.ts similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationCardView/types.d.ts rename to web/ee/src/components/Evaluations/EvaluationCardView/types.d.ts diff --git a/web/oss/src/components/Evaluations/EvaluationErrorModal.tsx b/web/ee/src/components/Evaluations/EvaluationErrorModal.tsx similarity index 100% rename from web/oss/src/components/Evaluations/EvaluationErrorModal.tsx rename to web/ee/src/components/Evaluations/EvaluationErrorModal.tsx diff --git a/web/oss/src/services/evaluationRuns/utils.ts b/web/ee/src/components/Evaluations/HumanEvaluationResult.tsx similarity index 100% rename from web/oss/src/services/evaluationRuns/utils.ts rename to web/ee/src/components/Evaluations/HumanEvaluationResult.tsx diff --git a/web/oss/src/components/Evaluations/ShareEvaluationModal.tsx b/web/ee/src/components/Evaluations/ShareEvaluationModal.tsx similarity index 100% rename from web/oss/src/components/Evaluations/ShareEvaluationModal.tsx rename to web/ee/src/components/Evaluations/ShareEvaluationModal.tsx diff --git a/web/oss/src/components/Evaluators/assets/cells/EvaluatorTagsCell.tsx b/web/ee/src/components/Evaluators/assets/cells/EvaluatorTagsCell.tsx similarity index 100% rename from web/oss/src/components/Evaluators/assets/cells/EvaluatorTagsCell.tsx rename to web/ee/src/components/Evaluators/assets/cells/EvaluatorTagsCell.tsx diff --git a/web/oss/src/components/Evaluators/assets/cells/EvaluatorTypePill.tsx b/web/ee/src/components/Evaluators/assets/cells/EvaluatorTypePill.tsx similarity index 100% rename from web/oss/src/components/Evaluators/assets/cells/EvaluatorTypePill.tsx rename to web/ee/src/components/Evaluators/assets/cells/EvaluatorTypePill.tsx diff --git a/web/oss/src/components/Evaluators/assets/cells/TableDropdownMenu/index.tsx b/web/ee/src/components/Evaluators/assets/cells/TableDropdownMenu/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/assets/cells/TableDropdownMenu/index.tsx rename to web/ee/src/components/Evaluators/assets/cells/TableDropdownMenu/index.tsx diff --git a/web/oss/src/components/Evaluators/assets/cells/TableDropdownMenu/types.ts b/web/ee/src/components/Evaluators/assets/cells/TableDropdownMenu/types.ts similarity index 100% rename from web/oss/src/components/Evaluators/assets/cells/TableDropdownMenu/types.ts rename to web/ee/src/components/Evaluators/assets/cells/TableDropdownMenu/types.ts diff --git a/web/oss/src/components/Evaluators/assets/constants.ts b/web/ee/src/components/Evaluators/assets/constants.ts similarity index 100% rename from web/oss/src/components/Evaluators/assets/constants.ts rename to web/ee/src/components/Evaluators/assets/constants.ts diff --git a/web/oss/src/components/Evaluators/assets/getColumns.tsx b/web/ee/src/components/Evaluators/assets/getColumns.tsx similarity index 100% rename from web/oss/src/components/Evaluators/assets/getColumns.tsx rename to web/ee/src/components/Evaluators/assets/getColumns.tsx diff --git a/web/oss/src/components/Evaluators/assets/types.ts b/web/ee/src/components/Evaluators/assets/types.ts similarity index 100% rename from web/oss/src/components/Evaluators/assets/types.ts rename to web/ee/src/components/Evaluators/assets/types.ts diff --git a/web/oss/src/components/Evaluators/assets/utils.ts b/web/ee/src/components/Evaluators/assets/utils.ts similarity index 100% rename from web/oss/src/components/Evaluators/assets/utils.ts rename to web/ee/src/components/Evaluators/assets/utils.ts diff --git a/web/oss/src/components/Evaluators/components/ConfigureEvaluator/assets/ConfigureEvaluatorSkeleton.tsx b/web/ee/src/components/Evaluators/components/ConfigureEvaluator/assets/ConfigureEvaluatorSkeleton.tsx similarity index 100% rename from web/oss/src/components/Evaluators/components/ConfigureEvaluator/assets/ConfigureEvaluatorSkeleton.tsx rename to web/ee/src/components/Evaluators/components/ConfigureEvaluator/assets/ConfigureEvaluatorSkeleton.tsx diff --git a/web/oss/src/components/Evaluators/components/ConfigureEvaluator/index.tsx b/web/ee/src/components/Evaluators/components/ConfigureEvaluator/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/components/ConfigureEvaluator/index.tsx rename to web/ee/src/components/Evaluators/components/ConfigureEvaluator/index.tsx diff --git a/web/oss/src/components/Evaluators/components/DeleteEvaluatorsModal/assets/DeleteEvaluatorsModalContent/index.tsx b/web/ee/src/components/Evaluators/components/DeleteEvaluatorsModal/assets/DeleteEvaluatorsModalContent/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/components/DeleteEvaluatorsModal/assets/DeleteEvaluatorsModalContent/index.tsx rename to web/ee/src/components/Evaluators/components/DeleteEvaluatorsModal/assets/DeleteEvaluatorsModalContent/index.tsx diff --git a/web/oss/src/components/Evaluators/components/DeleteEvaluatorsModal/index.tsx b/web/ee/src/components/Evaluators/components/DeleteEvaluatorsModal/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/components/DeleteEvaluatorsModal/index.tsx rename to web/ee/src/components/Evaluators/components/DeleteEvaluatorsModal/index.tsx diff --git a/web/oss/src/components/Evaluators/components/DeleteEvaluatorsModal/types.ts b/web/ee/src/components/Evaluators/components/DeleteEvaluatorsModal/types.ts similarity index 100% rename from web/oss/src/components/Evaluators/components/DeleteEvaluatorsModal/types.ts rename to web/ee/src/components/Evaluators/components/DeleteEvaluatorsModal/types.ts diff --git a/web/oss/src/components/Evaluators/components/SelectEvaluatorModal/assets/SelectEvaluatorModalContent/index.tsx b/web/ee/src/components/Evaluators/components/SelectEvaluatorModal/assets/SelectEvaluatorModalContent/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/components/SelectEvaluatorModal/assets/SelectEvaluatorModalContent/index.tsx rename to web/ee/src/components/Evaluators/components/SelectEvaluatorModal/assets/SelectEvaluatorModalContent/index.tsx diff --git a/web/oss/src/components/Evaluators/components/SelectEvaluatorModal/index.tsx b/web/ee/src/components/Evaluators/components/SelectEvaluatorModal/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/components/SelectEvaluatorModal/index.tsx rename to web/ee/src/components/Evaluators/components/SelectEvaluatorModal/index.tsx diff --git a/web/oss/src/components/Evaluators/components/SelectEvaluatorModal/types.ts b/web/ee/src/components/Evaluators/components/SelectEvaluatorModal/types.ts similarity index 100% rename from web/oss/src/components/Evaluators/components/SelectEvaluatorModal/types.ts rename to web/ee/src/components/Evaluators/components/SelectEvaluatorModal/types.ts diff --git a/web/oss/src/components/Evaluators/hooks/useEvaluatorsRegistryData.ts b/web/ee/src/components/Evaluators/hooks/useEvaluatorsRegistryData.ts similarity index 100% rename from web/oss/src/components/Evaluators/hooks/useEvaluatorsRegistryData.ts rename to web/ee/src/components/Evaluators/hooks/useEvaluatorsRegistryData.ts diff --git a/web/oss/src/components/Evaluators/index.tsx b/web/ee/src/components/Evaluators/index.tsx similarity index 100% rename from web/oss/src/components/Evaluators/index.tsx rename to web/ee/src/components/Evaluators/index.tsx diff --git a/web/oss/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx b/web/ee/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx rename to web/ee/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx diff --git a/web/oss/src/components/HumanEvaluationModal/assets/styles.ts b/web/ee/src/components/HumanEvaluationModal/assets/styles.ts similarity index 100% rename from web/oss/src/components/HumanEvaluationModal/assets/styles.ts rename to web/ee/src/components/HumanEvaluationModal/assets/styles.ts diff --git a/web/oss/src/components/HumanEvaluationModal/types.d.ts b/web/ee/src/components/HumanEvaluationModal/types.d.ts similarity index 100% rename from web/oss/src/components/HumanEvaluationModal/types.d.ts rename to web/ee/src/components/HumanEvaluationModal/types.d.ts diff --git a/web/oss/src/components/HumanEvaluations/AbTestingEvaluation.tsx b/web/ee/src/components/HumanEvaluations/AbTestingEvaluation.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/AbTestingEvaluation.tsx rename to web/ee/src/components/HumanEvaluations/AbTestingEvaluation.tsx diff --git a/web/oss/src/components/HumanEvaluations/SingleModelEvaluation.tsx b/web/ee/src/components/HumanEvaluations/SingleModelEvaluation.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/SingleModelEvaluation.tsx rename to web/ee/src/components/HumanEvaluations/SingleModelEvaluation.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/EvaluationStatusCell.tsx b/web/ee/src/components/HumanEvaluations/assets/EvaluationStatusCell.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/EvaluationStatusCell.tsx rename to web/ee/src/components/HumanEvaluations/assets/EvaluationStatusCell.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/LegacyEvalResultCell.tsx b/web/ee/src/components/HumanEvaluations/assets/LegacyEvalResultCell.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/LegacyEvalResultCell.tsx rename to web/ee/src/components/HumanEvaluations/assets/LegacyEvalResultCell.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartAxis.tsx b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartAxis.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartAxis.tsx rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartAxis.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartFrame.tsx b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartFrame.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartFrame.tsx rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ChartFrame.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveFrequencyChart.tsx b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveFrequencyChart.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveFrequencyChart.tsx rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveFrequencyChart.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveMetricChart.tsx b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveMetricChart.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveMetricChart.tsx rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/ResponsiveMetricChart.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/chartUtils.ts b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/chartUtils.ts similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/chartUtils.ts rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/chartUtils.ts diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils.ts b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils.ts similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils.ts rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils.ts diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/types.ts b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/types.ts similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/MetricDetailsPopover/types.ts rename to web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/types.ts diff --git a/web/oss/src/components/HumanEvaluations/assets/SingleModelEvaluationHeader/index.tsx b/web/ee/src/components/HumanEvaluations/assets/SingleModelEvaluationHeader/index.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/SingleModelEvaluationHeader/index.tsx rename to web/ee/src/components/HumanEvaluations/assets/SingleModelEvaluationHeader/index.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/TableDropdownMenu/index.tsx b/web/ee/src/components/HumanEvaluations/assets/TableDropdownMenu/index.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/TableDropdownMenu/index.tsx rename to web/ee/src/components/HumanEvaluations/assets/TableDropdownMenu/index.tsx diff --git a/web/oss/src/components/HumanEvaluations/assets/TableDropdownMenu/types.ts b/web/ee/src/components/HumanEvaluations/assets/TableDropdownMenu/types.ts similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/TableDropdownMenu/types.ts rename to web/ee/src/components/HumanEvaluations/assets/TableDropdownMenu/types.ts diff --git a/web/oss/src/components/HumanEvaluations/assets/styles.ts b/web/ee/src/components/HumanEvaluations/assets/styles.ts similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/styles.ts rename to web/ee/src/components/HumanEvaluations/assets/styles.ts diff --git a/web/oss/src/components/HumanEvaluations/assets/utils.tsx b/web/ee/src/components/HumanEvaluations/assets/utils.tsx similarity index 100% rename from web/oss/src/components/HumanEvaluations/assets/utils.tsx rename to web/ee/src/components/HumanEvaluations/assets/utils.tsx diff --git a/web/oss/src/components/HumanEvaluations/types.ts b/web/ee/src/components/HumanEvaluations/types.ts similarity index 100% rename from web/oss/src/components/HumanEvaluations/types.ts rename to web/ee/src/components/HumanEvaluations/types.ts diff --git a/web/oss/src/components/SaveTestsetModal/SaveTestsetModal.tsx b/web/ee/src/components/SaveTestsetModal/SaveTestsetModal.tsx similarity index 100% rename from web/oss/src/components/SaveTestsetModal/SaveTestsetModal.tsx rename to web/ee/src/components/SaveTestsetModal/SaveTestsetModal.tsx diff --git a/web/oss/src/components/SaveTestsetModal/types.d.ts b/web/ee/src/components/SaveTestsetModal/types.d.ts similarity index 100% rename from web/oss/src/components/SaveTestsetModal/types.d.ts rename to web/ee/src/components/SaveTestsetModal/types.d.ts diff --git a/web/oss/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorModal.tsx b/web/ee/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorModal.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorModal.tsx rename to web/ee/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorModal.tsx diff --git a/web/oss/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorPopover.tsx b/web/ee/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorPopover.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorPopover.tsx rename to web/ee/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorPopover.tsx diff --git a/web/oss/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorText.tsx b/web/ee/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorText.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorText.tsx rename to web/ee/src/components/pages/evaluations/EvaluationErrorProps/EvaluationErrorText.tsx diff --git a/web/oss/src/components/pages/evaluations/EvaluationsView.tsx b/web/ee/src/components/pages/evaluations/EvaluationsView.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/EvaluationsView.tsx rename to web/ee/src/components/pages/evaluations/EvaluationsView.tsx diff --git a/web/oss/src/components/pages/evaluations/FilterColumns/FilterColumns.tsx b/web/ee/src/components/pages/evaluations/FilterColumns/FilterColumns.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/FilterColumns/FilterColumns.tsx rename to web/ee/src/components/pages/evaluations/FilterColumns/FilterColumns.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/NewEvaluationModalContent.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/Components/NewEvaluationModalContent.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/Components/NewEvaluationModalContent.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/Components/NewEvaluationModalContent.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectAppSection.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectAppSection.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectAppSection.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectAppSection.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectEvaluatorSection/SelectEvaluatorSection.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectEvaluatorSection/SelectEvaluatorSection.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectEvaluatorSection/SelectEvaluatorSection.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectEvaluatorSection/SelectEvaluatorSection.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectTestsetSection.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectTestsetSection.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectTestsetSection.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectTestsetSection.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectVariantSection.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectVariantSection.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/Components/SelectVariantSection.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/Components/SelectVariantSection.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/index.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/index.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/index.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/index.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/types.ts b/web/ee/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/types.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/types.ts rename to web/ee/src/components/pages/evaluations/NewEvaluation/assets/TabLabel/types.ts diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/assets/constants.ts b/web/ee/src/components/pages/evaluations/NewEvaluation/assets/constants.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/assets/constants.ts rename to web/ee/src/components/pages/evaluations/NewEvaluation/assets/constants.ts diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/assets/styles.ts b/web/ee/src/components/pages/evaluations/NewEvaluation/assets/styles.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/assets/styles.ts rename to web/ee/src/components/pages/evaluations/NewEvaluation/assets/styles.ts diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/index.tsx b/web/ee/src/components/pages/evaluations/NewEvaluation/index.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/index.tsx rename to web/ee/src/components/pages/evaluations/NewEvaluation/index.tsx diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/types.ts b/web/ee/src/components/pages/evaluations/NewEvaluation/types.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/NewEvaluation/types.ts rename to web/ee/src/components/pages/evaluations/NewEvaluation/types.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/AdvancedSettings.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/AdvancedSettings.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/AdvancedSettings.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/AdvancedSettings.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DebugSection.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DebugSection.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DebugSection.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DebugSection.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DynamicFormField.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DynamicFormField.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DynamicFormField.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/DynamicFormField.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorTestcaseModal.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorTestcaseModal.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorTestcaseModal.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorTestcaseModal.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorVariantModal.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorVariantModal.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorVariantModal.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/EvaluatorVariantModal.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaEditor.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaEditor.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaEditor.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaEditor.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts similarity index 82% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts index a56de11836..b6acddb008 100644 --- a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts +++ b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/JSONSchemaGenerator.ts @@ -23,15 +23,15 @@ export function generateJSONSchema(config: SchemaConfig): GeneratedJSONSchema { const {responseFormat, includeReasoning, continuousConfig, categoricalOptions} = config const properties: Record = {} - const required: string[] = ["score"] + const required: string[] = ["correctness"] // Base description is always "The grade results" const baseDescription = "The grade results" - // Add the main score field based on response format + // Add the main correctness field based on response format switch (responseFormat) { case "continuous": - properties.score = { + properties.correctness = { type: "number", description: baseDescription, minimum: continuousConfig?.minimum ?? 0, @@ -40,7 +40,7 @@ export function generateJSONSchema(config: SchemaConfig): GeneratedJSONSchema { break case "boolean": - properties.score = { + properties.correctness = { type: "boolean", description: baseDescription, } @@ -53,14 +53,14 @@ export function generateJSONSchema(config: SchemaConfig): GeneratedJSONSchema { .map((opt) => `"${opt.name}": ${opt.description}`) .join("| ") - properties.score = { + properties.correctness = { type: "string", description: `${baseDescription}. Categories: ${categoryDescriptions}`, enum: enumValues, } } else { // Fallback if no categories defined - properties.score = { + properties.correctness = { type: "string", description: baseDescription, } @@ -97,43 +97,43 @@ export function parseJSONSchema(schemaString: string): SchemaConfig | null { // Handle both old format (direct schema) and new format (with name wrapper) const schema = parsed.schema || parsed - if (!schema.properties || !schema.properties.score) { + if (!schema.properties || !schema.properties.correctness) { return null } - const score = schema.properties.score + const correctness = schema.properties.correctness const hasReasoning = !!schema.properties.comment let responseFormat: SchemaConfig["responseFormat"] = "boolean" let continuousConfig: SchemaConfig["continuousConfig"] let categoricalOptions: SchemaConfig["categoricalOptions"] - if (score.type === "number") { + if (correctness.type === "number") { responseFormat = "continuous" continuousConfig = { - minimum: score.minimum ?? 0, - maximum: score.maximum ?? 10, + minimum: correctness.minimum ?? 0, + maximum: correctness.maximum ?? 10, } - } else if (score.type === "boolean") { + } else if (correctness.type === "boolean") { responseFormat = "boolean" - } else if (score.type === "string" && score.enum) { + } else if (correctness.type === "string" && correctness.enum) { responseFormat = "categorical" // Parse category descriptions from the description field - const desc = score.description || "" + const desc = correctness.description || "" const categoriesMatch = desc.match(/Categories: (.+)/) if (categoriesMatch) { const categoriesStr = categoriesMatch[1] const categoryPairs = categoriesStr.split("| ") - categoricalOptions = score.enum.map((name: string) => { + categoricalOptions = correctness.enum.map((name: string) => { const pair = categoryPairs.find((p: string) => p.startsWith(`"${name}":`)) const description = pair ? pair.split(": ")[1] || "" : "" return {name, description} }) } else { - categoricalOptions = score.enum.map((name: string) => ({ + categoricalOptions = correctness.enum.map((name: string) => ({ name, description: "", })) diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/index.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/index.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/index.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/index.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/types.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/types.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/types.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/JSONSchema/types.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/Messages.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/Messages.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/Messages.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/Messages.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/assets/styles.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/assets/styles.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/assets/styles.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/assets/styles.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/index.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/index.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/index.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/index.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/types.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/types.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/types.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/types.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/variantUtils.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/variantUtils.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/variantUtils.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/ConfigureEvaluator/variantUtils.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/DeleteModal.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/DeleteModal.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/DeleteModal.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/DeleteModal.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorCard.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorCard.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorCard.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorCard.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorList.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorList.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorList.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/EvaluatorList.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/index.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/index.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/index.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/Evaluators/index.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorCard.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorCard.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorCard.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorCard.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorList.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorList.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorList.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/NewEvaluatorList.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/index.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/index.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/index.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/NewEvaluator/index.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/Filters/SearchFilter.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/Filters/SearchFilter.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/Filters/SearchFilter.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/Filters/SearchFilter.tsx diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx b/web/ee/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx similarity index 99% rename from web/oss/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx rename to web/ee/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx index 783fb6804f..e52e9227fb 100644 --- a/web/oss/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx +++ b/web/ee/src/components/pages/evaluations/autoEvaluation/assets/AutoEvaluationHeader.tsx @@ -25,9 +25,9 @@ import {getMetricConfig} from "@/oss/lib/metrics/utils" import {EvaluationStatus} from "@/oss/lib/Types" import {getAppValues} from "@/oss/state/app" -import {statusMapper} from "../../cellRenderers/cellRenderers" +import {statusMapper} from "../../../evaluations/cellRenderers/cellRenderers" import {buildEvaluationNavigationUrl} from "../../utils" -import {useStyles} from "./styles" +import {useStyles} from "../assets/styles" import {AutoEvaluationHeaderProps} from "./types" diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/assets/styles.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/assets/styles.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/assets/styles.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/assets/styles.ts diff --git a/web/oss/src/components/pages/evaluations/autoEvaluation/assets/types.ts b/web/ee/src/components/pages/evaluations/autoEvaluation/assets/types.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/autoEvaluation/assets/types.ts rename to web/ee/src/components/pages/evaluations/autoEvaluation/assets/types.ts diff --git a/web/oss/src/components/pages/evaluations/cellRenderers/StatusRenderer.tsx b/web/ee/src/components/pages/evaluations/cellRenderers/StatusRenderer.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/cellRenderers/StatusRenderer.tsx rename to web/ee/src/components/pages/evaluations/cellRenderers/StatusRenderer.tsx diff --git a/web/oss/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx b/web/ee/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx rename to web/ee/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx diff --git a/web/oss/src/components/pages/evaluations/customEvaluation/CustomEvaluation.tsx b/web/ee/src/components/pages/evaluations/customEvaluation/CustomEvaluation.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/customEvaluation/CustomEvaluation.tsx rename to web/ee/src/components/pages/evaluations/customEvaluation/CustomEvaluation.tsx diff --git a/web/oss/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx b/web/ee/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx rename to web/ee/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx diff --git a/web/oss/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx b/web/ee/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx rename to web/ee/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluation.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluation.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluation.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluation.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluationDrawer.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluationDrawer.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluationDrawer.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/OnlineEvaluationDrawer.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/assets/helpers.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/assets/helpers.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/assets/helpers.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/assets/helpers.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/assets/state.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/assets/state.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/assets/state.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/assets/state.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/assets/styles.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/assets/styles.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/assets/styles.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/assets/styles.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorDetailsPreview.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorDetailsPreview.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorDetailsPreview.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorDetailsPreview.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorTypeTag.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorTypeTag.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorTypeTag.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/EvaluatorTypeTag.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/FiltersPreview.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/FiltersPreview.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/FiltersPreview.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/FiltersPreview.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/OnlineEvaluationRowActions.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/OnlineEvaluationRowActions.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/OnlineEvaluationRowActions.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/OnlineEvaluationRowActions.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/PromptPreview.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/PromptPreview.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/PromptPreview.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/PromptPreview.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersCell.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersCell.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersCell.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersCell.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersSummaryCard.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersSummaryCard.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersSummaryCard.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/QueryFiltersSummaryCard.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/ReadOnlyBox.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/ReadOnlyBox.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/ReadOnlyBox.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/ReadOnlyBox.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/components/SamplingRateControl.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/components/SamplingRateControl.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/components/SamplingRateControl.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/components/SamplingRateControl.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/constants.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/constants.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/constants.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/constants.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorDetails.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorDetails.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorDetails.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorDetails.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorSelection.tsx b/web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorSelection.tsx similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorSelection.tsx rename to web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorSelection.tsx diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeFromConfigs.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeFromConfigs.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeFromConfigs.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeFromConfigs.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeMeta.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeMeta.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeMeta.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useEvaluatorTypeMeta.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useOnlineEvaluations.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useOnlineEvaluations.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/hooks/useOnlineEvaluations.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/hooks/useOnlineEvaluations.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/types.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/types.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/types.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/types.ts diff --git a/web/oss/src/components/pages/evaluations/onlineEvaluation/utils/evaluatorDetails.ts b/web/ee/src/components/pages/evaluations/onlineEvaluation/utils/evaluatorDetails.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/onlineEvaluation/utils/evaluatorDetails.ts rename to web/ee/src/components/pages/evaluations/onlineEvaluation/utils/evaluatorDetails.ts diff --git a/web/oss/src/components/pages/evaluations/utils.ts b/web/ee/src/components/pages/evaluations/utils.ts similarity index 100% rename from web/oss/src/components/pages/evaluations/utils.ts rename to web/ee/src/components/pages/evaluations/utils.ts diff --git a/web/oss/src/contexts/RunIdContext.tsx b/web/ee/src/contexts/RunIdContext.tsx similarity index 100% rename from web/oss/src/contexts/RunIdContext.tsx rename to web/ee/src/contexts/RunIdContext.tsx diff --git a/web/ee/src/lib/helpers/evaluate.ts b/web/ee/src/lib/helpers/evaluate.ts new file mode 100644 index 0000000000..49c631c561 --- /dev/null +++ b/web/ee/src/lib/helpers/evaluate.ts @@ -0,0 +1,469 @@ +import {EvaluationType} from "@agenta/oss/src/lib/enums" +import {convertToCsv, downloadCsv} from "@agenta/oss/src/lib/helpers/fileManipulations" +import {formatCurrency, formatLatency} from "@agenta/oss/src/lib/helpers/formatters" +import {isDemo} from "@agenta/oss/src/lib/helpers/utils" +import { + Evaluation, + GenericObject, + TypedValue, + Variant, + _Evaluation, + EvaluationScenario, +} from "@agenta/oss/src/lib/Types" +import dayjs from "dayjs" +import capitalize from "lodash/capitalize" +import round from "lodash/round" + +import AlertPopup from "@/oss/components/AlertPopup/AlertPopup" +import {runningStatuses} from "@/oss/components/pages/evaluations/cellRenderers/cellRenderers" +import { + HumanEvaluationListTableDataType, + SingleModelEvaluationListTableDataType, +} from "@/oss/lib/Types" +import {fetchEvaluatonIdsByResource} from "@/oss/services/evaluations/api" + +export const exportExactEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => { + const exportRow = rows.map((data, ix) => { + return { + ["Inputs"]: + evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] || + data.inputs[0].input_value, + [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Correct answer"]: data.correctAnswer, + ["Evaluation"]: data.score, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportSimilarityEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => { + const exportRow = rows.map((data, ix) => { + return { + ["Inputs"]: + evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] || + data.inputs[0].input_value, + [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Correct answer"]: data.correctAnswer, + ["Score"]: data.score, + ["Evaluation"]: data.similarity, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportAICritiqueEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => { + const exportRow = rows.map((data, ix) => { + return { + ["Inputs"]: + evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] || + data.inputs[0].input_value, + [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Correct answer"]: data.correctAnswer, + ["Score"]: data.score, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportABTestingEvaluationData = ( + evaluation: Evaluation, + scenarios: EvaluationScenario[], + rows: GenericObject[], +) => { + const exportRow = rows.map((data, ix) => { + const inputColumns = evaluation.testset.testsetChatColumn + ? {Input: evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn]} + : data.inputs.reduce( + (columns: any, input: {input_name: string; input_value: string}) => { + columns[`${input.input_name}`] = input.input_value + return columns + }, + {}, + ) + return { + ...inputColumns, + [`App Variant ${evaluation.variants[0].variantName} Output 0`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + [`App Variant ${evaluation.variants[1].variantName} Output 1`]: data?.columnData1 + ? data?.columnData1 + : data.outputs[1]?.variant_output, + ["Vote"]: + evaluation.variants.find((v: Variant) => v.variantId === data.vote)?.variantName || + data.vote, + ["Expected Output"]: + scenarios[ix]?.correctAnswer || evaluation.testset.csvdata[ix].correct_answer, + ["Additional notes"]: scenarios[ix]?.note, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.variants[1].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportSingleModelEvaluationData = ( + evaluation: Evaluation, + scenarios: EvaluationScenario[], + rows: GenericObject[], +) => { + const exportRow = rows.map((data, ix) => { + const inputColumns = evaluation.testset.testsetChatColumn + ? {Input: evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn]} + : data.inputs.reduce( + (columns: any, input: {input_name: string; input_value: string}) => { + columns[`${input.input_name}`] = input.input_value + return columns + }, + {}, + ) + const numericScore = parseInt(data.score) + return { + ...inputColumns, + [`App Variant ${evaluation.variants[0].variantName} Output 0`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Score"]: isNaN(numericScore) ? "-" : numericScore, + ["Expected Output"]: + scenarios[ix]?.correctAnswer || evaluation.testset.csvdata[ix].correct_answer, + ["Additional notes"]: scenarios[ix]?.note, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportRegexEvaluationData = ( + evaluation: Evaluation, + rows: GenericObject[], + settings: GenericObject, +) => { + const exportRow = rows.map((data, ix) => { + const isCorrect = data.score === "correct" + const isMatch = settings.regexShouldMatch ? isCorrect : !isCorrect + + return { + ["Inputs"]: + evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] || + data.inputs[0].input_value, + [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Match / Mismatch"]: isMatch ? "Match" : "Mismatch", + ["Evaluation"]: data.score, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportWebhookEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => { + const exportRow = rows.map((data, ix) => { + return { + ["Inputs"]: + evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] || + data.inputs[0].input_value, + [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Correct answer"]: data.correctAnswer, + ["Score"]: data.score, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const exportCustomCodeEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => { + const exportRow = rows.map((data, ix) => { + return { + ["Inputs"]: + evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] || + data.inputs[0].input_value, + [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0 + ? data?.columnData0 + : data.outputs[0]?.variant_output, + ["Correct answer"]: data.correctAnswer, + ["Score"]: data.score, + } + }) + const exportCol = Object.keys(exportRow[0]) + + const csvData = convertToCsv(exportRow, exportCol) + const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` + downloadCsv(csvData, filename) +} + +export const calculateResultsDataAvg = (resultsData: Record, multiplier = 10) => { + const obj = {...resultsData} + Object.keys(obj).forEach((key) => { + if (isNaN(+key)) delete obj[key] + }) + + const count = Object.values(obj).reduce((acc, value) => acc + +value, 0) + const sum = Object.keys(obj).reduce((acc, key) => acc + (parseFloat(key) || 0) * +obj[key], 0) + return (sum / count) * multiplier +} + +export const getVotesPercentage = (record: HumanEvaluationListTableDataType, index: number) => { + const variant = record.votesData.variants[index] + return record.votesData.variants_votes_data[variant]?.percentage +} + +export const checkIfResourceValidForDeletion = async ( + data: Omit[0], "appId">, +) => { + if (isDemo()) { + const response = await fetchEvaluatonIdsByResource(data) + if (response.data.length > 0) { + const name = + (data.resourceType === "testset" + ? "Testset" + : data.resourceType === "evaluator_config" + ? "Evaluator" + : "Variant") + (data.resourceIds.length > 1 ? "s" : "") + + const suffix = response.data.length > 1 ? "s" : "" + AlertPopup({ + title: `${name} is in use`, + message: `The ${name} is currently in used by ${response.data.length} evaluation${suffix}. Please delete the evaluation${suffix} first.`, + cancelText: null, + okText: "Ok", + }) + return false + } + } + return true +} + +export function getTypedValue(res?: TypedValue) { + const {value, type, error} = res || {} + if (type === "error") { + return error?.message + } + + if (value === undefined) return "-" + + switch (type) { + case "number": + return round(Number(value), 2) + case "boolean": + case "bool": + return capitalize(value?.toString()) + case "cost": + return formatCurrency(Number(value)) + case "latency": + return formatLatency(Number(value)) + case "string": + case "text": + return value?.toString() ?? "-" + case "code": + case "regex": + return value?.toString() ?? "-" + case "object": + return typeof value === "object" + ? JSON.stringify(value, null, 2) + : (value?.toString() ?? "-") + case "messages": + return Array.isArray(value) + ? value + .map((msg) => (typeof msg === "string" ? msg : JSON.stringify(msg))) + .join("\n") + : (value?.toString() ?? "-") + case "multiple_choice": + return Array.isArray(value) ? value.join(", ") : (value?.toString() ?? "-") + case "hidden": + return "-" + default: + return value?.toString() ?? "-" + } +} + +type CellDataType = "number" | "text" | "date" +export function getFilterParams(type: CellDataType) { + const filterParams: GenericObject = {} + if (type == "date") { + filterParams.comparator = function ( + filterLocalDateAtMidnight: Date, + cellValue: string | null, + ) { + if (cellValue == null) return -1 + const cellDate = dayjs(cellValue).startOf("day").toDate() + if (filterLocalDateAtMidnight.getTime() === cellDate.getTime()) { + return 0 + } + if (cellDate < filterLocalDateAtMidnight) { + return -1 + } + if (cellDate > filterLocalDateAtMidnight) { + return 1 + } + } + } + + return { + sortable: true, + floatingFilter: true, + filter: + type === "number" + ? "agNumberColumnFilter" + : type === "date" + ? "agDateColumnFilter" + : "agTextColumnFilter", + cellDataType: type === "number" ? "text" : type, + filterParams, + comparator: getCustomComparator(type), + } +} + +export const calcEvalDuration = (evaluation: _Evaluation) => { + return dayjs( + runningStatuses.includes(evaluation.status.value) ? Date.now() : evaluation.updated_at, + ).diff(dayjs(evaluation.created_at), "milliseconds") +} + +const getCustomComparator = (type: CellDataType) => (valueA: string, valueB: string) => { + const getNumber = (val: string) => { + const num = parseFloat(val || "0") + return isNaN(num) ? 0 : num + } + + valueA = String(valueA) + valueB = String(valueB) + + switch (type) { + case "date": + return dayjs(valueA).diff(dayjs(valueB)) + case "text": + return valueA.localeCompare(valueB) + case "number": + return getNumber(valueA) - getNumber(valueB) + default: + return 0 + } +} + +export const removeCorrectAnswerPrefix = (str: string) => { + return str.replace(/^correctAnswer_/, "") +} + +export const mapTestcaseAndEvalValues = ( + settingsValues: Record, + selectedTestcase: Record, +) => { + const testcaseObj: Record = {} + const evalMapObj: Record = {} + + Object.entries(settingsValues).forEach(([key, value]) => { + if (typeof value === "string" && value.startsWith("testcase.")) { + testcaseObj[key] = selectedTestcase[value.split(".")[1]] + } else { + evalMapObj[key] = value + } + }) + + return {testcaseObj, evalMapObj} +} + +export const transformTraceKeysInSettings = ( + settingsValues: Record, +): Record => { + return Object.keys(settingsValues).reduce( + (acc, curr) => { + if ( + !acc[curr] && + typeof settingsValues[curr] === "string" && + settingsValues[curr].startsWith("trace.") + ) { + acc[curr] = settingsValues[curr].replace("trace.", "") + } else { + acc[curr] = settingsValues[curr] + } + + return acc + }, + {} as Record, + ) +} + +export const getEvaluatorTags = () => { + const evaluatorTags = [ + { + label: "Classifiers", + value: "classifiers", + }, + { + label: "Similarity", + value: "similarity", + }, + { + label: "AI / LLM", + value: "ai_llm", + }, + { + label: "Functional", + value: "functional", + }, + ] + + if (isDemo()) { + evaluatorTags.unshift({ + label: "RAG", + value: "rag", + }) + } + + return evaluatorTags +} + +export const calculateAvgScore = (evaluation: SingleModelEvaluationListTableDataType) => { + let score = 0 + if (evaluation.scoresData) { + score = + ((evaluation.scoresData.correct?.length || evaluation.scoresData.true?.length || 0) / + evaluation.scoresData.nb_of_rows) * + 100 + } else if (evaluation.resultsData) { + const multiplier = { + [EvaluationType.auto_webhook_test]: 100, + [EvaluationType.single_model_test]: 1, + } + score = calculateResultsDataAvg( + evaluation.resultsData, + multiplier[evaluation.evaluationType as keyof typeof multiplier], + ) + score = isNaN(score) ? 0 : score + } else if (evaluation.avgScore) { + score = evaluation.avgScore * 100 + } + + return score +} diff --git a/web/ee/src/lib/helpers/hashUtils.ts b/web/ee/src/lib/helpers/hashUtils.ts new file mode 100644 index 0000000000..5c66724e5a --- /dev/null +++ b/web/ee/src/lib/helpers/hashUtils.ts @@ -0,0 +1,73 @@ +// Utility to generate a hash ID for annotation/invocation steps, aligned with backend make_hash_id +// Uses blake2b if available, otherwise falls back to SHA-256 + +import blake from "blakejs" +// import { v4 as uuidv4 } from "uuid" // Use this for UUIDs if needed + +const REFERENCE_KEYS = [ + "application", + "application_variant", + "application_revision", + "testset", + "testcase", + "evaluator", +] + +// Recursively stable, whitespace-free JSON stringifier +function stableStringifyRecursive(obj: any): string { + if (obj === null || typeof obj !== "object") { + return JSON.stringify(obj) + } + if (Array.isArray(obj)) { + return `[${obj.map(stableStringifyRecursive).join(",")}]` + } + const keys = Object.keys(obj).sort() + const entries = keys.map( + (key) => `${JSON.stringify(key)}:${stableStringifyRecursive(obj[key])}`, + ) + return `{${entries.join(",")}}` +} + +export function makeHashId({ + references, + links, +}: { + references?: Record + links?: Record +}): string { + if (!references && !links) return "" + const payload: Record = {} + + for (const k of Object.keys(references || {})) { + if (REFERENCE_KEYS.includes(k)) { + const v = references![k] + // Only include 'id' field, not 'slug' + if (v.id != null) { + payload[k] = {id: v.id} + } + } + } + for (const k of Object.keys(links || {})) { + const v = links![k] + payload[k] = { + span_id: v.span_id, + trace_id: v.trace_id, + } + } + // Stable, deep, whitespace-free JSON + const serialized = stableStringifyRecursive(payload) + + // blake2b hash (digest_size=16) + try { + // Use blakejs (same as backend example) + return blake.blake2bHex(serialized, null, 16) + } catch (e) { + // Fallback: SHA-256 + if (window.crypto?.subtle) { + throw new Error( + "blake2b not available and crypto.subtle is async. Provide a polyfill or use a sync fallback.", + ) + } + return btoa(serialized) + } +} diff --git a/web/oss/src/lib/helpers/serviceValidations.ts b/web/ee/src/lib/helpers/serviceValidations.ts similarity index 100% rename from web/oss/src/lib/helpers/serviceValidations.ts rename to web/ee/src/lib/helpers/serviceValidations.ts diff --git a/web/oss/src/lib/helpers/traceUtils.ts b/web/ee/src/lib/helpers/traceUtils.ts similarity index 98% rename from web/oss/src/lib/helpers/traceUtils.ts rename to web/ee/src/lib/helpers/traceUtils.ts index 536a7b5bc6..f232711598 100644 --- a/web/oss/src/lib/helpers/traceUtils.ts +++ b/web/ee/src/lib/helpers/traceUtils.ts @@ -1,6 +1,6 @@ import {uuidToTraceId} from "@/oss/lib/hooks/useAnnotations/assets/helpers" -import {TraceData, TraceTree} from "@agenta/oss/src/lib/hooks/useEvaluationRunScenarioSteps/types" +import {TraceData, TraceTree} from "../hooks/useEvaluationRunScenarioSteps/types" export function findTraceForStep(traces: any[] | undefined, traceId?: string): any | undefined { if (!traces?.length || !traceId) return undefined diff --git a/web/oss/src/lib/hooks/useEvalScenarioQueue/index.ts b/web/ee/src/lib/hooks/useEvalScenarioQueue/index.ts similarity index 99% rename from web/oss/src/lib/hooks/useEvalScenarioQueue/index.ts rename to web/ee/src/lib/hooks/useEvalScenarioQueue/index.ts index 4250637c7d..bfeead6e7b 100644 --- a/web/oss/src/lib/hooks/useEvalScenarioQueue/index.ts +++ b/web/ee/src/lib/hooks/useEvalScenarioQueue/index.ts @@ -16,7 +16,7 @@ import {evaluationRunStateFamily} from "@/oss/lib/hooks/useEvaluationRunData/ass import {useJwtRefresher} from "@/oss/lib/hooks/useJWT" import {EvaluationStatus} from "@/oss/lib/Types" import {slugify} from "@/oss/lib/utils/slugify" -import type {ConfigMessage, ResultMessage, RunEvalMessage} from "@/oss/lib/evalRunner/types" +import type {ConfigMessage, ResultMessage, RunEvalMessage} from "@/oss/lib/workers/evalRunner/types" import {getProjectValues} from "@/oss/state/project" // import {setOptimisticStepData} from "../../../components/EvalRunDetails/assets/optimisticUtils" @@ -169,7 +169,7 @@ export function useEvalScenarioQueue(options?: {concurrency?: number; runId?: st useEffect(() => { if (!sharedWorker) { sharedWorker = new Worker( - new URL("@/oss/lib/evalRunner/evalRunner.worker.ts", import.meta.url), + new URL("@/oss/lib/workers/evalRunner/evalRunner.worker.ts", import.meta.url), ) } diff --git a/web/oss/src/lib/hooks/useEvalScenarioQueue/responseQueue.ts b/web/ee/src/lib/hooks/useEvalScenarioQueue/responseQueue.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvalScenarioQueue/responseQueue.ts rename to web/ee/src/lib/hooks/useEvalScenarioQueue/responseQueue.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/bulkFetch.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/bulkFetch.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/bulkFetch.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/bulkFetch.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/cache.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/cache.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/cache.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/cache.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/index.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/index.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/index.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/index.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/migrationHelper.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/migrationHelper.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/migrationHelper.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/migrationHelper.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/progress.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/progress.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/progress.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/progress.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedAtoms.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedAtoms.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedAtoms.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedAtoms.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts similarity index 99% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts index 4cea5caa76..41abde06a9 100644 --- a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts +++ b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics.ts @@ -23,7 +23,7 @@ import {evalAtomStore} from "./store" // Re-export the atom families for external use export {runMetricsCacheFamily, runMetricsStatsCacheFamily} -import {fetchRunMetricsViaWorker} from "@/agenta-oss-common/lib/evalRunner/runMetricsWorker" +import {fetchRunMetricsViaWorker} from "@/agenta-oss-common/lib/workers/evalRunner/runMetricsWorker" // Helper: flatten acc object and nested metrics similar to legacy mergedMetricsAtom export function flattenMetrics(raw: Record): Record { diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/types.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/types.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/types.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/utils.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/utils.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/atoms/utils.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/utils.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/constants.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/constants.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/constants.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/constants.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/buildRunIndex.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/buildRunIndex.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/buildRunIndex.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/buildRunIndex.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts similarity index 93% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts index b39b1c86e2..ace3e7fa8e 100644 --- a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts +++ b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioListViaWorker.ts @@ -7,7 +7,7 @@ let _worker: Worker | null = null function getWorker() { if (!_worker) { _worker = new Worker( - new URL("@/oss/lib/evalRunner/scenarioListWorker.ts", import.meta.url), + new URL("@/oss/lib/workers/evalRunner/scenarioListWorker.ts", import.meta.url), { type: "module", }, diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts similarity index 99% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts index 2b81ed1826..667b7858ba 100644 --- a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts +++ b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/fetchScenarioViaWorker.ts @@ -85,7 +85,7 @@ const performFetch = async ( const {jwt, apiUrl, projectId} = await buildAuthContext() const {fetchStepsViaWorker} = await import( - "@/oss/lib/evalRunner/bulkWorker" + "@/agenta-oss-common/lib/workers/evalRunner/bulkWorker" ) const store = evalAtomStore() diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/scenarioFilters.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/scenarioFilters.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/scenarioFilters.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/scenarioFilters.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/index.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/index.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/index.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/index.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/types.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/types.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/assets/helpers/workerContext/types.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/index.ts b/web/ee/src/lib/hooks/useEvaluationRunData/index.ts similarity index 99% rename from web/oss/src/lib/hooks/useEvaluationRunData/index.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/index.ts index 7be13af806..df4bef1420 100644 --- a/web/oss/src/lib/hooks/useEvaluationRunData/index.ts +++ b/web/ee/src/lib/hooks/useEvaluationRunData/index.ts @@ -83,7 +83,7 @@ const useEvaluationRunData = (evaluationTableId: string | null, debug = false, r const projectId = useAtomValue(projectIdAtom) const setProjectVariantReferences = useSetAtom(setProjectVariantReferencesAtom) const user = useAtomValue(userAtom) - const requireUser = true + const requireUser = isDemo() const enrichRun = useEnrichEvaluationRun({debug, evalType}) const suppressLoadingRef = useRef(false) diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/refreshLiveRun.ts b/web/ee/src/lib/hooks/useEvaluationRunData/refreshLiveRun.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/refreshLiveRun.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/refreshLiveRun.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/types.ts b/web/ee/src/lib/hooks/useEvaluationRunData/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/types.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/types.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx b/web/ee/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx rename to web/ee/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx diff --git a/web/oss/src/lib/hooks/useEvaluationRunData/useScenarioStepSnapshot.ts b/web/ee/src/lib/hooks/useEvaluationRunData/useScenarioStepSnapshot.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunData/useScenarioStepSnapshot.ts rename to web/ee/src/lib/hooks/useEvaluationRunData/useScenarioStepSnapshot.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunMetrics/assets/utils.ts b/web/ee/src/lib/hooks/useEvaluationRunMetrics/assets/utils.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunMetrics/assets/utils.ts rename to web/ee/src/lib/hooks/useEvaluationRunMetrics/assets/utils.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunMetrics/index.ts b/web/ee/src/lib/hooks/useEvaluationRunMetrics/index.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunMetrics/index.ts rename to web/ee/src/lib/hooks/useEvaluationRunMetrics/index.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunMetrics/types.ts b/web/ee/src/lib/hooks/useEvaluationRunMetrics/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunMetrics/types.ts rename to web/ee/src/lib/hooks/useEvaluationRunMetrics/types.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunScenarioSteps/types.ts b/web/ee/src/lib/hooks/useEvaluationRunScenarioSteps/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunScenarioSteps/types.ts rename to web/ee/src/lib/hooks/useEvaluationRunScenarioSteps/types.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunScenarios/index.ts b/web/ee/src/lib/hooks/useEvaluationRunScenarios/index.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunScenarios/index.ts rename to web/ee/src/lib/hooks/useEvaluationRunScenarios/index.ts diff --git a/web/oss/src/lib/hooks/useEvaluationRunScenarios/types.ts b/web/ee/src/lib/hooks/useEvaluationRunScenarios/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluationRunScenarios/types.ts rename to web/ee/src/lib/hooks/useEvaluationRunScenarios/types.ts diff --git a/web/oss/src/lib/hooks/useEvaluations.ts b/web/ee/src/lib/hooks/useEvaluations.ts similarity index 100% rename from web/oss/src/lib/hooks/useEvaluations.ts rename to web/ee/src/lib/hooks/useEvaluations.ts diff --git a/web/oss/src/lib/hooks/useInvocationResult/index.ts b/web/ee/src/lib/hooks/useInvocationResult/index.ts similarity index 99% rename from web/oss/src/lib/hooks/useInvocationResult/index.ts rename to web/ee/src/lib/hooks/useInvocationResult/index.ts index 201f709a48..a61fc59695 100644 --- a/web/oss/src/lib/hooks/useInvocationResult/index.ts +++ b/web/ee/src/lib/hooks/useInvocationResult/index.ts @@ -4,7 +4,7 @@ import {useAtomValue} from "jotai" import {renderChatMessages} from "@/oss/components/EvalRunDetails/assets/renderChatMessages" import {evalTypeAtom} from "@/oss/components/EvalRunDetails/state/evalType" -import {useRunId} from "@agenta/oss/src/contexts/RunIdContext" +import {useRunId} from "@/oss/contexts/RunIdContext" import axios from "@/oss/lib/api/assets/axiosConfig" import {snakeToCamelCaseKeys} from "@/oss/lib/helpers/casing" import {readInvocationResponse} from "@/oss/lib/helpers/traceUtils" diff --git a/web/oss/src/lib/hooks/useInvocationResult/types.ts b/web/ee/src/lib/hooks/useInvocationResult/types.ts similarity index 100% rename from web/oss/src/lib/hooks/useInvocationResult/types.ts rename to web/ee/src/lib/hooks/useInvocationResult/types.ts diff --git a/web/oss/src/lib/hooks/usePreviewEvaluations/assets/utils.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewEvaluations/assets/utils.ts rename to web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts diff --git a/web/oss/src/lib/hooks/usePreviewEvaluations/index.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/index.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewEvaluations/index.ts rename to web/ee/src/lib/hooks/usePreviewEvaluations/index.ts diff --git a/web/oss/src/lib/hooks/usePreviewEvaluations/projectVariantConfigs.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/projectVariantConfigs.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewEvaluations/projectVariantConfigs.ts rename to web/ee/src/lib/hooks/usePreviewEvaluations/projectVariantConfigs.ts diff --git a/web/oss/src/lib/hooks/usePreviewEvaluations/states/queryFilterAtoms.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/states/queryFilterAtoms.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewEvaluations/states/queryFilterAtoms.ts rename to web/ee/src/lib/hooks/usePreviewEvaluations/states/queryFilterAtoms.ts diff --git a/web/oss/src/lib/hooks/usePreviewEvaluations/types.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/types.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewEvaluations/types.ts rename to web/ee/src/lib/hooks/usePreviewEvaluations/types.ts diff --git a/web/oss/src/lib/hooks/usePreviewRunningEvaluations/index.ts b/web/ee/src/lib/hooks/usePreviewRunningEvaluations/index.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewRunningEvaluations/index.ts rename to web/ee/src/lib/hooks/usePreviewRunningEvaluations/index.ts diff --git a/web/oss/src/lib/hooks/usePreviewRunningEvaluations/states/runningEvalAtom.ts b/web/ee/src/lib/hooks/usePreviewRunningEvaluations/states/runningEvalAtom.ts similarity index 100% rename from web/oss/src/lib/hooks/usePreviewRunningEvaluations/states/runningEvalAtom.ts rename to web/ee/src/lib/hooks/usePreviewRunningEvaluations/states/runningEvalAtom.ts diff --git a/web/oss/src/lib/hooks/useRunMetricsMap/index.ts b/web/ee/src/lib/hooks/useRunMetricsMap/index.ts similarity index 100% rename from web/oss/src/lib/hooks/useRunMetricsMap/index.ts rename to web/ee/src/lib/hooks/useRunMetricsMap/index.ts diff --git a/web/oss/src/lib/metricColumnFactory.tsx b/web/ee/src/lib/metricColumnFactory.tsx similarity index 100% rename from web/oss/src/lib/metricColumnFactory.tsx rename to web/ee/src/lib/metricColumnFactory.tsx diff --git a/web/oss/src/lib/metricSorter.ts b/web/ee/src/lib/metricSorter.ts similarity index 100% rename from web/oss/src/lib/metricSorter.ts rename to web/ee/src/lib/metricSorter.ts diff --git a/web/oss/src/lib/metricUtils.ts b/web/ee/src/lib/metricUtils.ts similarity index 100% rename from web/oss/src/lib/metricUtils.ts rename to web/ee/src/lib/metricUtils.ts diff --git a/web/oss/src/lib/metrics/utils.ts b/web/ee/src/lib/metrics/utils.ts similarity index 100% rename from web/oss/src/lib/metrics/utils.ts rename to web/ee/src/lib/metrics/utils.ts diff --git a/web/oss/src/lib/tableUtils.ts b/web/ee/src/lib/tableUtils.ts similarity index 100% rename from web/oss/src/lib/tableUtils.ts rename to web/ee/src/lib/tableUtils.ts diff --git a/web/oss/src/lib/types_ee.ts b/web/ee/src/lib/types_ee.ts similarity index 100% rename from web/oss/src/lib/types_ee.ts rename to web/ee/src/lib/types_ee.ts diff --git a/web/oss/src/lib/evalRunner/bulkWorker.ts b/web/ee/src/lib/workers/evalRunner/bulkWorker.ts similarity index 100% rename from web/oss/src/lib/evalRunner/bulkWorker.ts rename to web/ee/src/lib/workers/evalRunner/bulkWorker.ts diff --git a/web/oss/src/lib/evalRunner/evalRunner.worker.ts b/web/ee/src/lib/workers/evalRunner/evalRunner.worker.ts similarity index 100% rename from web/oss/src/lib/evalRunner/evalRunner.worker.ts rename to web/ee/src/lib/workers/evalRunner/evalRunner.worker.ts diff --git a/web/oss/src/lib/evalRunner/fetchRunMetrics.worker.ts b/web/ee/src/lib/workers/evalRunner/fetchRunMetrics.worker.ts similarity index 100% rename from web/oss/src/lib/evalRunner/fetchRunMetrics.worker.ts rename to web/ee/src/lib/workers/evalRunner/fetchRunMetrics.worker.ts diff --git a/web/oss/src/lib/evalRunner/fetchSteps.worker.ts b/web/ee/src/lib/workers/evalRunner/fetchSteps.worker.ts similarity index 100% rename from web/oss/src/lib/evalRunner/fetchSteps.worker.ts rename to web/ee/src/lib/workers/evalRunner/fetchSteps.worker.ts diff --git a/web/oss/src/lib/evalRunner/pureEnrichment.ts b/web/ee/src/lib/workers/evalRunner/pureEnrichment.ts similarity index 100% rename from web/oss/src/lib/evalRunner/pureEnrichment.ts rename to web/ee/src/lib/workers/evalRunner/pureEnrichment.ts diff --git a/web/oss/src/lib/evalRunner/runMetricsWorker.ts b/web/ee/src/lib/workers/evalRunner/runMetricsWorker.ts similarity index 100% rename from web/oss/src/lib/evalRunner/runMetricsWorker.ts rename to web/ee/src/lib/workers/evalRunner/runMetricsWorker.ts diff --git a/web/oss/src/lib/evalRunner/scenarioListWorker.ts b/web/ee/src/lib/workers/evalRunner/scenarioListWorker.ts similarity index 100% rename from web/oss/src/lib/evalRunner/scenarioListWorker.ts rename to web/ee/src/lib/workers/evalRunner/scenarioListWorker.ts diff --git a/web/oss/src/lib/evalRunner/types.ts b/web/ee/src/lib/workers/evalRunner/types.ts similarity index 89% rename from web/oss/src/lib/evalRunner/types.ts rename to web/ee/src/lib/workers/evalRunner/types.ts index 16f1173255..1b98796efd 100644 --- a/web/oss/src/lib/evalRunner/types.ts +++ b/web/ee/src/lib/workers/evalRunner/types.ts @@ -1,6 +1,6 @@ import {EvaluationStatus} from "@/oss/lib/Types" -import {IStepResponse} from "@agenta/oss/src/lib/hooks/useEvaluationRunScenarioSteps/types" +import {IStepResponse} from "../../hooks/useEvaluationRunScenarioSteps/types" export interface RunEvalMessage { type: "run-invocation" diff --git a/web/oss/src/lib/evalRunner/workerFetch.ts b/web/ee/src/lib/workers/evalRunner/workerFetch.ts similarity index 98% rename from web/oss/src/lib/evalRunner/workerFetch.ts rename to web/ee/src/lib/workers/evalRunner/workerFetch.ts index 2cf69dc70b..2e45a07575 100644 --- a/web/oss/src/lib/evalRunner/workerFetch.ts +++ b/web/ee/src/lib/workers/evalRunner/workerFetch.ts @@ -20,8 +20,8 @@ import {PreviewTestcase, PreviewTestset} from "@/oss/lib/Types" import { deserializeRunIndex, RunIndex, -} from "@agenta/oss/src/lib/hooks/useEvaluationRunData/assets/helpers/buildRunIndex" -import {EvalRunDataContextType} from "@agenta/oss/src/lib/hooks/useEvaluationRunData/types" +} from "../../hooks/useEvaluationRunData/assets/helpers/buildRunIndex" +import {EvalRunDataContextType} from "../../hooks/useEvaluationRunData/types" import { buildScenarioCore, diff --git a/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx b/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx index 97047a940b..df1b8461be 100644 --- a/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx +++ b/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx @@ -1,3 +1,20 @@ -import EvaluatorConfigureRoute from "@agenta/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id]" +import {useMemo} from "react" + +import {useRouter} from "next/router" + +import ConfigureEvaluatorPage from "@/oss/components/Evaluators/components/ConfigureEvaluator" + +const EvaluatorConfigureRoute = () => { + const router = useRouter() + const evaluatorId = useMemo(() => { + const id = router.query.evaluator_id + if (Array.isArray(id)) { + return id[0] + } + return id ?? null + }, [router.query.evaluator_id]) + + return +} export default EvaluatorConfigureRoute diff --git a/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx b/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx index a5cb6daf29..7996228a65 100644 --- a/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx +++ b/web/ee/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx @@ -1,3 +1,7 @@ -import ProjectEvaluatorsPage from "@agenta/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index" +import EvaluatorsRegistry from "@/oss/components/Evaluators" + +const ProjectEvaluatorsPage = () => { + return +} export default ProjectEvaluatorsPage diff --git a/web/oss/src/services/evaluationRuns/api/index.ts b/web/ee/src/services/evaluationRuns/api/index.ts similarity index 100% rename from web/oss/src/services/evaluationRuns/api/index.ts rename to web/ee/src/services/evaluationRuns/api/index.ts diff --git a/web/oss/src/services/evaluationRuns/api/types.ts b/web/ee/src/services/evaluationRuns/api/types.ts similarity index 100% rename from web/oss/src/services/evaluationRuns/api/types.ts rename to web/ee/src/services/evaluationRuns/api/types.ts diff --git a/web/ee/src/services/evaluationRuns/utils.ts b/web/ee/src/services/evaluationRuns/utils.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/web/oss/src/services/evaluations/api/index.ts b/web/ee/src/services/evaluations/api/index.ts similarity index 100% rename from web/oss/src/services/evaluations/api/index.ts rename to web/ee/src/services/evaluations/api/index.ts diff --git a/web/oss/src/services/evaluations/api_ee/index.ts b/web/ee/src/services/evaluations/api_ee/index.ts similarity index 97% rename from web/oss/src/services/evaluations/api_ee/index.ts rename to web/ee/src/services/evaluations/api_ee/index.ts index cab7fe809b..4ae4376f4d 100644 --- a/web/oss/src/services/evaluations/api_ee/index.ts +++ b/web/ee/src/services/evaluations/api_ee/index.ts @@ -14,7 +14,7 @@ import { EvaluatorMappingInput, EvaluatorMappingOutput, EvaluatorOutputInterface, -} from "@agenta/oss/src/lib/types_ee" +} from "../../../lib/types_ee" export const createEvaluatorDataMapping = async ( config: EvaluatorMappingInput, diff --git a/web/oss/src/services/evaluations/workerUtils.ts b/web/ee/src/services/evaluations/workerUtils.ts similarity index 100% rename from web/oss/src/services/evaluations/workerUtils.ts rename to web/ee/src/services/evaluations/workerUtils.ts diff --git a/web/oss/src/services/human-evaluations/api/index.ts b/web/ee/src/services/human-evaluations/api/index.ts similarity index 100% rename from web/oss/src/services/human-evaluations/api/index.ts rename to web/ee/src/services/human-evaluations/api/index.ts diff --git a/web/oss/src/services/human-evaluations/hooks/useEvaluationResults.ts b/web/ee/src/services/human-evaluations/hooks/useEvaluationResults.ts similarity index 100% rename from web/oss/src/services/human-evaluations/hooks/useEvaluationResults.ts rename to web/ee/src/services/human-evaluations/hooks/useEvaluationResults.ts diff --git a/web/ee/src/services/onlineEvaluations/api.ts b/web/ee/src/services/onlineEvaluations/api.ts new file mode 100644 index 0000000000..e0650b45d9 --- /dev/null +++ b/web/ee/src/services/onlineEvaluations/api.ts @@ -0,0 +1,188 @@ +import axios from "@/oss/lib/api/assets/axiosConfig" +import {getAgentaApiUrl} from "@/oss/lib/helpers/api" +import {getProjectValues} from "@/oss/state/project" + +type LogicalOperator = "and" | "or" | "not" | "nand" | "nor" + +export interface QueryConditionPayload { + field: string + key?: string + value?: unknown + operator?: string + options?: Record +} + +export interface QueryFilteringPayload { + operator?: LogicalOperator + conditions: (QueryConditionPayload | QueryFilteringPayload)[] +} + +export interface QueryWindowingPayload { + newest?: string + oldest?: string + next?: string + limit?: number + order?: "ascending" | "descending" + interval?: number + rate?: number +} + +export interface QueryRevisionDataPayload { + filtering?: QueryFilteringPayload + windowing?: QueryWindowingPayload +} + +export interface SimpleQueryCreatePayload { + slug: string + name?: string + description?: string + flags?: Record + tags?: Record + meta?: Record + data?: QueryRevisionDataPayload +} + +export interface SimpleQueryCreateRequest { + query: SimpleQueryCreatePayload +} + +export interface SimpleQueryResponse { + count: number + query?: { + id: string + slug?: string + data?: QueryRevisionDataPayload + meta?: Record + } | null +} + +export interface QueryRevisionRetrieveRequest { + query_ref?: {id?: string; slug?: string} | null + query_variant_ref?: {id?: string; slug?: string} | null + query_revision_ref?: {id?: string; slug?: string} | null +} + +export interface QueryRevisionResponse { + count: number + query_revision?: { + id?: string + slug?: string + variant_id?: string + version?: string | number + data?: QueryRevisionDataPayload + } | null +} + +export interface SimpleEvaluationFlagsPayload { + is_live?: boolean + is_closed?: boolean + is_active?: boolean +} + +export interface SimpleEvaluationDataPayload { + status?: string + query_steps?: string[] | Record + testset_steps?: Record + application_steps?: Record + evaluator_steps?: string[] | Record + repeats?: number + // Structured references for online evaluations + query_ref?: {id?: string; slug?: string} | null + query_revision_ref?: {id?: string; slug?: string} | null + evaluator_ref?: {id?: string; slug?: string} | null + configuration?: Record +} + +export interface SimpleEvaluationCreatePayload { + name?: string + description?: string + flags?: SimpleEvaluationFlagsPayload + tags?: Record + meta?: Record + data: SimpleEvaluationDataPayload +} + +export interface SimpleEvaluationCreateRequest { + evaluation: SimpleEvaluationCreatePayload +} + +export interface SimpleEvaluationResponse { + count: number + evaluation?: SimpleEvaluationPayload | null +} + +export interface SimpleEvaluationPayload { + id?: string + slug?: string + name?: string + description?: string + created_at?: string + updated_at?: string + created_by_id?: string + updated_by_id?: string + flags?: SimpleEvaluationFlagsPayload + data?: SimpleEvaluationDataPayload + meta?: Record + tags?: Record +} + +export interface SimpleEvaluationsResponse { + count: number + evaluations: SimpleEvaluationPayload[] +} + +export interface SimpleEvaluationsQueryRequest { + evaluation?: { + flags?: SimpleEvaluationFlagsPayload + ids?: string[] + } + tags?: Record + meta?: Record +} + +const getProjectUrl = (path: string) => { + const {projectId} = getProjectValues() + return `${getAgentaApiUrl()}${path}?project_id=${projectId}` +} + +export const createSimpleQuery = async ( + payload: SimpleQueryCreateRequest, +): Promise => { + const {data} = await axios.post(getProjectUrl("/preview/simple/queries/"), payload) + return data as SimpleQueryResponse +} + +export const retrieveQueryRevision = async ( + payload: QueryRevisionRetrieveRequest, +): Promise => { + const {data} = await axios.post(getProjectUrl("/preview/queries/revisions/retrieve"), payload) + return data as QueryRevisionResponse +} + +export const createSimpleEvaluation = async ( + payload: SimpleEvaluationCreateRequest, +): Promise => { + const {data} = await axios.post(getProjectUrl("/preview/simple/evaluations/"), payload) + return data as SimpleEvaluationResponse +} + +export const querySimpleEvaluations = async ( + payload?: SimpleEvaluationsQueryRequest, +): Promise => { + const url = getProjectUrl("/preview/simple/evaluations/query") + const body = payload ?? {} + const {data} = await axios.post(url, body) + return data as SimpleEvaluationsResponse +} + +export const stopSimpleEvaluation = async (evaluationId: string) => { + const url = getProjectUrl(`/preview/simple/evaluations/${evaluationId}/stop`) + const {data} = await axios.post(url) + return data +} + +export const startSimpleEvaluation = async (evaluationId: string) => { + const url = getProjectUrl(`/preview/simple/evaluations/${evaluationId}/start`) + const {data} = await axios.post(url) + return data +} diff --git a/web/ee/src/services/promptVersioning/api/index.ts b/web/ee/src/services/promptVersioning/api/index.ts new file mode 100644 index 0000000000..d51cd8ac75 --- /dev/null +++ b/web/ee/src/services/promptVersioning/api/index.ts @@ -0,0 +1,41 @@ +import axios from "@/oss/lib/api/assets/axiosConfig" +import {getAgentaApiUrl} from "@/oss/lib/helpers/api" +import {getProjectValues} from "@/oss/state/project" + +//Prefix convention: +// - fetch: GET single entity from server +// - fetchAll: GET all entities from server +// - create: POST data to server +// - update: PUT data to server +// - delete: DELETE data from server + +// versioning +export const fetchAllPromptVersioning = async (variantId: string, ignoreAxiosError = false) => { + const {projectId} = getProjectValues() + + const {data} = await axios.get( + `${getAgentaApiUrl()}/variants/${variantId}/revisions?project_id=${projectId}`, + { + _ignoreError: ignoreAxiosError, + } as any, + ) + + return data +} + +export const fetchPromptRevision = async ( + variantId: string, + revisionNumber: number, + ignoreAxiosError = false, +) => { + const {projectId} = getProjectValues() + + const {data} = await axios.get( + `${getAgentaApiUrl()}/variants/${variantId}/revisions/${revisionNumber}?project_id=${projectId}`, + { + _ignoreError: ignoreAxiosError, + } as any, + ) + + return data +} diff --git a/web/oss/src/services/runMetrics/api/assets/contants.ts b/web/ee/src/services/runMetrics/api/assets/contants.ts similarity index 100% rename from web/oss/src/services/runMetrics/api/assets/contants.ts rename to web/ee/src/services/runMetrics/api/assets/contants.ts diff --git a/web/oss/src/services/runMetrics/api/index.ts b/web/ee/src/services/runMetrics/api/index.ts similarity index 100% rename from web/oss/src/services/runMetrics/api/index.ts rename to web/ee/src/services/runMetrics/api/index.ts diff --git a/web/oss/src/services/runMetrics/api/types.ts b/web/ee/src/services/runMetrics/api/types.ts similarity index 100% rename from web/oss/src/services/runMetrics/api/types.ts rename to web/ee/src/services/runMetrics/api/types.ts diff --git a/web/oss/src/services/variantConfigs/api/index.ts b/web/ee/src/services/variantConfigs/api/index.ts similarity index 100% rename from web/oss/src/services/variantConfigs/api/index.ts rename to web/ee/src/services/variantConfigs/api/index.ts diff --git a/web/oss/src/state/url/focusDrawer.ts b/web/ee/src/state/url/focusDrawer.ts similarity index 100% rename from web/oss/src/state/url/focusDrawer.ts rename to web/ee/src/state/url/focusDrawer.ts diff --git a/web/oss/next.config.ts b/web/oss/next.config.ts index fd65177200..be4f43f167 100644 --- a/web/oss/next.config.ts +++ b/web/oss/next.config.ts @@ -1,13 +1,7 @@ -import {createRequire} from "module" import path from "path" import type {NextConfig} from "next" -const require = createRequire(import.meta.url) -const reduxToolkitCjsEntry = path.join( - path.dirname(require.resolve("@reduxjs/toolkit/package.json")), - "dist/cjs/index.js", -) const isDevelopment = process.env.NODE_ENV === "development" const COMMON_CONFIG: NextConfig = { @@ -68,12 +62,6 @@ const COMMON_CONFIG: NextConfig = { "@ant-design/icons-svg", ], webpack: (config, {webpack, isServer}) => { - config.resolve ??= {} - config.resolve.alias = { - ...(config.resolve.alias ?? {}), - "@reduxjs/toolkit": reduxToolkitCjsEntry, - } - const envs: Record = {} config.cache = false diff --git a/web/oss/package.json b/web/oss/package.json index e5e89e0037..520e433bde 100644 --- a/web/oss/package.json +++ b/web/oss/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/oss", - "version": "0.62.1", + "version": "0.61.2", "private": true, "engines": { "node": ">=18" @@ -113,7 +113,6 @@ "react-resizable": "^3.0.5", "react-syntax-highlighter": "^15.6.0", "react-window": "^1.8.11", - "recharts": "^3.1.0", "semver": "^7.7.2", "shiki": "^3.12.2", "stable-hash": "^0.0.6", diff --git a/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx b/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx index 2b52c69b21..dd8fa3cbbd 100644 --- a/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx +++ b/web/oss/src/components/Sidebar/hooks/useSidebarConfig/index.tsx @@ -55,14 +55,14 @@ export const useSidebarConfig = () => { key: "project-evaluators-link", title: "Evaluators", link: `${projectURL}/evaluators`, - // isHidden: !isDemo(), + isHidden: !isDemo(), icon: , }, { key: "project-evaluations-link", title: "Evaluations", link: `${projectURL}/evaluations`, - // isHidden: !isDemo(), + isHidden: !isDemo(), icon: , }, { @@ -96,7 +96,7 @@ export const useSidebarConfig = () => { key: "app-evaluations-link", title: "Evaluations", link: `${appURL || recentlyVisitedAppURL}/evaluations`, - isHidden: !currentApp && !recentlyVisitedAppId, + isHidden: (!currentApp && !recentlyVisitedAppId) || !isDemo(), icon: , }, { diff --git a/web/oss/src/lib/atoms/breadcrumb/index.ts b/web/oss/src/lib/atoms/breadcrumb/index.ts index f32ee2c155..9a88b979e8 100644 --- a/web/oss/src/lib/atoms/breadcrumb/index.ts +++ b/web/oss/src/lib/atoms/breadcrumb/index.ts @@ -56,6 +56,25 @@ export const prependBreadcrumbAtom = atom(null, (get, set, item: BreadcrumbAtom) set(breadcrumbOverridesAtom, {...item, ...current}) }) +export const removeBreadcrumbsAtom = atom(null, (get, set, keys: string[] | undefined) => { + if (!keys || keys.length === 0) return + + const current = get(breadcrumbOverridesAtom) || {} + let changed = false + const next = {...current} + + keys.forEach((key) => { + if (Object.prototype.hasOwnProperty.call(next, key)) { + delete next[key] + changed = true + } + }) + + if (changed) { + set(breadcrumbOverridesAtom, next) + } +}) + // Helper atom to clear breadcrumbs (reset to URL-based) export const clearBreadcrumbsAtom = atom(null, (get, set) => { set(breadcrumbOverridesAtom, {}) diff --git a/web/oss/src/lib/helpers/buildBreadcrumbs.ts b/web/oss/src/lib/helpers/buildBreadcrumbs.ts index 5b36db44eb..2f920c98d6 100644 --- a/web/oss/src/lib/helpers/buildBreadcrumbs.ts +++ b/web/oss/src/lib/helpers/buildBreadcrumbs.ts @@ -2,7 +2,7 @@ import {BreadcrumbAtom} from "@/oss/lib/atoms/breadcrumb/types" import {isUuid} from "@/oss/lib/helpers/utils" import {ListAppsItem} from "@/oss/lib/Types" -const IGNORE_PATHS = new Set(["testsets", "evaluations", "settings"]) +const IGNORE_PATHS = new Set(["testsets", "evaluations", "settings", "configure"]) export interface BreadcrumbContext { uriPath: string diff --git a/web/oss/src/lib/helpers/evaluate.ts b/web/oss/src/lib/helpers/evaluate.ts index c3c280f3e1..b172237700 100644 --- a/web/oss/src/lib/helpers/evaluate.ts +++ b/web/oss/src/lib/helpers/evaluate.ts @@ -1,341 +1,3 @@ -import {EvaluationType} from "@agenta/oss/src/lib/enums" -import {convertToCsv, downloadCsv} from "@agenta/oss/src/lib/helpers/fileManipulations" -import {formatCurrency, formatLatency} from "@agenta/oss/src/lib/helpers/formatters" -import {isDemo} from "@agenta/oss/src/lib/helpers/utils" -import { - Evaluation, - GenericObject, - TypedValue, - Variant, - _Evaluation, - EvaluationScenario, -} from "@agenta/oss/src/lib/Types" -import dayjs from "dayjs" -import capitalize from "lodash/capitalize" -import round from "lodash/round" - -import AlertPopup from "@/oss/components/AlertPopup/AlertPopup" -import {runningStatuses} from "@/oss/components/pages/evaluations/cellRenderers/cellRenderers" -import { - HumanEvaluationListTableDataType, - SingleModelEvaluationListTableDataType, -} from "@/oss/lib/Types" -import {fetchEvaluatonIdsByResource} from "@/oss/services/evaluations/api" - -export const exportABTestingEvaluationData = ( - evaluation: Evaluation, - scenarios: EvaluationScenario[], - rows: GenericObject[], -) => { - const exportRow = rows.map((data, ix) => { - const inputColumns = evaluation.testset.testsetChatColumn - ? {Input: evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn]} - : data.inputs.reduce( - (columns: any, input: {input_name: string; input_value: string}) => { - columns[`${input.input_name}`] = input.input_value - return columns - }, - {}, - ) - return { - ...inputColumns, - [`App Variant ${evaluation.variants[0].variantName} Output 0`]: data?.columnData0 - ? data?.columnData0 - : data.outputs[0]?.variant_output, - [`App Variant ${evaluation.variants[1].variantName} Output 1`]: data?.columnData1 - ? data?.columnData1 - : data.outputs[1]?.variant_output, - ["Vote"]: - evaluation.variants.find((v: Variant) => v.variantId === data.vote)?.variantName || - data.vote, - ["Expected Output"]: - scenarios[ix]?.correctAnswer || evaluation.testset.csvdata[ix].correct_answer, - ["Additional notes"]: scenarios[ix]?.note, - } - }) - const exportCol = Object.keys(exportRow[0]) - - const csvData = convertToCsv(exportRow, exportCol) - const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.variants[1].variantName}_${evaluation.evaluationType}.csv` - downloadCsv(csvData, filename) -} - -export const exportSingleModelEvaluationData = ( - evaluation: Evaluation, - scenarios: EvaluationScenario[], - rows: GenericObject[], -) => { - const exportRow = rows.map((data, ix) => { - const inputColumns = evaluation.testset.testsetChatColumn - ? {Input: evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn]} - : data.inputs.reduce( - (columns: any, input: {input_name: string; input_value: string}) => { - columns[`${input.input_name}`] = input.input_value - return columns - }, - {}, - ) - const numericScore = parseInt(data.score) - return { - ...inputColumns, - [`App Variant ${evaluation.variants[0].variantName} Output 0`]: data?.columnData0 - ? data?.columnData0 - : data.outputs[0]?.variant_output, - ["Score"]: isNaN(numericScore) ? "-" : numericScore, - ["Expected Output"]: - scenarios[ix]?.correctAnswer || evaluation.testset.csvdata[ix].correct_answer, - ["Additional notes"]: scenarios[ix]?.note, - } - }) - const exportCol = Object.keys(exportRow[0]) - - const csvData = convertToCsv(exportRow, exportCol) - const filename = `${evaluation.appName}_${evaluation.variants[0].variantName}_${evaluation.evaluationType}.csv` - downloadCsv(csvData, filename) -} - -export const calculateResultsDataAvg = (resultsData: Record, multiplier = 10) => { - const obj = {...resultsData} - Object.keys(obj).forEach((key) => { - if (isNaN(+key)) delete obj[key] - }) - - const count = Object.values(obj).reduce((acc, value) => acc + +value, 0) - const sum = Object.keys(obj).reduce((acc, key) => acc + (parseFloat(key) || 0) * +obj[key], 0) - return (sum / count) * multiplier -} - -export const getVotesPercentage = (record: HumanEvaluationListTableDataType, index: number) => { - const variant = record.votesData.variants[index] - return record.votesData.variants_votes_data[variant]?.percentage -} - -export const checkIfResourceValidForDeletion = async ( - data: Omit[0], "appId">, -) => { - if (isDemo()) { - const response = await fetchEvaluatonIdsByResource(data) - if (response.data.length > 0) { - const name = - (data.resourceType === "testset" - ? "Testset" - : data.resourceType === "evaluator_config" - ? "Evaluator" - : "Variant") + (data.resourceIds.length > 1 ? "s" : "") - - const suffix = response.data.length > 1 ? "s" : "" - AlertPopup({ - title: `${name} is in use`, - message: `The ${name} is currently in used by ${response.data.length} evaluation${suffix}. Please delete the evaluation${suffix} first.`, - cancelText: null, - okText: "Ok", - }) - return false - } - } +export const checkIfResourceValidForDeletion = async (data: any) => { return true } - -export function getTypedValue(res?: TypedValue) { - const {value, type, error} = res || {} - if (type === "error") { - return error?.message - } - - if (value === undefined) return "-" - - switch (type) { - case "number": - return round(Number(value), 2) - case "boolean": - case "bool": - return capitalize(value?.toString()) - case "cost": - return formatCurrency(Number(value)) - case "latency": - return formatLatency(Number(value)) - case "string": - case "text": - return value?.toString() ?? "-" - case "code": - case "regex": - return value?.toString() ?? "-" - case "object": - return typeof value === "object" - ? JSON.stringify(value, null, 2) - : (value?.toString() ?? "-") - case "messages": - return Array.isArray(value) - ? value - .map((msg) => (typeof msg === "string" ? msg : JSON.stringify(msg))) - .join("\n") - : (value?.toString() ?? "-") - case "multiple_choice": - return Array.isArray(value) ? value.join(", ") : (value?.toString() ?? "-") - case "hidden": - return "-" - default: - return value?.toString() ?? "-" - } -} - -type CellDataType = "number" | "text" | "date" -export function getFilterParams(type: CellDataType) { - const filterParams: GenericObject = {} - if (type == "date") { - filterParams.comparator = function ( - filterLocalDateAtMidnight: Date, - cellValue: string | null, - ) { - if (cellValue == null) return -1 - const cellDate = dayjs(cellValue).startOf("day").toDate() - if (filterLocalDateAtMidnight.getTime() === cellDate.getTime()) { - return 0 - } - if (cellDate < filterLocalDateAtMidnight) { - return -1 - } - if (cellDate > filterLocalDateAtMidnight) { - return 1 - } - } - } - - return { - sortable: true, - floatingFilter: true, - filter: - type === "number" - ? "agNumberColumnFilter" - : type === "date" - ? "agDateColumnFilter" - : "agTextColumnFilter", - cellDataType: type === "number" ? "text" : type, - filterParams, - comparator: getCustomComparator(type), - } -} - -export const calcEvalDuration = (evaluation: _Evaluation) => { - return dayjs( - runningStatuses.includes(evaluation.status.value) ? Date.now() : evaluation.updated_at, - ).diff(dayjs(evaluation.created_at), "milliseconds") -} - -const getCustomComparator = (type: CellDataType) => (valueA: string, valueB: string) => { - const getNumber = (val: string) => { - const num = parseFloat(val || "0") - return isNaN(num) ? 0 : num - } - - valueA = String(valueA) - valueB = String(valueB) - - switch (type) { - case "date": - return dayjs(valueA).diff(dayjs(valueB)) - case "text": - return valueA.localeCompare(valueB) - case "number": - return getNumber(valueA) - getNumber(valueB) - default: - return 0 - } -} - -export const removeCorrectAnswerPrefix = (str: string) => { - return str.replace(/^correctAnswer_/, "") -} - -export const mapTestcaseAndEvalValues = ( - settingsValues: Record, - selectedTestcase: Record, -) => { - const testcaseObj: Record = {} - const evalMapObj: Record = {} - - Object.entries(settingsValues).forEach(([key, value]) => { - if (typeof value === "string" && value.startsWith("testcase.")) { - testcaseObj[key] = selectedTestcase[value.split(".")[1]] - } else { - evalMapObj[key] = value - } - }) - - return {testcaseObj, evalMapObj} -} - -export const transformTraceKeysInSettings = ( - settingsValues: Record, -): Record => { - return Object.keys(settingsValues).reduce( - (acc, curr) => { - if ( - !acc[curr] && - typeof settingsValues[curr] === "string" && - settingsValues[curr].startsWith("trace.") - ) { - acc[curr] = settingsValues[curr].replace("trace.", "") - } else { - acc[curr] = settingsValues[curr] - } - - return acc - }, - {} as Record, - ) -} - -export const getEvaluatorTags = () => { - const evaluatorTags = [ - { - label: "Classifiers", - value: "classifiers", - }, - { - label: "Similarity", - value: "similarity", - }, - { - label: "AI / LLM", - value: "ai_llm", - }, - { - label: "Functional", - value: "functional", - }, - ] - - if (isDemo()) { - evaluatorTags.unshift({ - label: "RAG", - value: "rag", - }) - } - - return evaluatorTags -} - -export const calculateAvgScore = (evaluation: SingleModelEvaluationListTableDataType) => { - let score = 0 - if (evaluation.scoresData) { - score = - ((evaluation.scoresData.correct?.length || evaluation.scoresData.true?.length || 0) / - evaluation.scoresData.nb_of_rows) * - 100 - } else if (evaluation.resultsData) { - const multiplier = { - [EvaluationType.auto_webhook_test]: 100, - [EvaluationType.single_model_test]: 1, - } - score = calculateResultsDataAvg( - evaluation.resultsData, - multiplier[evaluation.evaluationType as keyof typeof multiplier], - ) - score = isNaN(score) ? 0 : score - } else if (evaluation.avgScore) { - score = evaluation.avgScore * 100 - } - - return score -} diff --git a/web/oss/src/lib/hooks/useBreadcrumbs.ts b/web/oss/src/lib/hooks/useBreadcrumbs.ts index ace4c83327..6d0228f5c4 100644 --- a/web/oss/src/lib/hooks/useBreadcrumbs.ts +++ b/web/oss/src/lib/hooks/useBreadcrumbs.ts @@ -6,6 +6,7 @@ import { appendBreadcrumbAtom, clearBreadcrumbsAtom, prependBreadcrumbAtom, + removeBreadcrumbsAtom, setBreadcrumbsAtom, type BreadcrumbAtom, } from "@/oss/lib/atoms/breadcrumb" @@ -15,12 +16,14 @@ export const useBreadcrumbs = () => { const appendBreadcrumb = useSetAtom(appendBreadcrumbAtom) const prependBreadcrumb = useSetAtom(prependBreadcrumbAtom) const clearBreadcrumbs = useSetAtom(clearBreadcrumbsAtom) + const removeBreadcrumbs = useSetAtom(removeBreadcrumbsAtom) return { setBreadcrumbs, appendBreadcrumb, prependBreadcrumb, clearBreadcrumbs, + removeBreadcrumbs, } } @@ -43,24 +46,37 @@ export const useBreadcrumbsEffect = ( }: {breadcrumbs: BreadcrumbAtom; type?: "prepend" | "append" | "new"; condition?: boolean}, deps: React.DependencyList = [], ) => { - const {setBreadcrumbs, clearBreadcrumbs, appendBreadcrumb, prependBreadcrumb} = useBreadcrumbs() + const { + setBreadcrumbs, + clearBreadcrumbs, + appendBreadcrumb, + prependBreadcrumb, + removeBreadcrumbs, + } = useBreadcrumbs() useEffect(() => { - if (condition) { - if (type === "prepend") { - prependBreadcrumb(breadcrumbs) - } else if (type === "append") { - appendBreadcrumb(breadcrumbs) - } else { - setBreadcrumbs(breadcrumbs) + if (!condition) return + + const keys = Object.keys(breadcrumbs) + if (!keys.length) return + + if (type === "prepend") { + prependBreadcrumb(breadcrumbs) + return () => { + removeBreadcrumbs(keys) } } - // Cleanup: reset to URL-based breadcrumbs when component unmounts - return () => { - if (type === "new") { - clearBreadcrumbs() + if (type === "append") { + appendBreadcrumb(breadcrumbs) + return () => { + removeBreadcrumbs(keys) } } + + setBreadcrumbs(breadcrumbs) + return () => { + clearBreadcrumbs() + } }, deps) } diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/human_a_b_testing/[evaluation_id]/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/human_a_b_testing/[evaluation_id]/index.tsx deleted file mode 100644 index 76e6526898..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/human_a_b_testing/[evaluation_id]/index.tsx +++ /dev/null @@ -1,115 +0,0 @@ -import {useEffect, useState} from "react" - -import {useAtom, useAtomValue} from "jotai" -import dynamic from "next/dynamic" -import {useRouter} from "next/router" - -// Avoid SSR for this heavy component to prevent server-side ReferenceErrors from client-only libs -const ABTestingEvaluationTable = dynamic( - () => import("@/oss/components/EvaluationTable/ABTestingEvaluationTable"), - {ssr: false}, -) -import useURL from "@/oss/hooks/useURL" -import {evaluationAtom, evaluationScenariosAtom} from "@/oss/lib/atoms/evaluation" -import {getTestsetChatColumn} from "@/oss/lib/helpers/testset" -import {useBreadcrumbsEffect} from "@/oss/lib/hooks/useBreadcrumbs" -import type {Evaluation} from "@/oss/lib/Types" -import { - fetchLoadEvaluation, - fetchAllLoadEvaluationsScenarios, -} from "@/oss/services/human-evaluations/api" -import {fetchTestset} from "@/oss/services/testsets/api" -import {projectIdAtom} from "@/oss/state/project" -import {variantsAtom} from "@/oss/state/variant/atoms/fetcher" - -export default function Evaluation() { - const router = useRouter() - const projectId = useAtomValue(projectIdAtom) - const evaluationTableId = router.query.evaluation_id - ? router.query.evaluation_id.toString() - : "" - const [evaluationScenarios, setEvaluationScenarios] = useAtom(evaluationScenariosAtom) - const [evaluation, setEvaluation] = useAtom(evaluationAtom) - const [isLoading, setIsLoading] = useState(true) - const appId = router.query.app_id as string - const columnsCount = 2 - const {baseAppURL} = useURL() - // variants from global store - const variantsStore = useAtomValue(variantsAtom) - - useEffect(() => { - if (!evaluation || !projectId) { - return - } - const init = async () => { - setIsLoading(true) - try { - const data = await fetchAllLoadEvaluationsScenarios(evaluationTableId, evaluation) - setEvaluationScenarios(data) - } finally { - setTimeout(() => setIsLoading(false), 1000) - } - } - init() - }, [evaluation, projectId]) - - useEffect(() => { - if (!evaluationTableId) { - return - } - const init = async () => { - const evaluation: Evaluation = await fetchLoadEvaluation(evaluationTableId) - const backendVariants = variantsStore - const testset = await fetchTestset(evaluation.testset._id) - // Create a map for faster access to first array elements - const backendVariantsMap = new Map() - backendVariants.forEach((obj) => backendVariantsMap.set(obj.variantId, obj)) - - // Update variants in second object - evaluation.variants = evaluation.variants.map((variant) => { - const backendVariant = backendVariantsMap.get(variant.variantId) - return backendVariant ? backendVariant : variant - }) - evaluation.testset = { - ...evaluation.testset, - ...testset, - testsetChatColumn: getTestsetChatColumn(testset.csvdata), - } - setEvaluation(evaluation) - } - - init() - }, [evaluationTableId]) - - // breadcrumbs - useBreadcrumbsEffect( - { - breadcrumbs: { - appPage: { - label: "human ab testing", - href: `${baseAppURL}/${appId}/evaluations?selectedEvaluation=human_ab_testing`, - }, - "eval-detail": { - label: evaluationTableId, - value: evaluationTableId, - }, - }, - type: "append", - condition: !!evaluationTableId, - }, - [evaluationTableId], - ) - - return ( -

- {evaluationTableId && evaluationScenarios && evaluation && ( - - )} -
- ) -} diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/index.tsx deleted file mode 100644 index 5f9c0ce406..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import EvaluationsView from "@/oss/components/pages/evaluations/EvaluationsView" - -const AppEvaluationsPage = () => { - return -} - -export default AppEvaluationsPage diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/[evaluation_id]/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/[evaluation_id]/index.tsx deleted file mode 100644 index 8a3e7e4523..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/[evaluation_id]/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import {useRouter} from "next/router" - -import EvalRunDetailsPage from "@/oss/components/EvalRunDetails" - -const AppEvaluationResultsPage = () => { - const router = useRouter() - const rawType = - (Array.isArray(router.query.eval_type) - ? router.query.eval_type[0] - : router.query.eval_type) || - (Array.isArray(router.query.type) ? router.query.type[0] : router.query.type) - const normalized = - rawType === "online" - ? "online" - : rawType === "human" - ? "human" - : rawType === "custom" - ? "custom" - : "auto" - return -} - -export default AppEvaluationResultsPage diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/compare/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/compare/index.tsx deleted file mode 100644 index 9a24e505d7..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/results/compare/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import EvaluationCompare from "@/oss/components/pages/evaluations/evaluationCompare/EvaluationCompare" - -const EvaluationCompareDetails = () => { - return -} - -export default EvaluationCompareDetails diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/single_model_test/[evaluation_id]/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/single_model_test/[evaluation_id]/index.tsx deleted file mode 100644 index 209e1772ec..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/evaluations/single_model_test/[evaluation_id]/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import EvalRunDetailsPage from "@/oss/components/EvalRunDetails" - -const EvaluationPage = () => { - return -} - -export default EvaluationPage diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/overview/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/overview/index.tsx index 9afd259da5..baa1f07530 100644 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/overview/index.tsx +++ b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/apps/[app_id]/overview/index.tsx @@ -15,6 +15,7 @@ import {openEditAppModalAtom} from "@/oss/components/pages/app-management/modals import DeploymentOverview from "@/oss/components/pages/overview/deployments/DeploymentOverview" import VariantsOverview from "@/oss/components/pages/overview/variants/VariantsOverview" import useURL from "@/oss/hooks/useURL" +import {isDemo} from "@/oss/lib/helpers/utils" import type {JSSTheme} from "@/oss/lib/Types" import {deleteApp} from "@/oss/services/app-selector/api" import {useEnvironments} from "@/oss/services/deployment/hooks/useEnvironments" @@ -125,10 +126,13 @@ const OverviewPage = () => { - - - - + {isDemo() && ( + <> + + + + + )} { - return -} - -export default ProjectEvaluationsPage diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/[evaluation_id]/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/[evaluation_id]/index.tsx deleted file mode 100644 index 8cabe50e3a..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/[evaluation_id]/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import {useRouter} from "next/router" - -import EvalRunDetailsPage from "@/oss/components/EvalRunDetails" - -const ProjectEvaluationResultsPage = () => { - const router = useRouter() - const rawType = - (Array.isArray(router.query.eval_type) - ? router.query.eval_type[0] - : router.query.eval_type) || - (Array.isArray(router.query.type) ? router.query.type[0] : router.query.type) - const normalized = - rawType === "online" - ? "online" - : rawType === "human" - ? "human" - : rawType === "custom" - ? "custom" - : "auto" - return -} - -export default ProjectEvaluationResultsPage diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/compare/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/compare/index.tsx deleted file mode 100644 index 4fc96755ce..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/results/compare/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import EvaluationCompare from "@/oss/components/pages/evaluations/evaluationCompare/EvaluationCompare" - -const ProjectEvaluationCompareDetails = () => { - return -} - -export default ProjectEvaluationCompareDetails diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/single_model_test/[evaluation_id]/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/single_model_test/[evaluation_id]/index.tsx deleted file mode 100644 index 67c0827984..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluations/single_model_test/[evaluation_id]/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import EvalRunDetailsPage from "@/oss/components/EvalRunDetails" - -const ProjectHumanEvaluationPage = () => { - return -} - -export default ProjectHumanEvaluationPage diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx deleted file mode 100644 index df1b8461be..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/configure/[evaluator_id].tsx +++ /dev/null @@ -1,20 +0,0 @@ -import {useMemo} from "react" - -import {useRouter} from "next/router" - -import ConfigureEvaluatorPage from "@/oss/components/Evaluators/components/ConfigureEvaluator" - -const EvaluatorConfigureRoute = () => { - const router = useRouter() - const evaluatorId = useMemo(() => { - const id = router.query.evaluator_id - if (Array.isArray(id)) { - return id[0] - } - return id ?? null - }, [router.query.evaluator_id]) - - return -} - -export default EvaluatorConfigureRoute diff --git a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx b/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx deleted file mode 100644 index 7996228a65..0000000000 --- a/web/oss/src/pages/w/[workspace_id]/p/[project_id]/evaluators/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import EvaluatorsRegistry from "@/oss/components/Evaluators" - -const ProjectEvaluatorsPage = () => { - return -} - -export default ProjectEvaluatorsPage diff --git a/web/oss/src/services/onlineEvaluations/api.ts b/web/oss/src/services/onlineEvaluations/api.ts index e0650b45d9..3ffa241891 100644 --- a/web/oss/src/services/onlineEvaluations/api.ts +++ b/web/oss/src/services/onlineEvaluations/api.ts @@ -2,179 +2,11 @@ import axios from "@/oss/lib/api/assets/axiosConfig" import {getAgentaApiUrl} from "@/oss/lib/helpers/api" import {getProjectValues} from "@/oss/state/project" -type LogicalOperator = "and" | "or" | "not" | "nand" | "nor" - -export interface QueryConditionPayload { - field: string - key?: string - value?: unknown - operator?: string - options?: Record -} - -export interface QueryFilteringPayload { - operator?: LogicalOperator - conditions: (QueryConditionPayload | QueryFilteringPayload)[] -} - -export interface QueryWindowingPayload { - newest?: string - oldest?: string - next?: string - limit?: number - order?: "ascending" | "descending" - interval?: number - rate?: number -} - -export interface QueryRevisionDataPayload { - filtering?: QueryFilteringPayload - windowing?: QueryWindowingPayload -} - -export interface SimpleQueryCreatePayload { - slug: string - name?: string - description?: string - flags?: Record - tags?: Record - meta?: Record - data?: QueryRevisionDataPayload -} - -export interface SimpleQueryCreateRequest { - query: SimpleQueryCreatePayload -} - -export interface SimpleQueryResponse { - count: number - query?: { - id: string - slug?: string - data?: QueryRevisionDataPayload - meta?: Record - } | null -} - -export interface QueryRevisionRetrieveRequest { - query_ref?: {id?: string; slug?: string} | null - query_variant_ref?: {id?: string; slug?: string} | null - query_revision_ref?: {id?: string; slug?: string} | null -} - -export interface QueryRevisionResponse { - count: number - query_revision?: { - id?: string - slug?: string - variant_id?: string - version?: string | number - data?: QueryRevisionDataPayload - } | null -} - -export interface SimpleEvaluationFlagsPayload { - is_live?: boolean - is_closed?: boolean - is_active?: boolean -} - -export interface SimpleEvaluationDataPayload { - status?: string - query_steps?: string[] | Record - testset_steps?: Record - application_steps?: Record - evaluator_steps?: string[] | Record - repeats?: number - // Structured references for online evaluations - query_ref?: {id?: string; slug?: string} | null - query_revision_ref?: {id?: string; slug?: string} | null - evaluator_ref?: {id?: string; slug?: string} | null - configuration?: Record -} - -export interface SimpleEvaluationCreatePayload { - name?: string - description?: string - flags?: SimpleEvaluationFlagsPayload - tags?: Record - meta?: Record - data: SimpleEvaluationDataPayload -} - -export interface SimpleEvaluationCreateRequest { - evaluation: SimpleEvaluationCreatePayload -} - -export interface SimpleEvaluationResponse { - count: number - evaluation?: SimpleEvaluationPayload | null -} - -export interface SimpleEvaluationPayload { - id?: string - slug?: string - name?: string - description?: string - created_at?: string - updated_at?: string - created_by_id?: string - updated_by_id?: string - flags?: SimpleEvaluationFlagsPayload - data?: SimpleEvaluationDataPayload - meta?: Record - tags?: Record -} - -export interface SimpleEvaluationsResponse { - count: number - evaluations: SimpleEvaluationPayload[] -} - -export interface SimpleEvaluationsQueryRequest { - evaluation?: { - flags?: SimpleEvaluationFlagsPayload - ids?: string[] - } - tags?: Record - meta?: Record -} - const getProjectUrl = (path: string) => { const {projectId} = getProjectValues() return `${getAgentaApiUrl()}${path}?project_id=${projectId}` } -export const createSimpleQuery = async ( - payload: SimpleQueryCreateRequest, -): Promise => { - const {data} = await axios.post(getProjectUrl("/preview/simple/queries/"), payload) - return data as SimpleQueryResponse -} - -export const retrieveQueryRevision = async ( - payload: QueryRevisionRetrieveRequest, -): Promise => { - const {data} = await axios.post(getProjectUrl("/preview/queries/revisions/retrieve"), payload) - return data as QueryRevisionResponse -} - -export const createSimpleEvaluation = async ( - payload: SimpleEvaluationCreateRequest, -): Promise => { - const {data} = await axios.post(getProjectUrl("/preview/simple/evaluations/"), payload) - return data as SimpleEvaluationResponse -} - -export const querySimpleEvaluations = async ( - payload?: SimpleEvaluationsQueryRequest, -): Promise => { - const url = getProjectUrl("/preview/simple/evaluations/query") - const body = payload ?? {} - const {data} = await axios.post(url, body) - return data as SimpleEvaluationsResponse -} - export const stopSimpleEvaluation = async (evaluationId: string) => { const url = getProjectUrl(`/preview/simple/evaluations/${evaluationId}/stop`) const {data} = await axios.post(url) diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index 0057554c07..dee9870f08 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -84,7 +84,7 @@ importers: version: 0.1.13(prettier@3.6.0) ts-node: specifier: ^10.9.2 - version: 10.9.2(@swc/core@1.11.8(@swc/helpers@0.5.17))(@types/node@20.19.19)(typescript@5.8.3) + version: 10.9.2(@swc/core@1.11.8(@swc/helpers@0.5.17))(@types/node@20.19.13)(typescript@5.8.3) tsconfig-paths: specifier: ^4.2.0 version: 4.2.0 @@ -267,7 +267,7 @@ importers: version: 21.1.0 swc-loader: specifier: ^0.2.6 - version: 0.2.6(@swc/core@1.11.8(@swc/helpers@0.5.17))(webpack@5.98.0(@swc/core@1.11.8(@swc/helpers@0.5.17))) + version: 0.2.6(@swc/core@1.11.8(@swc/helpers@0.5.17))(webpack@5.98.0(@swc/core@1.11.8(@swc/helpers@0.5.17))(esbuild@0.25.10)) swr: specifier: ^2.3.0 version: 2.3.3(react@19.0.0) @@ -593,9 +593,6 @@ importers: react-window: specifier: ^1.8.11 version: 1.8.11(react-dom@19.0.0(react@19.0.0))(react@19.0.0) - recharts: - specifier: ^3.1.0 - version: 3.1.0(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react-is@18.3.1)(react@19.0.0)(redux@5.0.1) semver: specifier: ^7.7.2 version: 7.7.2 @@ -613,7 +610,7 @@ importers: version: 21.1.0 swc-loader: specifier: ^0.2.6 - version: 0.2.6(@swc/core@1.11.8(@swc/helpers@0.5.17))(webpack@5.98.0(@swc/core@1.11.8(@swc/helpers@0.5.17))(esbuild@0.25.10)) + version: 0.2.6(@swc/core@1.11.8(@swc/helpers@0.5.17))(webpack@5.98.0(@swc/core@1.11.8(@swc/helpers@0.5.17))) swr: specifier: ^2.3.0 version: 2.3.3(react@19.0.0) @@ -2054,6 +2051,9 @@ packages: '@types/node@20.19.11': resolution: {integrity: sha512-uug3FEEGv0r+jrecvUUpbY8lLisvIjg6AAic6a2bSP5OEOLeJsDSnvhCDov7ipFFMXS3orMpzlmi0ZcuGkBbow==} + '@types/node@20.19.13': + resolution: {integrity: sha512-yCAeZl7a0DxgNVteXFHt9+uyFbqXGy/ShC4BlcHkoE0AfGXYv/BUiplV72DjMYXHDBXFjhvr6DD1NiRVfB4j8g==} + '@types/node@20.19.19': resolution: {integrity: sha512-pb1Uqj5WJP7wrcbLU7Ru4QtA0+3kAXrkutGiD26wUKzSMgNNaPARTUDQmElUXp64kh3cWdou3Q0C7qwwxqSFmg==} @@ -3974,6 +3974,7 @@ packages: resolution: {integrity: sha512-Quz3MvAwHxVYNXsOByL7xI5EB2WYOeFswqaHIA3qOK3isRWTxiplBEocmmru6XmxDB2L7jDNYtYA4FyimoAFEw==} engines: {node: '>=8.17.0'} hasBin: true + bundledDependencies: [] jsonpointer@5.0.1: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} @@ -7283,6 +7284,10 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/node@20.19.13': + dependencies: + undici-types: 6.21.0 + '@types/node@20.19.19': dependencies: undici-types: 6.21.0 @@ -11368,14 +11373,14 @@ snapshots: '@swc/core': 1.11.8(@swc/helpers@0.5.17) optional: true - ts-node@10.9.2(@swc/core@1.11.8(@swc/helpers@0.5.17))(@types/node@20.19.19)(typescript@5.8.3): + ts-node@10.9.2(@swc/core@1.11.8(@swc/helpers@0.5.17))(@types/node@20.19.13)(typescript@5.8.3): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.19.19 + '@types/node': 20.19.13 acorn: 8.15.0 acorn-walk: 8.3.4 arg: 4.1.3