From 5ecb68e1d4858d308657155a1511d48506f4d1f0 Mon Sep 17 00:00:00 2001 From: Alexander Mahler Date: Mon, 21 Jun 2021 20:38:49 +0200 Subject: [PATCH 01/38] See next commit for commit information. Commited due to computer switch --- adapter/build.gradle | 5 ++- .../datasource/DatasourceManager.java | 43 ++++++++++++++++-- .../api/rest/v1/DatasourceEndpoint.java | 1 + .../datasource/model/DataImport.java | 16 ++++++- .../datasource/model/DatasourceMetadata.java | 2 +- scheduler/src/api/http/adapter-client.ts | 9 ++++ scheduler/src/initializer.test.ts | 3 +- scheduler/src/scheduling.test.ts | 3 +- start.sh | 3 ++ ui/src/datasource/DatasourceCreate.vue | 4 +- ui/src/datasource/DatasourceEdit.vue | 2 - ui/src/datasource/DatasourceOverview.vue | 44 +++++++++++++++++++ ui/src/datasource/datasource.ts | 7 +++ ui/src/datasource/datasourceRest.ts | 7 ++- ui/src/datasource/schemaSuggestionRest.ts | 1 + 15 files changed, 135 insertions(+), 15 deletions(-) create mode 100644 start.sh diff --git a/adapter/build.gradle b/adapter/build.gradle index fe46ff70f..d3f4b0a37 100644 --- a/adapter/build.gradle +++ b/adapter/build.gradle @@ -25,6 +25,7 @@ repositories { mavenCentral() maven { url 'https://repo.spring.io/snapshot' } maven { url 'https://repo.spring.io/milestone' } + maven { url 'https://jitpack.io' } } test { @@ -51,10 +52,12 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv' implementation 'org.postgresql:postgresql' - + implementation 'org.json:json:20210307' + implementation 'com.github.everit-org.json-schema:org.everit.json.schema:1.12.3' implementation 'com.vladmihalcea:hibernate-types-52:2.10.2' testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' } + } diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java index 70b730fc6..507bc9a26 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java @@ -14,11 +14,19 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.everit.json.schema.Schema; +import org.everit.json.schema.loader.SchemaLoader; +import org.everit.json.schema.ValidationException; +import org.json.JSONObject; +import org.json.JSONTokener; + import java.io.IOException; import java.util.Collection; import java.util.Date; import java.util.stream.StreamSupport; +import javax.sql.DataSource; + @Slf4j @Service @AllArgsConstructor @@ -71,6 +79,16 @@ public void deleteAllDatasources() { StreamSupport.stream(allDatasourceConfigs.spliterator(), true).forEach(amqpPublisher::publishDeletion); } + public void validate(Long id) throws DatasourceNotFoundException, AdapterException, IOException, ValidationException { + Datasource datasource = getDatasource(id); + DataImport dataImport = getLatestDataImportForDatasource(id); + JSONObject rawSchema = (JSONObject) datasource.getSchema(); + Schema schema = SchemaLoader.load(rawSchema); + + JSONObject validationData = new JSONObject(dataImport.getData()); + schema.validate(validationData); + } + public DataImport.MetaData trigger(Long id, RuntimeParameters runtimeParameters) throws DatasourceNotFoundException, AdapterException, IOException { try { return executeImport(id, runtimeParameters); @@ -101,13 +119,30 @@ DataImport.MetaData executeImport(Long id, RuntimeParameters runtimeParameters) throws DatasourceNotFoundException, ImporterParameterException, InterpreterParameterException, IOException { Datasource datasource = getDatasource(id); AdapterConfig adapterConfig = datasource.toAdapterConfig(runtimeParameters); - DataImportResponse executionResult = adapter.executeJob(adapterConfig); - DataImport dataImport = new DataImport(datasource, executionResult.getData()); + try { + DataImportResponse executionResult = adapter.executeJob(adapterConfig); - DataImport savedDataImport = dataImportRepository.save(dataImport); + if (datasource.getSchema() != null) { + validate(id); + } + + DataImport dataImport = new DataImport(datasource, executionResult.getData()); + + DataImport savedDataImport = dataImportRepository.save(dataImport); - amqpPublisher.publishImportSuccess(id, savedDataImport.getData()); + amqpPublisher.publishImportSuccess(id, savedDataImport.getData()); + return savedDataImport.getMetaData(); + } catch (ValidationException e) { + return handleImportException(id, datasource, "WARNING", e); + } catch (Exception e) { + return handleImportException(id, datasource, "FAILED", e); + } + } + DataImport.MetaData handleImportException(Long id, Datasource datasource, String healthStatus, Exception e) { + DataImport dataImport = new DataImport(datasource, null, healthStatus); + DataImport savedDataImport = dataImportRepository.save(dataImport); + publishImportFailure(id, e); return savedDataImport.getMetaData(); } diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/api/rest/v1/DatasourceEndpoint.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/api/rest/v1/DatasourceEndpoint.java index 5b55783e1..dcacdbee3 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/api/rest/v1/DatasourceEndpoint.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/api/rest/v1/DatasourceEndpoint.java @@ -2,6 +2,7 @@ import lombok.AllArgsConstructor; +import org.jvalue.ods.adapterservice.adapter.interpreter.JsonInterpreter; import org.jvalue.ods.adapterservice.adapter.model.exceptions.AdapterException; import org.jvalue.ods.adapterservice.datasource.DatasourceManager; import org.jvalue.ods.adapterservice.datasource.model.*; diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java index d3cb8d61a..0307edaa0 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java @@ -23,6 +23,8 @@ public class DataImport { private Date timestamp; + private String health; + @ManyToOne(fetch = FetchType.EAGER) @JoinColumn(name="datasource_id") @JsonIgnore @@ -31,6 +33,14 @@ public class DataImport { public DataImport(Datasource datasource, String data) { this.datasource = datasource; this.data = data.getBytes(StandardCharsets.UTF_8); + this.health = "OK"; + this.timestamp = new Date(); + } + + public DataImport(Datasource datasource, String data, String health) { + this.datasource = datasource; + this.data = data.getBytes(StandardCharsets.UTF_8); + this.health = health; this.timestamp = new Date(); } @@ -40,7 +50,7 @@ public String getData() { @JsonIgnore public MetaData getMetaData() { - return new MetaData(this.id, this.timestamp, this.datasource); + return new MetaData(this.id, this.timestamp, this.health, this.datasource); } // json representation without the actual data (for import list) @@ -51,6 +61,10 @@ public static class MetaData { @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", locale = "UTC") private final Date timestamp; + + @JsonIgnore + private final String health; + @JsonIgnore private final Datasource datasource; diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceMetadata.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceMetadata.java index 80524e596..5f571f49f 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceMetadata.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceMetadata.java @@ -1,4 +1,4 @@ -package org.jvalue.ods.adapterservice.datasource.model; + package org.jvalue.ods.adapterservice.datasource.model; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonFormat; diff --git a/scheduler/src/api/http/adapter-client.ts b/scheduler/src/api/http/adapter-client.ts index 95a1d3b1a..6074f7451 100644 --- a/scheduler/src/api/http/adapter-client.ts +++ b/scheduler/src/api/http/adapter-client.ts @@ -14,6 +14,15 @@ export async function getAllDatasources (): Promise { return response.data } +export async function getDatasourceById (datasourceId: number): Promise { + const response = await http.get(`/datasources/${datasourceId}`) + return response.data +} + +export async function updateDatasource (datasourceId: number, datasource: DatasourceConfig): Promise { + return await http.put(`/datasources/${datasourceId}`, datasource) +} + export async function triggerDatasource (datasourceId: number): Promise { return await http.post(`/datasources/${datasourceId}/trigger`) } diff --git a/scheduler/src/initializer.test.ts b/scheduler/src/initializer.test.ts index da06f281c..1af645641 100644 --- a/scheduler/src/initializer.test.ts +++ b/scheduler/src/initializer.test.ts @@ -32,7 +32,8 @@ describe('Scheduler initializer', () => { periodic: false, firstExecution: new Date(), interval: 60000 - } + }, + health: '' } mockedGetAllDatasources.mockResolvedValue([config]) diff --git a/scheduler/src/scheduling.test.ts b/scheduler/src/scheduling.test.ts index bc5b4cc44..0c06e6895 100644 --- a/scheduler/src/scheduling.test.ts +++ b/scheduler/src/scheduling.test.ts @@ -214,6 +214,7 @@ function generateConfig (periodic: boolean, firstExecution: Date, interval: numb periodic: periodic, firstExecution: firstExecution, interval: interval - } + }, + health: 'WANRNING' } } diff --git a/start.sh b/start.sh new file mode 100644 index 000000000..f4963a437 --- /dev/null +++ b/start.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +docker-compose -f ./docker-compose.yml -f ./../schema-recommendation/docker-compose.yml up \ No newline at end of file diff --git a/ui/src/datasource/DatasourceCreate.vue b/ui/src/datasource/DatasourceCreate.vue index c26c17825..d79156657 100644 --- a/ui/src/datasource/DatasourceCreate.vue +++ b/ui/src/datasource/DatasourceCreate.vue @@ -67,9 +67,7 @@ export default class DatasourceCreate extends Vue { type: 'JSON', parameters: {} }, - schema: { - a: 1 - }, + schema: {}, metadata: { author: '', license: '', diff --git a/ui/src/datasource/DatasourceEdit.vue b/ui/src/datasource/DatasourceEdit.vue index 0c17ec7ae..e3e4bd0db 100644 --- a/ui/src/datasource/DatasourceEdit.vue +++ b/ui/src/datasource/DatasourceEdit.vue @@ -68,8 +68,6 @@ export default class DatasourceEdit extends Vue { if (this.datasource === null) { return } - - await DatasourceREST.updateDatasource(this.datasource) this.routeToOverview() } diff --git a/ui/src/datasource/DatasourceOverview.vue b/ui/src/datasource/DatasourceOverview.vue index cb2543e07..ceefa89b6 100644 --- a/ui/src/datasource/DatasourceOverview.vue +++ b/ui/src/datasource/DatasourceOverview.vue @@ -97,6 +97,16 @@ + + + + @@ -132,23 +141,29 @@ const namespace = { namespace: 'pipeline' } @Component({}) export default class PipelineOverview extends Vue { @Action('loadPipelines', namespace) private loadPipelinesAction!: () => void + @Action('loadPipelineStates', namespace) private loadPipelineStatesAction!: () => void @Action('deletePipeline', namespace) private deletePipelineAction!: (id: number) => void @State('isLoadingPipelines', namespace) private isLoadingPipelines!: boolean + @State('isLoadingPipelineStates', namespace) private isLoadingPipelineStates!: boolean @State('pipelines', namespace) private pipelines!: Pipeline[] + @State('pipelineStates', namespace) private pipelineStates!: Map + private headers = [ { text: 'Id', value: 'id' }, { text: 'Datasource ID', value: 'datasourceId' }, { text: 'Pipeline Name', value: 'metadata.displayName', sortable: false }, // sorting to be implemented { text: 'Author', value: 'metadata.author', sortable: false }, - { text: 'Action', value: 'action', sortable: false } + { text: 'Action', value: 'action', sortable: false }, + { text: 'Status', value: 'health', sortable: false } ] private search = '' private mounted (): void { this.loadPipelinesAction() + this.loadPipelineStatesAction() } private onShowPipelineData (pipeline: Pipeline): void { diff --git a/ui/src/pipeline/pipeline.module.ts b/ui/src/pipeline/pipeline.module.ts index a2096b175..c03c9bfef 100644 --- a/ui/src/pipeline/pipeline.module.ts +++ b/ui/src/pipeline/pipeline.module.ts @@ -1,12 +1,15 @@ import { Module, VuexModule, Mutation, Action } from 'vuex-module-decorators' -import Pipeline from './pipeline' +import Pipeline, { HealthStatus, TransformedDataMetaData } from './pipeline' import * as RestService from './pipelineRest' +import * as RestTransService from './pipelineTransRest' @Module({ namespaced: true }) export default class PipelineModule extends VuexModule { private pipelines: Pipeline[] = [] + private pipelineStates: Map = new Map() private selectedPipeline?: Pipeline = undefined private isLoadingPipelines = true + private isLoadingPipelineStates = true @Mutation public setPipelines (pipelines: Pipeline[]): void { @@ -14,6 +17,12 @@ export default class PipelineModule extends VuexModule { this.isLoadingPipelines = false } + @Mutation + public setPipelineStates (value: Map): void { + this.pipelineStates = value + this.isLoadingPipelineStates = false + } + @Mutation public setSelectedPipeline (pipeline: Pipeline): void { this.selectedPipeline = pipeline @@ -24,6 +33,11 @@ export default class PipelineModule extends VuexModule { this.isLoadingPipelines = value } + @Mutation + public setIsLoadingPipelineStates (value: boolean): void { + this.isLoadingPipelines = value + } + @Action({ commit: 'setPipelines', rawError: true }) public async loadPipelines (): Promise { this.context.commit('setIsLoadingPipelines', true) @@ -31,6 +45,18 @@ export default class PipelineModule extends VuexModule { return await RestService.getAllPipelines() } + @Action({ commit: 'setPipelineStates', rawError: true }) + private async loadPipelineStates (): Promise> { + this.context.commit('setIsLoadingPipelineStates', true) + const pipelineStates = new Map() + for (const element of this.pipelines) { + const transformedData: TransformedDataMetaData = await RestTransService.getLatestTransformedData(element.id) + pipelineStates.set(element.id, this.getHealthColor(transformedData.health)) + } + + return pipelineStates + } + @Action({ commit: 'setSelectedPipeline', rawError: true }) public async loadPipelineById (id: number): Promise { return await RestService.getPipelineById(id) @@ -58,4 +84,14 @@ export default class PipelineModule extends VuexModule { await RestService.deletePipeline(pipelineId) return await RestService.getAllPipelines() } + + private getHealthColor (status: HealthStatus): string { + if (status === HealthStatus.OK) { + return 'success' + } else if (status === HealthStatus.WARINING) { + return 'orange' + } else { + return 'red' + } + } } diff --git a/ui/src/pipeline/pipeline.ts b/ui/src/pipeline/pipeline.ts index 3d165e86e..b030716a2 100644 --- a/ui/src/pipeline/pipeline.ts +++ b/ui/src/pipeline/pipeline.ts @@ -16,3 +16,15 @@ export interface PipelineMetaData { export interface TransformationConfig { func: string } + +export interface TransformedDataMetaData { + id: number + health: HealthStatus + timestamp: string +} + +export enum HealthStatus { + OK = 'OK', + WARINING = 'WARNING', + FAILED = 'FAILED' +} diff --git a/ui/src/pipeline/pipelineTransRest.ts b/ui/src/pipeline/pipelineTransRest.ts new file mode 100644 index 000000000..950232656 --- /dev/null +++ b/ui/src/pipeline/pipelineTransRest.ts @@ -0,0 +1,21 @@ +import axios from 'axios' +import { TransformedDataMetaData } from './pipeline' +import { PIPELINE_SERVICE_URL } from '@/env' + +/** + * Axios instances with default headers and base url. + * The option transformResponse is set to an empty array + * because of explicit JSON.parser call with custom reviver. + */ + +const http = axios.create({ + baseURL: `${PIPELINE_SERVICE_URL}/transdata`, + headers: { 'Content-Type': 'application/json' }, + transformResponse: [] +}) + +export async function getLatestTransformedData (id: number): Promise { + const importResponse = await http.get(`/${id}/transforms/latest`) + const jsonResponse: TransformedDataMetaData = JSON.parse(importResponse.data) + return jsonResponse +} From 89a16831308e4c7386330d4a2d1553824776ee65 Mon Sep 17 00:00:00 2001 From: Alexander Mahler Date: Thu, 8 Jul 2021 22:10:18 +0200 Subject: [PATCH 14/38] small adjustments. Testing pipeline --- docker-compose.it.yml | 2 +- .../pipeline-config/model/pipelineConfig.ts | 4 +- .../model/pipelineTransformedData.ts | 4 +- .../pipeline-config/outboxEventPublisher.ts | 11 +++- .../pipeline-config/pipelineConfigManager.ts | 55 ++++++++++++------- 5 files changed, 48 insertions(+), 28 deletions(-) diff --git a/docker-compose.it.yml b/docker-compose.it.yml index d5061cbe0..ebf67323d 100644 --- a/docker-compose.it.yml +++ b/docker-compose.it.yml @@ -80,7 +80,7 @@ services: AMQP_IMPORT_SUCCESS_TOPIC: "datasource.execution.success" PUBLICATION_WAIT_TIME_MS: 2000 - PIPELINE_EXECUTION_WAIT_TIME_MS: 10000 + PIPELINE_EXECUTION_WAIT_TIME_MS: 100000 CONNECTION_RETRIES: 3 CONNECTION_BACKOFF: 2000 diff --git a/pipeline/src/pipeline-config/model/pipelineConfig.ts b/pipeline/src/pipeline-config/model/pipelineConfig.ts index 3d093ffd4..bd54e9d47 100644 --- a/pipeline/src/pipeline-config/model/pipelineConfig.ts +++ b/pipeline/src/pipeline-config/model/pipelineConfig.ts @@ -5,7 +5,7 @@ export interface PipelineConfig { datasourceId: number transformation: TransformationConfig metadata: Metadata - schema: object + schema?: object } export interface TransformationConfig { @@ -27,7 +27,7 @@ export interface PipelineConfigDTO { datasourceId: number transformation: TransformationConfig metadata: MetadataDTO - schema: object + schema?: object } export interface MetadataDTO { diff --git a/pipeline/src/pipeline-config/model/pipelineTransformedData.ts b/pipeline/src/pipeline-config/model/pipelineTransformedData.ts index a9155b353..1108b55d9 100644 --- a/pipeline/src/pipeline-config/model/pipelineTransformedData.ts +++ b/pipeline/src/pipeline-config/model/pipelineTransformedData.ts @@ -5,7 +5,7 @@ export interface PipelineTransformedData { pipelineId: number healthStatus: string data: unknown - schema: object + schema?: object createdAt?: string } @@ -19,7 +19,7 @@ export interface PipelineTransformedDataDTO { pipelineId: number healthStatus: string data: unknown - schema: object + schema?: object } export class PipelineTransformedDataDTOValidator { diff --git a/pipeline/src/pipeline-config/outboxEventPublisher.ts b/pipeline/src/pipeline-config/outboxEventPublisher.ts index 57ce1f813..ccb02b971 100644 --- a/pipeline/src/pipeline-config/outboxEventPublisher.ts +++ b/pipeline/src/pipeline-config/outboxEventPublisher.ts @@ -42,11 +42,16 @@ export async function publishError export async function publishSuccess (client: ClientBase, pipelineId: number, pipelineName: string, result: unknown, schema?: object): Promise { - const content = { + let content: any = { pipelineId: pipelineId, pipelineName: pipelineName, - data: result, - schema: schema + data: result + } + if (schema !== undefined || schema !== null) { + content = { + ...content, + schema: schema as object + } } return await insertEvent(client, AMQP_PIPELINE_EXECUTION_SUCCESS_TOPIC, content) } diff --git a/pipeline/src/pipeline-config/pipelineConfigManager.ts b/pipeline/src/pipeline-config/pipelineConfigManager.ts index 822dde782..e3e2242fe 100644 --- a/pipeline/src/pipeline-config/pipelineConfigManager.ts +++ b/pipeline/src/pipeline-config/pipelineConfigManager.ts @@ -72,32 +72,47 @@ export class PipelineConfigManager { } else if ('data' in result) { let validate: any let valid: boolean = true - if (config.schema !== undefined || config.schema !== null) { - validate = ajv.compile(config.schema) - valid = validate(result.data) - } - - const transformedData: PipelineTransformedDataDTO = { + let transformedData: PipelineTransformedDataDTO = { pipelineId: config.id, healthStatus: 'OK', - data: result.data as object, - schema: config.schema + data: result.data as object } + if (config.schema !== undefined || config.schema !== null) { + validate = ajv.compile(config.schema as object) + valid = validate(result.data) - if (!valid) { - transformedData.healthStatus = 'WARNING' - } - await this.pipelineTransformedDataManager.insert(transformedData) + transformedData = { + ...transformedData, + schema: config.schema as object + } - await this.pgClient.transaction(async client => - await EventPublisher.publishSuccess( - client, - config.id, - config.metadata.displayName, - result.data, - config.schema + if (!valid) { + transformedData.healthStatus = 'WARNING' + } + await this.pipelineTransformedDataManager.insert(transformedData) + + await this.pgClient.transaction(async client => + await EventPublisher.publishSuccess( + client, + config.id, + config.metadata.displayName, + result.data, + config.schema + ) ) - ) + } else { + console.debug('***********WITHOUT SCHEMA************') + await this.pipelineTransformedDataManager.insert(transformedData) + + await this.pgClient.transaction(async client => + await EventPublisher.publishSuccess( + client, + config.id, + config.metadata.displayName, + result.data + ) + ) + } } else { console.error(`Pipeline ${config.id} executed with ambiguous result: no data and no error!`) } From da988fd1076f757325543f9221e09f6abcbb8aad Mon Sep 17 00:00:00 2001 From: Alexander Mahler Date: Sat, 10 Jul 2021 13:08:19 +0200 Subject: [PATCH 15/38] Pipeline UI is now reactive --- docker-compose.it.yml | 2 +- .../pipeline-config/outboxEventPublisher.ts | 1 + ui/src/pipeline/PipelineOverview.vue | 10 ++++----- ui/src/pipeline/pipeline.module.ts | 22 +++++++++---------- ui/src/pipeline/pipeline.ts | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/docker-compose.it.yml b/docker-compose.it.yml index ebf67323d..d5061cbe0 100644 --- a/docker-compose.it.yml +++ b/docker-compose.it.yml @@ -80,7 +80,7 @@ services: AMQP_IMPORT_SUCCESS_TOPIC: "datasource.execution.success" PUBLICATION_WAIT_TIME_MS: 2000 - PIPELINE_EXECUTION_WAIT_TIME_MS: 100000 + PIPELINE_EXECUTION_WAIT_TIME_MS: 10000 CONNECTION_RETRIES: 3 CONNECTION_BACKOFF: 2000 diff --git a/pipeline/src/pipeline-config/outboxEventPublisher.ts b/pipeline/src/pipeline-config/outboxEventPublisher.ts index ccb02b971..de9dc9690 100644 --- a/pipeline/src/pipeline-config/outboxEventPublisher.ts +++ b/pipeline/src/pipeline-config/outboxEventPublisher.ts @@ -53,5 +53,6 @@ export async function publishSuccess schema: schema as object } } + console.log('**********PIPELOINEEXECUTIONSUCCESS************') return await insertEvent(client, AMQP_PIPELINE_EXECUTION_SUCCESS_TOPIC, content) } diff --git a/ui/src/pipeline/PipelineOverview.vue b/ui/src/pipeline/PipelineOverview.vue index 9dc193102..3531727da 100644 --- a/ui/src/pipeline/PipelineOverview.vue +++ b/ui/src/pipeline/PipelineOverview.vue @@ -140,8 +140,8 @@ const namespace = { namespace: 'pipeline' } @Component({}) export default class PipelineOverview extends Vue { - @Action('loadPipelines', namespace) private loadPipelinesAction!: () => void - @Action('loadPipelineStates', namespace) private loadPipelineStatesAction!: () => void + @Action('loadPipelines', namespace) private loadPipelinesAction!: () => Promise + @Action('loadPipelineStates', namespace) private loadPipelineStatesAction!: () => Promise @Action('deletePipeline', namespace) private deletePipelineAction!: (id: number) => void @State('isLoadingPipelines', namespace) private isLoadingPipelines!: boolean @@ -161,9 +161,9 @@ export default class PipelineOverview extends Vue { private search = '' - private mounted (): void { - this.loadPipelinesAction() - this.loadPipelineStatesAction() + private async mounted (): Promise { + await this.loadPipelinesAction() + await this.loadPipelineStatesAction() } private onShowPipelineData (pipeline: Pipeline): void { diff --git a/ui/src/pipeline/pipeline.module.ts b/ui/src/pipeline/pipeline.module.ts index c03c9bfef..20c766ae7 100644 --- a/ui/src/pipeline/pipeline.module.ts +++ b/ui/src/pipeline/pipeline.module.ts @@ -51,7 +51,17 @@ export default class PipelineModule extends VuexModule { const pipelineStates = new Map() for (const element of this.pipelines) { const transformedData: TransformedDataMetaData = await RestTransService.getLatestTransformedData(element.id) - pipelineStates.set(element.id, this.getHealthColor(transformedData.health)) + + let healthStatus: string + if (transformedData.healthStatus === HealthStatus.OK) { + healthStatus = 'success' + } else if (transformedData.healthStatus === HealthStatus.WARINING) { + healthStatus = 'orange' + } else { + healthStatus = 'red' + } + + pipelineStates.set(element.id, healthStatus) } return pipelineStates @@ -84,14 +94,4 @@ export default class PipelineModule extends VuexModule { await RestService.deletePipeline(pipelineId) return await RestService.getAllPipelines() } - - private getHealthColor (status: HealthStatus): string { - if (status === HealthStatus.OK) { - return 'success' - } else if (status === HealthStatus.WARINING) { - return 'orange' - } else { - return 'red' - } - } } diff --git a/ui/src/pipeline/pipeline.ts b/ui/src/pipeline/pipeline.ts index b030716a2..2a59503ef 100644 --- a/ui/src/pipeline/pipeline.ts +++ b/ui/src/pipeline/pipeline.ts @@ -19,7 +19,7 @@ export interface TransformationConfig { export interface TransformedDataMetaData { id: number - health: HealthStatus + healthStatus: HealthStatus timestamp: string } From 88f3e0b5b4f74b5526b981a0b9d10f307b3404d9 Mon Sep 17 00:00:00 2001 From: Alexander Mahler Date: Sun, 11 Jul 2021 16:22:11 +0200 Subject: [PATCH 16/38] Adjusted error message handling for schema validation --- .../datasource/DatasourceManager.java | 8 ++++++-- .../datasource/model/DataImport.java | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java index 85a50e297..2cc4d31ce 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java @@ -21,6 +21,9 @@ import org.json.JSONObject; import org.json.JSONTokener; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.util.List; +import java.lang.reflect.Type; import java.io.IOException; import java.util.Collection; @@ -145,7 +148,6 @@ DataImport.MetaData executeImport(Long id, RuntimeParameters runtimeParameters) return savedDataImport.getMetaData(); } catch (ValidationException e) { - System.out.println("exception catch validation"); return handleImportWarning(datasource, dataImport, e, runtimeParameters); } catch (ImporterParameterException | InterpreterParameterException | IOException e) { handleImportFailed(datasource, dataImport, e); @@ -160,8 +162,10 @@ private DataImport.MetaData handleImportWarning ( ValidationException e, RuntimeParameters runtimeParameters ) { - System.out.println("seting warning"); dataImport.setHealth("WARNING"); + Type listType = new TypeToken>() {}.getType(); + String errorMessagesAsString = new Gson().toJson(e.getAllMessages(), listType); + dataImport.setErrorMessages(errorMessagesAsString); DataImport savedDataImport = dataImportRepository.save(dataImport); amqpPublisher.publishImportSuccess(datasource.getId(), savedDataImport.getData()); return savedDataImport.getMetaData(); diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java index 951728e16..aa188f6c0 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java @@ -8,7 +8,11 @@ import javax.persistence.*; import java.nio.charset.StandardCharsets; import java.util.Date; -import java.util.List; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; +import com.vladmihalcea.hibernate.type.json.JsonBinaryType; @Entity @NoArgsConstructor @@ -26,8 +30,8 @@ public class DataImport { private String health; - @ElementCollection - private List errorMessages; + @Column(columnDefinition="TEXT") + private String errorMessages; @ManyToOne(fetch = FetchType.EAGER) @JoinColumn(name="datasource_id") @@ -42,7 +46,7 @@ public DataImport(Datasource datasource, String data, String health) { this(datasource, data, health, null); } - public DataImport(Datasource datasource, String data, String health, List errorMessages) { + public DataImport(Datasource datasource, String data, String health, String errorMessages) { this.datasource = datasource; this.data = data.getBytes(StandardCharsets.UTF_8); this.health = health; @@ -54,6 +58,10 @@ public void setHealth(String health) { this.health = health; } + public void setErrorMessages(String errorMessages) { + this.errorMessages = errorMessages; + } + public String getData() { return new String(data, StandardCharsets.UTF_8); } @@ -74,7 +82,7 @@ public static class MetaData { private final String health; - private final List errorMessages; + private final String errorMessages; @JsonIgnore private final Datasource datasource; From ce228fa98b5e478d7daa048abcb9101d513e6f3b Mon Sep 17 00:00:00 2001 From: Alexander Mahler Date: Sun, 11 Jul 2021 22:50:55 +0200 Subject: [PATCH 17/38] Validator refactored and now interchangable. Tests added --- adapter/build.gradle | 2 +- .../datasource/DatasourceManager.java | 59 ++----------------- .../datasource/model/DataImport.java | 23 ++++---- .../validator/JsonSchemaValidator.java | 51 ++++++++++++++++ .../validator/ValidationMetaData.java | 29 +++++++++ .../datasource/validator/Validator.java | 9 +++ .../datasource/DatasourceManagerTest.java | 2 +- .../{model => config}/DatasourceConfig.json | 0 .../config/DatasourceConfigNoSchema.json | 25 ++++++++ .../datasource/model/DataImportTest.java | 2 - .../datasource/model/DatasourceTest.java | 2 +- .../validator/JsonSchemaValidatorTest.java | 59 +++++++++++++++++++ 12 files changed, 192 insertions(+), 71 deletions(-) create mode 100644 adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidator.java create mode 100644 adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/ValidationMetaData.java create mode 100644 adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/Validator.java rename adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/{model => config}/DatasourceConfig.json (100%) create mode 100644 adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfigNoSchema.json create mode 100644 adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidatorTest.java diff --git a/adapter/build.gradle b/adapter/build.gradle index 92f5409c4..f3770f0e2 100644 --- a/adapter/build.gradle +++ b/adapter/build.gradle @@ -33,7 +33,7 @@ test { testLogging { events "passed", "skipped", "failed" - showStandardStreams = true + //showStandardStreams = true afterSuite { desc, result -> if (!desc.parent) { println "Tests run: ${result.testCount}, " + diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java index 2cc4d31ce..4ed23ca2c 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/DatasourceManager.java @@ -13,17 +13,9 @@ import org.jvalue.ods.adapterservice.datasource.repository.DatasourceRepository; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; - -import org.everit.json.schema.Schema; -import org.everit.json.schema.loader.SchemaLoader; import org.everit.json.schema.ValidationException; -import org.json.JSONArray; -import org.json.JSONObject; -import org.json.JSONTokener; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import java.util.List; -import java.lang.reflect.Type; + +import org.jvalue.ods.adapterservice.datasource.validator.*; import java.io.IOException; import java.util.Collection; @@ -82,25 +74,6 @@ public void deleteAllDatasources() { StreamSupport.stream(allDatasourceConfigs.spliterator(), true).forEach(amqpPublisher::publishDeletion); } - public void validate(Datasource datasource, DataImport dataImport) throws IOException, ValidationException { - try - { - String schemaString = new Gson().toJson(datasource.getSchema()); - JSONObject rawSchema = new JSONObject(schemaString); - Schema schema = SchemaLoader.load(rawSchema); - String dataString = dataImport.getData(); - if (dataString.substring(0, 1).equals("[")) { - schema.validate(new JSONArray(dataImport.getData())); - } - else { - schema.validate(new JSONObject(dataImport.getData())); - } - } catch ( ValidationException e) { - throw e; - } - - } - public DataImport.MetaData trigger(Long id, RuntimeParameters runtimeParameters) throws DatasourceNotFoundException, AdapterException, IOException { try { return executeImport(id, runtimeParameters); @@ -130,47 +103,25 @@ public DataImport.MetaData trigger(Long id, RuntimeParameters runtimeParameters) DataImport.MetaData executeImport(Long id, RuntimeParameters runtimeParameters) throws DatasourceNotFoundException, ImporterParameterException, InterpreterParameterException, IOException { Datasource datasource = getDatasource(id); - DataImport dataImport = new DataImport(datasource, "", "FAILED"); + DataImport dataImport = new DataImport(datasource, "", ValidationMetaData.HealthStatus.FAILED); + Validator validator = new JsonSchemaValidator(); try { - AdapterConfig adapterConfig = datasource.toAdapterConfig(runtimeParameters); DataImportResponse executionResult = adapter.executeJob(adapterConfig); String responseData = executionResult.getData(); dataImport = new DataImport(datasource, responseData); - - if (datasource.getSchema() != null) { - validate(datasource, dataImport); - } + dataImport.setValidationMetaData(validator.validate(dataImport)); DataImport savedDataImport = dataImportRepository.save(dataImport); - amqpPublisher.publishImportSuccess(id, savedDataImport.getData()); return savedDataImport.getMetaData(); - } catch (ValidationException e) { - return handleImportWarning(datasource, dataImport, e, runtimeParameters); } catch (ImporterParameterException | InterpreterParameterException | IOException e) { handleImportFailed(datasource, dataImport, e); throw e; } } - @Transactional - private DataImport.MetaData handleImportWarning ( - Datasource datasource, - DataImport dataImport, - ValidationException e, - RuntimeParameters runtimeParameters - ) { - dataImport.setHealth("WARNING"); - Type listType = new TypeToken>() {}.getType(); - String errorMessagesAsString = new Gson().toJson(e.getAllMessages(), listType); - dataImport.setErrorMessages(errorMessagesAsString); - DataImport savedDataImport = dataImportRepository.save(dataImport); - amqpPublisher.publishImportSuccess(datasource.getId(), savedDataImport.getData()); - return savedDataImport.getMetaData(); - } - @Transactional void handleImportFailed(Datasource datasource, DataImport dataImport, Exception e){ DataImport savedDataImport = dataImportRepository.save(dataImport); diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java index aa188f6c0..45dc86aa4 100644 --- a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/model/DataImport.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.jvalue.ods.adapterservice.datasource.api.rest.v1.Mappings; +import org.jvalue.ods.adapterservice.datasource.validator.ValidationMetaData; import javax.persistence.*; import java.nio.charset.StandardCharsets; @@ -28,7 +29,8 @@ public class DataImport { private Date timestamp; - private String health; + @Enumerated(EnumType.STRING) + private ValidationMetaData.HealthStatus health; @Column(columnDefinition="TEXT") private String errorMessages; @@ -39,14 +41,14 @@ public class DataImport { private Datasource datasource; public DataImport(Datasource datasource, String data) { - this(datasource, data, "OK", null); + this(datasource, data, ValidationMetaData.HealthStatus.OK, ""); } - public DataImport(Datasource datasource, String data, String health) { - this(datasource, data, health, null); + public DataImport(Datasource datasource, String data, ValidationMetaData.HealthStatus health) { + this(datasource, data, health, ""); } - public DataImport(Datasource datasource, String data, String health, String errorMessages) { + public DataImport(Datasource datasource, String data, ValidationMetaData.HealthStatus health, String errorMessages) { this.datasource = datasource; this.data = data.getBytes(StandardCharsets.UTF_8); this.health = health; @@ -54,12 +56,9 @@ public DataImport(Datasource datasource, String data, String health, String erro this.timestamp = new Date(); } - public void setHealth(String health) { - this.health = health; - } - - public void setErrorMessages(String errorMessages) { - this.errorMessages = errorMessages; + public void setValidationMetaData(ValidationMetaData validationData) { + this.health = validationData.getHealthStatus(); + this.errorMessages = validationData.getErrorMessages(); } public String getData() { @@ -80,7 +79,7 @@ public static class MetaData { @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", locale = "UTC") private final Date timestamp; - private final String health; + private final ValidationMetaData.HealthStatus health; private final String errorMessages; diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidator.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidator.java new file mode 100644 index 000000000..ff85ef248 --- /dev/null +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidator.java @@ -0,0 +1,51 @@ +package org.jvalue.ods.adapterservice.datasource.validator; + +import org.jvalue.ods.adapterservice.datasource.validator.ValidationMetaData; +import org.jvalue.ods.adapterservice.datasource.model.*; +import org.jvalue.ods.adapterservice.datasource.model.exceptions.*; +import java.io.IOException; +import org.everit.json.schema.ValidationException; + +import org.everit.json.schema.Schema; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONTokener; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.util.List; +import java.lang.reflect.Type; + +public class JsonSchemaValidator implements Validator { + + @Override + public ValidationMetaData validate(DataImport dataImport){ + ValidationMetaData validationMetaData = new ValidationMetaData(dataImport.getHealth(), ""); + if (dataImport.getDatasource().getSchema() == null) { + return new ValidationMetaData(ValidationMetaData.HealthStatus.OK, ""); + } + try { + String schemaString = new Gson().toJson(dataImport.getDatasource().getSchema()); + JSONObject rawSchema = new JSONObject(schemaString); + Schema schema = SchemaLoader.load(rawSchema); + String dataString = dataImport.getData(); + if (dataString.substring(0, 1).equals("[")) { + schema.validate(new JSONArray(dataImport.getData())); + } + else { + schema.validate(new JSONObject(dataImport.getData())); + } + + validationMetaData.setHealthStatus(ValidationMetaData.HealthStatus.OK); + return validationMetaData; + } catch ( ValidationException e) { + Type listType = new TypeToken>() {}.getType(); + String errorMessagesAsString = new Gson().toJson(e.getAllMessages(), listType); + + validationMetaData.setErrorMessages(errorMessagesAsString); + validationMetaData.setHealthStatus(ValidationMetaData.HealthStatus.WARNING); + + return validationMetaData; + } + } +} \ No newline at end of file diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/ValidationMetaData.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/ValidationMetaData.java new file mode 100644 index 000000000..df76bd429 --- /dev/null +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/ValidationMetaData.java @@ -0,0 +1,29 @@ +package org.jvalue.ods.adapterservice.datasource.validator; + +import lombok.*; + +@AllArgsConstructor +@Getter +@Setter +public class ValidationMetaData { + private HealthStatus healthStatus; + private String errorMessages; + + public static enum HealthStatus { + OK { + public String toString() { + return "OK"; + } + }, + WARNING { + public String toString() { + return "WARNING"; + } + }, + FAILED { + public String toString() { + return "FAILED"; + } + } + } +} diff --git a/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/Validator.java b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/Validator.java new file mode 100644 index 000000000..4d341977a --- /dev/null +++ b/adapter/src/main/java/org/jvalue/ods/adapterservice/datasource/validator/Validator.java @@ -0,0 +1,9 @@ +package org.jvalue.ods.adapterservice.datasource.validator; + +import org.jvalue.ods.adapterservice.datasource.model.*; +import org.jvalue.ods.adapterservice.datasource.validator.ValidationMetaData; + +public interface Validator { + public ValidationMetaData validate(DataImport dataImport); +} + diff --git a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/DatasourceManagerTest.java b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/DatasourceManagerTest.java index c1595bb2f..2049fec39 100644 --- a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/DatasourceManagerTest.java +++ b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/DatasourceManagerTest.java @@ -42,7 +42,7 @@ public class DatasourceManagerTest { private final ObjectMapper mapper = new ObjectMapper(); private final File configFile = new File( - "src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceConfig.json"); + "src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfig.json"); @Mock DatasourceRepository datasourceRepository; diff --git a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceConfig.json b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfig.json similarity index 100% rename from adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceConfig.json rename to adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfig.json diff --git a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfigNoSchema.json b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfigNoSchema.json new file mode 100644 index 000000000..e70dd2043 --- /dev/null +++ b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfigNoSchema.json @@ -0,0 +1,25 @@ +{ + "id": 123, + "protocol": { + "type": "HTTP", + "parameters": { + "location": "http://www.test-url.com" + } + }, + "format": { + "type": "XML", + "parameters": {} + + }, + "trigger": { + "firstExecution": "1905-12-01T02:30:00.123Z", + "periodic": true, + "interval": 50000 + }, + "metadata": { + "author": "person", + "license": "none", + "displayName": "TestName", + "description": "Describing..." + } +} diff --git a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DataImportTest.java b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DataImportTest.java index ed45ead36..486fa8b47 100644 --- a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DataImportTest.java +++ b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DataImportTest.java @@ -24,7 +24,6 @@ public void testSerialization() throws ParseException, IOException { JsonNode result = mapper.valueToTree(dataImport); - System.out.println(result.toString()); assertEquals(5, result.size()); assertEquals("null", result.get("id").asText()); assertEquals("{\"whateverwillbe\":\"willbe\",\"quesera\":\"sera\"}", result.get("data").asText()); @@ -38,7 +37,6 @@ public void testMetaDataSerialization() throws ParseException, IOException { JsonNode result = mapper.valueToTree(dataImport.getMetaData()); - System.out.println(result.toString()); assertEquals(5, result.size()); assertEquals("null", result.get("id").asText()); assertEquals("/datasources/null/imports/null/data", result.get("location").asText()); diff --git a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceTest.java b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceTest.java index 5509321c7..c8c9c698b 100644 --- a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceTest.java +++ b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceTest.java @@ -28,7 +28,7 @@ public class DatasourceTest { @Test public void testDeserialization() throws IOException, ParseException { - File datasourceConfig = new File("src/test/java/org/jvalue/ods/adapterservice/datasource/model/DatasourceConfig.json"); + File datasourceConfig = new File("src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfig.json"); Datasource result = mapper.readValue(datasourceConfig, Datasource.class); Datasource expectedDatasource = generateDatasource(HTTP, XML, "http://www.test-url.com"); diff --git a/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidatorTest.java b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidatorTest.java new file mode 100644 index 000000000..1984fd4d7 --- /dev/null +++ b/adapter/src/test/java/org/jvalue/ods/adapterservice/datasource/validator/JsonSchemaValidatorTest.java @@ -0,0 +1,59 @@ +package org.jvalue.ods.adapterservice.datasource.validator; + +import org.jvalue.ods.adapterservice.datasource.model.*; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; + + +public class JsonSchemaValidatorTest { + private final ObjectMapper mapper = new ObjectMapper(); + private final Validator validator = new JsonSchemaValidator(); + + @Test + public void testValidationSuccess() throws IOException { + File datasourceConfig = new File("src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfig.json"); + Datasource datasourceConfigComplete = mapper.readValue(datasourceConfig, Datasource.class); + String data = "{\"hallo\":\"test\"}"; + DataImport dataImport = new DataImport(datasourceConfigComplete, data); + + ValidationMetaData expectedMetaData = new ValidationMetaData(ValidationMetaData.HealthStatus.OK, ""); + ValidationMetaData result = validator.validate(dataImport); + + assertEquals(expectedMetaData.getHealthStatus(), result.getHealthStatus()); + assertEquals(expectedMetaData.getErrorMessages(), result.getErrorMessages()); + } + + @Test + public void testValidationNoSchema() throws IOException { + File datasourceConfig = new File("src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfigNoSchema.json"); + Datasource datasourceConfigNoSchema = mapper.readValue(datasourceConfig, Datasource.class); + String data = "{\"hallo\":\"test\"}"; + DataImport dataImport = new DataImport(datasourceConfigNoSchema, data); + + ValidationMetaData expectedMetaData = new ValidationMetaData(ValidationMetaData.HealthStatus.OK, ""); + ValidationMetaData result = validator.validate(dataImport); + + assertEquals(expectedMetaData.getHealthStatus(), result.getHealthStatus()); + assertEquals(expectedMetaData.getErrorMessages(), result.getErrorMessages()); + } + + @Test + public void testValidationWarning() throws IOException { + File datasourceConfig = new File("src/test/java/org/jvalue/ods/adapterservice/datasource/config/DatasourceConfig.json"); + Datasource datasourceConfigComplete = mapper.readValue(datasourceConfig, Datasource.class); + String data = "{\"hallo\":1}"; + DataImport dataImport = new DataImport(datasourceConfigComplete, data); + + ValidationMetaData expectedMetaData = new ValidationMetaData( + ValidationMetaData.HealthStatus.WARNING, + "[\"#/hallo: expected type: String, found: Integer\"]"); + + ValidationMetaData result = validator.validate(dataImport); + assertEquals(expectedMetaData.getHealthStatus(), result.getHealthStatus()); + assertEquals(expectedMetaData.getErrorMessages(), result.getErrorMessages()); + } +} From dea75625c8d2b95d01add70644bd5c2a5cc64861 Mon Sep 17 00:00:00 2001 From: Alexander Mahler Date: Sun, 11 Jul 2021 23:29:22 +0200 Subject: [PATCH 18/38] removed start.sh --- start.sh | 5 ----- ui/src/datasource/DatasourceCreate.vue | 1 - ui/src/datasource/DatasourceOverview.vue | 13 ++++++------- ui/src/datasource/datasource.ts | 2 +- .../datasource/edit/schema/DatasourceSchemaEdit.vue | 4 +++- 5 files changed, 10 insertions(+), 15 deletions(-) delete mode 100644 start.sh diff --git a/start.sh b/start.sh deleted file mode 100644 index 466dc1f78..000000000 --- a/start.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -docker-compose -f ./docker-compose.yml -f ./../schema-recommendation/docker-compose.yml up -d adapter-db -sleep 3 -docker-compose -f ./docker-compose.yml -f ./../schema-recommendation/docker-compose.yml up \ No newline at end of file diff --git a/ui/src/datasource/DatasourceCreate.vue b/ui/src/datasource/DatasourceCreate.vue index d79156657..b6145b522 100644 --- a/ui/src/datasource/DatasourceCreate.vue +++ b/ui/src/datasource/DatasourceCreate.vue @@ -67,7 +67,6 @@ export default class DatasourceCreate extends Vue { type: 'JSON', parameters: {} }, - schema: {}, metadata: { author: '', license: '', diff --git a/ui/src/datasource/DatasourceOverview.vue b/ui/src/datasource/DatasourceOverview.vue index 7357d5fdc..995a847da 100644 --- a/ui/src/datasource/DatasourceOverview.vue +++ b/ui/src/datasource/DatasourceOverview.vue @@ -100,7 +100,7 @@