Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions packages/gooddata-dbt/src/gooddata_dbt/dbt/metrics.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# (C) 2023 GoodData Corporation
import json
import re
from typing import Optional

import attrs
from gooddata_sdk import CatalogDeclarativeMetric, CatalogDeclarativeModel
Expand All @@ -26,8 +25,8 @@

@attrs.define(auto_attribs=True, kw_only=True)
class DbtModelMetaGoodDataMetricProps(Base):
model_id: Optional[str] = None
format: Optional[str] = None
model_id: str | None = None
format: str | None = None


@attrs.define(auto_attribs=True, kw_only=True)
Expand All @@ -49,11 +48,11 @@ class DbtModelMetric(DbtModelBase):
model: str
calculation_method: str
expression: str
filters: Optional[list[DbtModelMetricFilter]] = None
filters: list[DbtModelMetricFilter] | None = None


class DbtModelMetrics:
def __init__(self, model_ids: Optional[list[str]], ldm: CatalogDeclarativeModel) -> None:
def __init__(self, model_ids: list[str] | None, ldm: CatalogDeclarativeModel) -> None:
self.model_ids = model_ids
self.ldm = ldm
with open(DBT_PATH_TO_MANIFEST) as fp:
Expand Down Expand Up @@ -104,7 +103,7 @@ def get_entity_type(self, table_name: str, expression_entity: str) -> str:
else:
raise Exception(f"Unsupported entity type {table_name=} {expression_entity=}")

def make_entity_id(self, table_name: str, token: str) -> Optional[str]:
def make_entity_id(self, table_name: str, token: str) -> str | None:
entity_type = self.get_entity_type(table_name, token)
if not entity_type:
return None
Expand All @@ -125,7 +124,7 @@ def resolve_entities_in_expression(self, expression: str, table_name: str) -> st
result_tokens.append(entity_id or token)
return " ".join(result_tokens)

def make_gooddata_filter(self, table_name: str, dbt_filters: Optional[list[DbtModelMetricFilter]] = None) -> str:
def make_gooddata_filter(self, table_name: str, dbt_filters: list[DbtModelMetricFilter] | None = None) -> str:
# TODO - Quite naive implementation
# e.g. missing polishing of values (e.g. SQL vs MAQL enclosers)
gd_maql_filters = []
Expand Down
6 changes: 3 additions & 3 deletions packages/gooddata-dbt/src/gooddata_dbt/dbt/profiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import argparse
import os
import re
from typing import Optional, Union
from typing import Union
from urllib.parse import quote_plus

import attrs
Expand Down Expand Up @@ -97,7 +97,7 @@ class DbtOutputSnowflake(Base):
database: str
warehouse: str
schema: str
query_tag: Optional[str] = None
query_tag: str | None = None

def to_gooddata(self, data_source_id: str, schema_name: str) -> CatalogDataSourceSnowflake:
return CatalogDataSourceSnowflake(
Expand Down Expand Up @@ -222,7 +222,7 @@ def inject_env_vars(output_def: dict) -> None:
# else do nothing, real value seems to be stored in dbt profile

@staticmethod
def to_data_class(output: str, output_def: dict) -> Optional[DbtOutput]:
def to_data_class(output: str, output_def: dict) -> DbtOutput | None:
db_type = output_def["type"]
if db_type == "postgres":
return DbtOutputPostgreSQL.from_dict({"name": output, **output_def})
Expand Down
38 changes: 19 additions & 19 deletions packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import json
import re
from pathlib import Path
from typing import Optional, Union
from typing import Union

import attrs
from gooddata_sdk import CatalogDeclarativeColumn, CatalogDeclarativeTable, CatalogDeclarativeTables
Expand All @@ -27,22 +27,22 @@

@attrs.define(auto_attribs=True, kw_only=True)
class DbtModelMetaGoodDataTableProps(Base):
model_id: Optional[str] = None
model_id: str | None = None


@attrs.define(auto_attribs=True, kw_only=True)
class DbtModelMetaGoodDataColumnProps(Base):
id: Optional[str] = None
ldm_type: Optional[GoodDataLdmType] = None
referenced_table: Optional[str] = None
label_type: Optional[GoodDataLabelType] = None
attribute_column: Optional[str] = None
sort_column: Optional[str] = None
sort_direction: Optional[GoodDataSortDirection] = None
default_view: Optional[bool] = None
id: str | None = None
ldm_type: GoodDataLdmType | None = None
referenced_table: str | None = None
label_type: GoodDataLabelType | None = None
attribute_column: str | None = None
sort_column: str | None = None
sort_direction: GoodDataSortDirection | None = None
default_view: bool | None = None

@property
def gooddata_ref_table_ldm_id(self) -> Optional[str]:
def gooddata_ref_table_ldm_id(self) -> str | None:
if self.referenced_table:
return self.referenced_table.lower()
return None
Expand Down Expand Up @@ -73,7 +73,7 @@ class DbtModelBase(Base):
tags: list[str]
# If 2+ references point to the same table, the table plays multiple roles,
# it must be generated as multiple datasets
role_name: Optional[str] = None
role_name: str | None = None

# TODO - duplicate of backend logic.
# Solution: use result of generateLdm as a master template, and override based on dbt metadata only if necessary
Expand Down Expand Up @@ -120,7 +120,7 @@ def upper_case_name(self) -> None:

@attrs.define(auto_attribs=True, kw_only=True)
class DbtModelColumn(DbtModelBase):
data_type: Optional[str]
data_type: str | None
meta: DbtModelMetaGoodDataColumn = attrs.field(factory=DbtModelMetaGoodDataColumn)

# Enable to override LDM ID for LDM entities derived from columns (attributes, ...)
Expand All @@ -130,21 +130,21 @@ def ldm_id(self) -> str:
return self.meta.gooddata.id or self.gooddata_ldm_id

@property
def ldm_type(self) -> Optional[str]:
def ldm_type(self) -> str | None:
if self.meta.gooddata.ldm_type is None:
return None
else:
return self.meta.gooddata.ldm_type.value

@property
def label_type(self) -> Optional[str]:
def label_type(self) -> str | None:
if self.meta.gooddata.label_type is None:
return None
else:
return self.meta.gooddata.label_type.value

@property
def sort_direction(self) -> Optional[str]:
def sort_direction(self) -> str | None:
if self.meta.gooddata.sort_direction is None:
return None
else:
Expand Down Expand Up @@ -387,7 +387,7 @@ def make_facts(table: DbtModelTable) -> list[dict]:
return facts

@staticmethod
def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple[list[dict], Optional[dict]]:
def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple[list[dict], dict | None]:
labels = []
default_view = None
for column in table.columns.values():
Expand Down Expand Up @@ -507,7 +507,7 @@ def populate_role_playing_tables(tables: list[DbtModelTable], role_playing_table
result.append(table)
return result

def make_declarative_datasets(self, data_source_id: str, model_ids: Optional[list[str]]) -> dict:
def make_declarative_datasets(self, data_source_id: str, model_ids: list[str] | None) -> dict:
result: dict[str, list] = {"datasets": [], "date_instances": []}
model_tables = [t for t in self.tables if not model_ids or t.meta.gooddata.model_id in model_ids]
role_playing_tables = self.find_role_playing_tables(model_tables)
Expand All @@ -517,7 +517,7 @@ def make_declarative_datasets(self, data_source_id: str, model_ids: Optional[lis
result = self.make_dataset(data_source_id, table, role_playing_tables, result)
return result

def get_entity_type(self, table_name: str, column_name: str) -> Optional[str]:
def get_entity_type(self, table_name: str, column_name: str) -> str | None:
comp_table_name = table_name
if self.upper_case:
comp_table_name = table_name.upper()
Expand Down
9 changes: 4 additions & 5 deletions packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from asyncio import Semaphore
from pathlib import Path
from time import time
from typing import Optional

import tabulate
import yaml
Expand Down Expand Up @@ -36,7 +35,7 @@ def generate_and_put_ldm(
data_source_id: str,
workspace_id: str,
dbt_tables: DbtModelTables,
model_ids: Optional[list[str]],
model_ids: list[str] | None,
) -> None:
scan_request = CatalogScanModelRequest(scan_tables=True, scan_views=True)
logger.info(f"Scan data source {data_source_id=}")
Expand Down Expand Up @@ -68,7 +67,7 @@ def deploy_ldm(
args: Namespace,
all_model_ids: list[str],
sdk_wrapper: GoodDataSdkWrapper,
model_ids: Optional[list[str]],
model_ids: list[str] | None,
workspace_id: str,
) -> None:
logger.info("Generate and put LDM")
Expand Down Expand Up @@ -186,7 +185,7 @@ async def test_visualizations(
logger: logging.Logger,
sdk_wrapper: GoodDataSdkWrapper,
workspace_id: str,
skip_tests: Optional[list[str]],
skip_tests: list[str] | None,
test_visualizations_parallelism: int = 1,
) -> None:
start = time()
Expand Down Expand Up @@ -334,7 +333,7 @@ def process_organization(
logger: logging.Logger,
sdk_wrapper: GoodDataSdkWrapper,
gd_config: GoodDataConfig,
organization: Optional[GoodDataConfigOrganization] = None,
organization: GoodDataConfigOrganization | None = None,
) -> None:
if args.method == "upload_notification":
dbt_profiles = DbtProfiles(args)
Expand Down
7 changes: 3 additions & 4 deletions packages/gooddata-dbt/src/gooddata_dbt/gooddata/config.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# (C) 2023 GoodData Corporation
from typing import Optional

import attr
import attrs
Expand Down Expand Up @@ -37,8 +36,8 @@ class GoodDataConfigProduct(Base):
name: str
environment_setup_id: str
model_ids: list[str] = attr.field(factory=list)
localization: Optional[GoodDataConfigLocalization] = None
skip_tests: Optional[list[str]] = None
localization: GoodDataConfigLocalization | None = None
skip_tests: list[str] | None = None


@attrs.define(auto_attribs=True, kw_only=True)
Expand All @@ -49,7 +48,7 @@ class GoodDataConfigOrganization(Base):

@attrs.define(auto_attribs=True, kw_only=True)
class GoodDataGlobalConfig(Base):
test_visualizations_parallelism: Optional[int] = 1
test_visualizations_parallelism: int | None = 1


@attrs.define(auto_attribs=True, kw_only=True)
Expand Down
7 changes: 3 additions & 4 deletions packages/gooddata-dbt/src/gooddata_dbt/sdk_wrapper.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# (C) 2023 GoodData Corporation
import argparse
from logging import Logger
from typing import Optional

from gooddata_sdk import GoodDataSdk

Expand All @@ -11,7 +10,7 @@
class GoodDataSdkWrapper:
# Timeout=600 because supporting waiting for GoodData services to start
def __init__(
self, args: argparse.Namespace, logger: Logger, profile: Optional[str] = None, timeout: int = 600
self, args: argparse.Namespace, logger: Logger, profile: str | None = None, timeout: int = 600
) -> None:
self.args = args
self.logger = logger
Expand All @@ -22,7 +21,7 @@ def __init__(
if not self.args.dry_run:
self.wait_for_gooddata_is_up(self.timeout)

def get_host_from_sdk(self) -> Optional[str]:
def get_host_from_sdk(self) -> str | None:
# TODO - make _hostname public in gooddata_sdk
return self.sdk.client._hostname

Expand Down Expand Up @@ -54,7 +53,7 @@ def wait_for_gooddata_is_up(self, timeout: int) -> None:
self.sdk.support.wait_till_available(timeout=timeout)
self.logger.info(f"Host {host} is up")

def pre_cache_visualizations(self, workspaces: Optional[list] = None) -> None:
def pre_cache_visualizations(self, workspaces: list | None = None) -> None:
if not workspaces:
workspaces = [w.id for w in self.sdk.catalog_workspace.list_workspaces()]
for workspace_id in workspaces:
Expand Down
4 changes: 2 additions & 2 deletions packages/gooddata-fdw/src/gooddata_fdw/column_utils.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# (C) 2022 GoodData Corporation
from __future__ import annotations

from typing import Optional, Union
from typing import Union

import gooddata_sdk as sdk
from gooddata_sdk import Attribute, CatalogAttribute, Metric
Expand All @@ -26,7 +26,7 @@ def table_col_as_computable(col: ColumnDefinition) -> Union[Attribute, Metric]:
)


def column_data_type_for(attribute: Optional[CatalogAttribute]) -> str:
def column_data_type_for(attribute: CatalogAttribute | None) -> str:
"""
Determine what postgres type should be used for `attribute`.

Expand Down
6 changes: 3 additions & 3 deletions packages/gooddata-fdw/src/gooddata_fdw/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

from typing import Any, Optional, Union
from typing import Any, Union

try:
import multicorn
Expand Down Expand Up @@ -88,12 +88,12 @@ def import_schema(
schema: str,
srv_options: dict[str, str],
options: dict[str, str],
restriction_type: Optional[str],
restriction_type: str | None,
restricts: list[str],
) -> list[TableDefinition]:
return NotImplemented

def execute(self, quals: list[Qual], columns: list[str], sortkeys: Optional[list[Any]] = None):
def execute(self, quals: list[Qual], columns: list[str], sortkeys: list[Any] | None = None):
pass

ForeignDataWrapper = ForeignDataWrapperStub
10 changes: 5 additions & 5 deletions packages/gooddata-fdw/src/gooddata_fdw/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from __future__ import annotations

from collections.abc import Generator
from typing import Any, NamedTuple, Optional
from typing import Any, NamedTuple

from gooddata_sdk import GoodDataSdk

Expand Down Expand Up @@ -37,7 +37,7 @@ def validate_columns_def(self) -> None:
validator.validate(column_name, column_def)

def execute(
self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None
self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None
) -> Generator[dict[str, Any], None, None]:
raise NotImplementedError()

Expand All @@ -59,7 +59,7 @@ def can_react(cls, inputs: InitData) -> bool:
return inputs.table_options.insight is not None

def execute(
self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None
self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None
) -> Generator[dict[str, Any], None, None]:
results_reader = InsightTableResultReader(self._table_columns, columns)
insight = self._sdk.visualizations.get_visualization(self._workspace, self._insight)
Expand All @@ -81,7 +81,7 @@ def can_react(cls, inputs: InitData) -> bool:
return inputs.table_options.compute is not None

def execute(
self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None
self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None
) -> Generator[dict[str, Any], None, None]:
col_val.validate_columns_in_table_def(self._table_columns, columns)
items = [column_utils.table_col_as_computable(self._table_columns[col_name]) for col_name in columns]
Expand All @@ -105,7 +105,7 @@ def can_react(cls, inputs: InitData) -> bool:
return True

def execute(
self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None
self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None
) -> Generator[dict[str, Any], None, None]:
items = [column_utils.table_col_as_computable(col) for col in self._table_columns.values()]
# TODO: pushdown more filters that are included in quals
Expand Down
Loading
Loading