diff --git a/pyproject.toml b/pyproject.toml index 369fe88..f4e7a05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "defopt>=7.0.0", "email_validator", "hdx-python-country>=4.0.1", - "hdx-python-utilities>=4.0.2", + "hdx-python-utilities>=4.0.3", "libhxl>=5.2.2", "makefun", "quantulum3", diff --git a/requirements.txt b/requirements.txt index 48785d5..ed8eb82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ frictionless==5.18.1 # via hdx-python-utilities hdx-python-country==4.0.1 # via hdx-python-api (pyproject.toml) -hdx-python-utilities==4.0.2 +hdx-python-utilities==4.0.3 # via # hdx-python-api (pyproject.toml) # hdx-python-country @@ -86,6 +86,8 @@ num2words==0.5.14 # via quantulum3 openpyxl==3.1.5 # via hdx-python-utilities +packaging==26.0 + # via wheel petl==1.7.17 # via frictionless ply==3.11 @@ -144,7 +146,7 @@ rpds-py==0.30.0 # referencing ruamel-yaml==0.19.1 # via hdx-python-utilities -setuptools==80.9.0 +setuptools==80.10.1 # via ckanapi shellingham==1.5.4 # via typer @@ -194,7 +196,7 @@ urllib3==2.6.3 # requests validators==0.35.0 # via frictionless -wheel==0.45.1 +wheel==0.46.3 # via libhxl xlrd==2.0.2 # via hdx-python-utilities diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index d861662..ad02d6f 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -2410,6 +2410,7 @@ def generate_resource( datecol: int | str | None = None, yearcol: int | str | None = None, date_function: Callable[[dict], dict | None] | None = None, + no_empty: bool = True, ) -> tuple[bool, dict]: """Write rows to file and create resource, adding it to the dataset. The headers argument is either a row number (rows start counting at 1), or the actual @@ -2440,9 +2441,10 @@ def generate_resource( columns: Columns to write. Defaults to all. format: Format to write. Defaults to csv. encoding: Encoding to use. Defaults to None (infer encoding). - datecol: Optional[Union[int, str]] = None, - yearcol: Optional[Union[int, str]] = None, - date_function: Optional[Callable[[Dict], Optional[Dict]]] = None, + datecol: Date column for setting time period. Defaults to None (don't set). + yearcol: Year column for setting dataset year range. Defaults to None (don't set). + date_function: Date function to call for each row. Defaults to None. + no_empty: Don't generate resource if there are no data rows. Defaults to True. Returns: (True if resource added, dictionary of results) @@ -2504,6 +2506,7 @@ def process_row(row: Sequence | Mapping) -> Sequence | Mapping | None: format=format, encoding=encoding, row_function=process_row, + no_empty=no_empty, ) if not rows: logger.error(f"No data rows in {filename}!") @@ -2731,6 +2734,7 @@ def download_generate_resource( datecol: int | str | None = None, yearcol: int | str | None = None, date_function: Callable[[dict], dict | None] | None = None, + no_empty: bool = True, **kwargs: Any, ) -> tuple[bool, dict]: """Download url, write rows to csv and create resource, adding to it @@ -2769,9 +2773,10 @@ def download_generate_resource( columns: Columns to write. Defaults to all. format: Format to write. Defaults to csv. encoding: Encoding to use. Defaults to None (infer encoding). - datecol: Optional[Union[int, str]] = None, - yearcol: Optional[Union[int, str]] = None, - date_function: Optional[Callable[[Dict], Optional[Dict]]] = None, + datecol: Date column for setting time period. Defaults to None (don't set). + yearcol: Year column for setting dataset year range. Defaults to None (don't set). + date_function: Date function to call for each row. Defaults to None. + no_empty: Don't generate resource if there are no data rows. Defaults to True. **kwargs: Any additional args to pass to downloader.get_tabular_rows Returns: @@ -2797,6 +2802,7 @@ def download_generate_resource( datecol=datecol, yearcol=yearcol, date_function=date_function, + no_empty=no_empty, ) def download_and_generate_resource( diff --git a/tests/hdx/data/test_dataset_resource_generation.py b/tests/hdx/data/test_dataset_resource_generation.py index 86041b2..980d4c5 100644 --- a/tests/hdx/data/test_dataset_resource_generation.py +++ b/tests/hdx/data/test_dataset_resource_generation.py @@ -500,6 +500,23 @@ def process_year(row): ) assert success is False url = "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/gen_resource/test_data_no_data.csv" + success, results = dataset.download_generate_resource( + downloader, + url, + folder, + filename, + resourcedata, + ) + assert success is False + success, results = dataset.download_generate_resource( + downloader, + url, + folder, + filename, + resourcedata, + no_empty=False, + ) + assert success is True success, results = dataset.download_generate_resource( downloader, url,