From 1eaa5f99bcfe055b5b089f4c3a4e4a4f778d78fe Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 16:00:40 +1300 Subject: [PATCH 1/7] Remove all usages of package_hxl_update --- src/hdx/data/dataset.py | 34 ++----------------- tests/hdx/api/test_ckan.py | 7 ++-- tests/hdx/data/test_dataset_core.py | 19 ----------- .../hdx/data/test_update_dataset_resources.py | 1 - tests/hdx/data/test_update_logic.py | 3 -- 5 files changed, 4 insertions(+), 60 deletions(-) diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index f1ed03b..249d07c 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -150,7 +150,6 @@ def actions() -> dict[str, str]: "reorder": "package_resource_reorder", "list": "package_list", "autocomplete": "package_autocomplete", - "hxl": "package_hxl_update", "create_default_views": "package_create_default_resource_views", } @@ -392,9 +391,7 @@ def number_of_resources(self) -> int: """ return len(self._resources) - def reorder_resources( - self, resource_ids: Sequence[str], hxl_update: bool = True - ) -> None: + def reorder_resources(self, resource_ids: Sequence[str]) -> None: """Reorder resources in dataset according to provided list. Resources are updated in the dataset object to match new order. However, the dataset is not refreshed by rereading from HDX. If only some resource ids are supplied then @@ -403,7 +400,6 @@ def reorder_resources( Args: resource_ids: List of resource ids - hxl_update: Whether to call package_hxl_update. Defaults to True. Returns: None @@ -421,8 +417,6 @@ def reorder_resources( resource = next(x for x in self._resources if x["id"] == resource_id) reordered_resources.append(resource) self._resources = reordered_resources - if hxl_update: - self.hxl_update() def move_resource( self, @@ -767,7 +761,6 @@ def _revise_dataset( resources_to_delete: Sequence[int], filestore_resources: dict[int, str], new_resource_order: Sequence[str] | None, - hxl_update: bool, create_default_views: bool = False, test: bool = False, **kwargs: Any, @@ -781,7 +774,6 @@ def _revise_dataset( resources_to_delete: List of indexes of resources to delete filestore_resources: List of (index of resources, file to upload) new_resource_order: New resource order to use or None - hxl_update: Whether to call package_hxl_update. create_default_views: Whether to create default views. Defaults to False. test: Whether running in a test. Defaults to False. **kwargs: See below @@ -851,14 +843,10 @@ def _revise_dataset( (x["name"], x["format"].lower()) ), ) - self.reorder_resources( - [x["id"] for x in sorted_resources], hxl_update=False - ) + self.reorder_resources([x["id"] for x in sorted_resources]) if create_default_views: self.create_default_views() self._create_preview_resourceview() - if hxl_update: - self.hxl_update() return results def _dataset_update_resources( @@ -1012,7 +1000,6 @@ def _dataset_hdx_update( remove_additional_resources: bool, match_resource_order: bool, create_default_views: bool, - hxl_update: bool, **kwargs: Any, ) -> tuple[dict, dict]: """Helper method to compare new and existing dataset data, update @@ -1031,7 +1018,6 @@ def _dataset_hdx_update( remove_additional_resources: Remove additional resources found in dataset (if updating) match_resource_order: Match order of given resources by name create_default_views: Whether to call package_create_default_resource_views. - hxl_update: Whether to call package_hxl_update. Returns: Tuple of (resource status codes, revise call info) @@ -1072,7 +1058,6 @@ def _dataset_hdx_update( resources_to_delete, filestore_resources, new_resource_order, - hxl_update, create_default_views=create_default_views, **kwargs, ) @@ -1087,7 +1072,6 @@ def update_in_hdx( remove_additional_resources: bool = False, match_resource_order: bool = False, create_default_views: bool = True, - hxl_update: bool = True, **kwargs: Any, ) -> dict: """Check if dataset exists in HDX and if so, update it. match_resources_by_metadata uses ids if they are @@ -1110,7 +1094,6 @@ def update_in_hdx( remove_additional_resources: Remove additional resources found in dataset. Defaults to False. match_resource_order: Match order of given resources by name. Defaults to False. create_default_views: Whether to call package_create_default_resource_views. Defaults to True. - hxl_update: Whether to call package_hxl_update. Defaults to True. **kwargs: See below keep_crisis_tags (bool): Whether to keep existing crisis tags. Defaults to True. updated_by_script (str): String to identify your script. Defaults to your user agent. @@ -1140,7 +1123,6 @@ def update_in_hdx( remove_additional_resources=remove_additional_resources, match_resource_order=match_resource_order, create_default_views=create_default_views, - hxl_update=hxl_update, **kwargs, ) logger.info(f"Updated {self.get_hdx_url()}") @@ -1155,7 +1137,6 @@ def create_in_hdx( remove_additional_resources: bool = False, match_resource_order: bool = False, create_default_views: bool = True, - hxl_update: bool = True, **kwargs: Any, ) -> dict: """Check if dataset exists in HDX and if so, update it, otherwise create it. match_resources_by_metadata uses @@ -1178,7 +1159,6 @@ def create_in_hdx( remove_additional_resources: Remove additional resources found in dataset (if updating). Defaults to False. match_resource_order: Match order of given resources by name. Defaults to False. create_default_views: Whether to call package_create_default_resource_views (if updating). Defaults to True. - hxl_update: Whether to call package_hxl_update. Defaults to True. **kwargs: See below keep_crisis_tags (bool): Whether to keep existing crisis tags. Defaults to True. updated_by_script (str): String to identify your script. Defaults to your user agent. @@ -1208,7 +1188,6 @@ def create_in_hdx( remove_additional_resources=remove_additional_resources, match_resource_order=match_resource_order, create_default_views=create_default_views, - hxl_update=hxl_update, **kwargs, ) logger.info(f"Updated {self.get_hdx_url()}") @@ -1243,7 +1222,6 @@ def create_in_hdx( [], filestore_resources, None, - hxl_update, **kwargs, ) logger.info(f"Created {self.get_hdx_url()}") @@ -1257,14 +1235,6 @@ def delete_from_hdx(self) -> None: """ self._delete_from_hdx("dataset", "id") - def hxl_update(self) -> None: - """Checks dataset for HXL in resources and updates tags and other metadata to trigger HXL preview. - - Returns: - None - """ - self._read_from_hdx("dataset", self.data["id"], action=self.actions()["hxl"]) - @classmethod def search_in_hdx( cls, diff --git a/tests/hdx/api/test_ckan.py b/tests/hdx/api/test_ckan.py index 6a0e79e..776ad01 100644 --- a/tests/hdx/api/test_ckan.py +++ b/tests/hdx/api/test_ckan.py @@ -168,9 +168,7 @@ def create_resource(): for i in range(10): create_resource() - dataset.create_in_hdx( - hxl_update=False, updated_by_script="hdx_python_api_ignore" - ) + dataset.create_in_hdx(updated_by_script="hdx_python_api_ignore") # check created dataset dataset = Dataset.read_from_hdx(name) @@ -219,7 +217,7 @@ def create_resource(): dataset.add_update_resources(resources) sleep(2) - dataset.update_in_hdx(hxl_update=False, remove_additional_resources=True) + dataset.update_in_hdx(remove_additional_resources=True) sleep(2) # check updated dataset @@ -264,7 +262,6 @@ def create_resource(): sleep(2) dataset.create_in_hdx( - hxl_update=False, remove_additional_resources=True, keys_to_delete=("caveats",), ) diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index 7385aaa..9f1b6ad 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -170,19 +170,6 @@ def mockall(url, datadict): ) -def mockhxlupdate(url, datadict): - if "hxl" not in url: - return MockResponse( - 404, - '{"success": false, "error": {"message": "TEST ERROR: Not HXL Update", "__type": "TEST ERROR: Not HXL Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_hxl_update"}', - ) - result = json.dumps(hxlupdate_list) - return MockResponse( - 200, - f'{{"success": true, "result": {result}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_hxl_update"}}', - ) - - class TestDatasetCore: @pytest.fixture(scope="class") def static_yaml(self, configfolder): @@ -244,8 +231,6 @@ def post(url, data, headers, files, allow_redirects, auth=None): return vocabulary_mockshow(url, datadict) if "show" in url: return dataset_mockshow(url, datadict) - if "hxl" in url: - return mockhxlupdate(url, datadict) if "default" in url: result = json.dumps(resource_view_list) return MockResponse( @@ -331,8 +316,6 @@ def post(url, data, headers, files, allow_redirects, auth=None): return vocabulary_mockshow(url, datadict) if "show" in url: return dataset_mockshow(url, datadict) - if "hxl" in url: - return mockhxlupdate(url, datadict) if "resource" in url: resultdictcopy = copy.deepcopy(resources_data[0]) merge_two_dictionaries(resultdictcopy, datadict) @@ -386,8 +369,6 @@ def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(decodedata) if "show" in url: return dataset_mockshow(url, datadict) - if "hxl" in url: - return mockhxlupdate(url, datadict) if "reorder" not in url: return MockResponse( 404, diff --git a/tests/hdx/data/test_update_dataset_resources.py b/tests/hdx/data/test_update_dataset_resources.py index 4fa1939..6aaed64 100644 --- a/tests/hdx/data/test_update_dataset_resources.py +++ b/tests/hdx/data/test_update_dataset_resources.py @@ -156,7 +156,6 @@ def test_dataset_update_resources( resources_to_delete, filestore_resources, new_resource_order, - hxl_update=False, create_default_views=False, test=True, ) diff --git a/tests/hdx/data/test_update_logic.py b/tests/hdx/data/test_update_logic.py index cbc1637..ae51218 100644 --- a/tests/hdx/data/test_update_logic.py +++ b/tests/hdx/data/test_update_logic.py @@ -185,7 +185,6 @@ def test_update_logic_1( remove_additional_resources=True, match_resource_order=False, create_default_views=False, - hxl_update=False, test=True, ) assert statuses == { @@ -909,7 +908,6 @@ def test_update_logic_2( remove_additional_resources=True, match_resource_order=False, create_default_views=False, - hxl_update=False, test=True, ) assert statuses == { @@ -1633,7 +1631,6 @@ def test_update_logic_3( remove_additional_resources=True, match_resource_order=False, create_default_views=False, - hxl_update=False, test=True, ) assert statuses == { From cd17aea077d917413c40ff474da8c87a5e4cc332 Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 16:33:27 +1300 Subject: [PATCH 2/7] QuickCharts noop --- src/hdx/data/dataset.py | 347 +----------------- tests/hdx/data/test_dataset_noncore.py | 158 +------- .../data/test_dataset_resource_generation.py | 110 ------ 3 files changed, 27 insertions(+), 588 deletions(-) diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 249d07c..3d4c56a 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -7,7 +7,6 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from copy import deepcopy from datetime import datetime -from os.path import isfile from pathlib import Path from typing import ( TYPE_CHECKING, @@ -29,14 +28,12 @@ from hdx.utilities.dictandlist import merge_two_dictionaries from hdx.utilities.downloader import Download from hdx.utilities.loader import load_json -from hdx.utilities.path import script_dir_plus_file from hdx.utilities.saver import save_iterable, save_json from hdx.utilities.uuid import is_valid_uuid from hxl.input import HXLIOException, InputOptions, munge_url import hdx.data.organization as org_module import hdx.data.resource as res_module -import hdx.data.resource_view as resource_view import hdx.data.showcase as sc_module import hdx.data.user as user import hdx.data.vocabulary as vocabulary @@ -391,7 +388,9 @@ def number_of_resources(self) -> int: """ return len(self._resources) - def reorder_resources(self, resource_ids: Sequence[str]) -> None: + def reorder_resources( + self, resource_ids: Sequence[str], hxl_update: bool = True + ) -> None: """Reorder resources in dataset according to provided list. Resources are updated in the dataset object to match new order. However, the dataset is not refreshed by rereading from HDX. If only some resource ids are supplied then @@ -1072,6 +1071,7 @@ def update_in_hdx( remove_additional_resources: bool = False, match_resource_order: bool = False, create_default_views: bool = True, + hxl_update: bool = True, **kwargs: Any, ) -> dict: """Check if dataset exists in HDX and if so, update it. match_resources_by_metadata uses ids if they are @@ -1137,6 +1137,7 @@ def create_in_hdx( remove_additional_resources: bool = False, match_resource_order: bool = False, create_default_views: bool = True, + hxl_update: bool = True, **kwargs: Any, ) -> dict: """Check if dataset exists in HDX and if so, update it, otherwise create it. match_resources_by_metadata uses @@ -2219,10 +2220,10 @@ def preview_resource(self) -> None: """ self.data["dataset_preview"] = "resource_id" - def set_quickchart_resource( + def set_preview_resource( self, resource: Union["Resource", dict, str, int] ) -> "Resource": - """Set the resource that will be used for displaying QuickCharts in dataset preview + """Set the resource that will be used for displaying previews in dataset preview Args: resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position @@ -2256,17 +2257,14 @@ def set_quickchart_resource( dataset_resource.disable_dataset_preview() return preview_resource - def quickcharts_resource_last(self) -> bool: - """Move the QuickCharts resource to be last. Assumes that it's name begins 'QuickCharts-'. - - Returns: - True if QuickCharts resource found, False if not - """ - for i, resource in enumerate(self._resources): - if resource["name"][:12] == "QuickCharts-": - self._resources.append(self._resources.pop(i)) - return True - return False + def set_quickcharts_resource( + self, resource: Union["Resource", dict, str, int] + ) -> "Resource": + warnings.warn( + "set_quickcharts_resource() is deprecated, use set_preview_resource() instead", + DeprecationWarning, + ) + return self.set_preview_resource(resource) def create_default_views(self, create_datastore_views: bool = False) -> None: """Create default resource views for all resources in dataset @@ -2298,164 +2296,6 @@ def _create_preview_resourceview(self) -> None: self._preview_resourceview = None break - def _generate_resource_view( - self, - resource: Union["Resource", dict, str, int] = 0, - path: Path | str | None = None, - bites_disabled: Sequence[bool] | None = None, - indicators: Sequence[dict] | None = None, - findreplace: dict | None = None, - ) -> resource_view.ResourceView | None: - """Create QuickCharts for the given resource in a dataset. If you do - not supply a path, then the internal indicators resource view template - will be used. You can disable specific bites by providing - bites_disabled, a list of 3 bools where True indicates a specific bite - is disabled and False indicates leave enabled. The parameter indicators - is a list with 3 dictionaries of form: - {"code": "MY_INDICATOR_CODE", "title": "MY_INDICATOR_TITLE", - "unit": "MY_INDICATOR_UNIT"}. Optionally, the following defaults can be - overridden in the parameter indicators: {"code_col": "#indicator+code", - "value_col": "#indicator+value+num", "date_col": "#date+year", - "date_format": "%Y", "aggregate_col": "null"}. - - Creation of the resource view will be delayed until after the next - dataset create or update if a resource id is not yet available. - - Args: - resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position. Defaults to 0. - path: Path to YAML resource view metadata. Defaults to None (config/hdx_resource_view_static.yaml or internal template). - bites_disabled: Which QC bites should be disabled. Defaults to None (all bites enabled). - indicators: Indicator codes, QC titles and units for resource view template. Defaults to None (don't use template). - findreplace: Replacements for anything else in resource view. Defaults to None. - - Returns: - The resource view if QuickCharts created, None is not - """ - if not bites_disabled: - bites_disabled = [False, False, False] - elif all(i for i in bites_disabled): - return None - res = self.set_quickchart_resource(resource) - if res is None: - return None - if "id" in res: - resourceview_data = {"resource_id": res["id"]} - else: - resourceview_data = {"resource_name": res["name"]} - resourceview = resource_view.ResourceView(resourceview_data) - if path is None: - if indicators is None: - path = Path("config", "hdx_resource_view_static.yaml") - if not isfile(path): - path = path.with_suffix(".yml") - else: - path = script_dir_plus_file( - "indicator_resource_view_template.yaml", - NotRequestableError, - ) - resourceview.update_from_yaml(path=path) - - def replace_string(instr, what, withwhat): - return instr.replace(what, str(withwhat)) - - defaults = { - "code_col": "#indicator+code", - "value_col": "#indicator+value+num", - "date_col": "#date+year", - "date_format": "%Y", - "aggregate_col": "null", - } - - def replace_col(hxl_preview_cfg, col_str, ind, col, with_quotes=False): - if with_quotes: - col_str = f'"{col_str}"' - replace = ind.get(col) - if replace: - if with_quotes: - replace = f'"{replace}"' - else: - replace = defaults[col] - return replace_string(hxl_preview_cfg, col_str, replace) - - hxl_preview_config = resourceview["hxl_preview_config"] - if indicators is None: - indicators_notexist = [False, False, False] - else: - len_indicators = len(indicators) - if len_indicators == 0: - return None - indicators_notexist = [True, True, True] - - def replace_indicator(qc_config, index): - indicator = indicators[index] - ind_str = str(index + 1) - qc_config = replace_string( - qc_config, f"CODE_VALUE_{ind_str}", str(indicator["code"]) - ) - replace = indicator.get("description", "") - qc_config = replace_string( - qc_config, f"DESCRIPTION_VALUE_{ind_str}", replace - ) - qc_config = replace_string( - qc_config, f"TITLE_VALUE_{ind_str}", indicator["title"] - ) - replace = indicator.get("unit", "") - qc_config = replace_string(qc_config, f"UNIT_VALUE_{ind_str}", replace) - qc_config = replace_col( - qc_config, f"CODE_COL_{ind_str}", indicator, "code_col" - ) - qc_config = replace_col( - qc_config, f"VALUE_COL_{ind_str}", indicator, "value_col" - ) - qc_config = replace_col( - qc_config, f"DATE_COL_{ind_str}", indicator, "date_col" - ) - qc_config = replace_col( - qc_config, - f"DATE_FORMAT_{ind_str}", - indicator, - "date_format", - ) - qc_config = replace_col( - qc_config, - f"AGGREGATE_COL_{ind_str}", - indicator, - "aggregate_col", - True, - ) - indicators_notexist[index] = False - return qc_config - - if indicators[0]: - hxl_preview_config = replace_indicator(hxl_preview_config, 0) - if len_indicators > 1 and indicators[1]: - hxl_preview_config = replace_indicator(hxl_preview_config, 1) - if len_indicators > 2 and indicators[2]: - hxl_preview_config = replace_indicator(hxl_preview_config, 2) - if indicators_notexist == [True, True, True]: - return None - hxl_preview_config = json.loads(hxl_preview_config) - bites = hxl_preview_config["bites"] - for i, disable in reversed(list(enumerate(bites_disabled))): - if disable or indicators_notexist[i]: - del bites[i] - hxl_preview_config = json.dumps( - hxl_preview_config, indent=None, separators=(",", ":") - ) - if findreplace: - for find in findreplace: - hxl_preview_config = replace_string( - hxl_preview_config, find, findreplace[find] - ) - resourceview["hxl_preview_config"] = hxl_preview_config - - if "resource_id" in resourceview: - resourceview.create_in_hdx() - self._preview_resourceview = None - else: - self._preview_resourceview = resourceview - return resourceview - def generate_quickcharts( self, resource: Union["Resource", dict, str, int] = 0, @@ -2463,22 +2303,8 @@ def generate_quickcharts( bites_disabled: Sequence[bool] | None = None, indicators: Sequence[dict] | None = None, findreplace: dict | None = None, - ) -> resource_view.ResourceView: - """Create QuickCharts for the given resource in a dataset. If you do - not supply a path, then the internal indicators resource view template - will be used. You can disable specific bites by providing - bites_disabled, a list of 3 bools where True indicates a specific bite - is disabled and False indicates leave enabled. The parameter indicators - is a list with 3 dictionaries of form: - {"code": "MY_INDICATOR_CODE", "title": "MY_INDICATOR_TITLE", - "unit": "MY_INDICATOR_UNIT"}. Optionally, the following defaults can be - overridden in the parameter indicators: {"code_col": "#indicator+code", - "value_col": "#indicator+value+num", "date_col": "#date+year", - "date_format": "%Y", "aggregate_col": "null"}. - - Creation of the resource view will be delayed until after the next - dataset create or update if a resource id is not yet available and will - be disabled if there are no valid charts to display. + ) -> None: + """To be removed Args: resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position. Defaults to 0. @@ -2488,14 +2314,9 @@ def generate_quickcharts( findreplace: Replacements for anything else in resource view. Defaults to None. Returns: - The resource view if QuickCharts created, None is not + None """ - resourceview = self._generate_resource_view( - resource, path, bites_disabled, indicators, findreplace=findreplace - ) - if resourceview is None: - self.preview_off() - return resourceview + return None def get_name_or_id(self, prefer_name: bool = True) -> str | None: """Get dataset name or id eg. for use in urls. If prefer_name is True, @@ -2772,25 +2593,6 @@ def generate_resource_from_iterable( the time period and are returned in the results dictionary in keys startdate and enddate. - If the parameter quickcharts is supplied then various QuickCharts - related actions will occur depending upon the keys given in the - dictionary and the returned dictionary will contain the QuickCharts - resource in the key qc_resource. If the keys: hashtag - the HXL hashtag - to examine - and values - the 3 values to look for in that column - are - supplied, then a list of booleans indicating which QuickCharts bites - should be enabled will be returned in the key bites_disabled in the - returned dictionary. For the 3 values, if the key: numeric_hashtag is - supplied then if that column for a given value contains no numbers, - then the corresponding bite will be disabled. If the key: cutdown is - given, if it is 1, then a separate cut down list is created containing - only columns with HXL hashtags and rows with desired values (if hashtag - and values are supplied) for the purpose of driving QuickCharts. It is - returned in the key qcrows in the returned dictionary with the matching - headers in qcheaders. If cutdown is 2, then a resource is created using - the cut down list. If the key cutdownhashtags is supplied, then only - the provided hashtags are used for cutting down otherwise the full list - of HXL tags is used. - Args: headers: Headers iterable: Iterable returning rows @@ -2801,7 +2603,6 @@ def generate_resource_from_iterable( datecol: Date column for setting time period. Defaults to None (don't set). yearcol: Year column for setting dataset year range. Defaults to None (don't set). date_function: Date function to call for each row. Defaults to None. - quickcharts: Dictionary containing optional keys: hashtag, values, cutdown and/or cutdownhashtags encoding: Encoding to use. Defaults to None (infer encoding). Returns: @@ -2818,40 +2619,6 @@ def generate_resource_from_iterable( return False, retdict rows = [Download.hxl_row(headers, hxltags, dict_form=True)] dates = [default_enddate, default_date] - qc = {"cutdown": 0} - bites_disabled = [True, True, True] - if quickcharts is not None: - hashtag = quickcharts.get("hashtag") - if hashtag: - qc["column"] = next( - key for key, value in hxltags.items() if value == hashtag - ) # reverse dict lookup - else: - qc["column"] = None - numeric_hashtag = quickcharts.get("numeric_hashtag") - if numeric_hashtag: - qc["numeric"] = next( - key for key, value in hxltags.items() if value == numeric_hashtag - ) # reverse lookup - else: - qc["numeric"] = None - cutdown = quickcharts.get("cutdown", 0) - if cutdown: - qc["cutdown"] = cutdown - cutdownhashtags = quickcharts.get("cutdownhashtags") - if cutdownhashtags is None: - cutdownhashtags = list(hxltags.keys()) - else: - cutdownhashtags = [ - key - for key, value in hxltags.items() - if value in cutdownhashtags - ] - if numeric_hashtag and qc["numeric"] not in cutdownhashtags: - cutdownhashtags.append(qc["numeric"]) - qc["headers"] = [x for x in headers if x in cutdownhashtags] - qc["rows"] = [Download.hxl_row(qc["headers"], hxltags, dict_form=True)] - if yearcol is not None: def yearcol_function(row): @@ -2893,24 +2660,6 @@ def datecol_function(row): if enddate > dates[1]: dates[1] = enddate rows.append(row) - if quickcharts is not None: - if qc["column"] is None: - if qc["cutdown"]: - qcrow = {x: row[x] for x in qc["headers"]} - qc["rows"].append(qcrow) - else: - value = row[qc["column"]] - for i, lookup in enumerate(quickcharts["values"]): - if value == lookup: - if qc["numeric"]: - try: - float(row[qc["numeric"]]) - except (TypeError, ValueError): - continue - bites_disabled[i] = False - if qc["cutdown"]: - qcrow = {x: row[x] for x in qc["headers"]} - qc["rows"].append(qcrow) if len(rows) == 1: logger.error(f"No data rows in {filename}!") return False, retdict @@ -2933,25 +2682,6 @@ def datecol_function(row): retdict["resource"] = resource retdict["headers"] = headers retdict["rows"] = rows - if quickcharts is not None: - retdict["bites_disabled"] = bites_disabled - if qc["cutdown"]: - retdict["qcheaders"] = qc["headers"] - retdict["qcrows"] = qc["rows"] - if qc["cutdown"] == 2: - qc_resourcedata = { - "name": f"QuickCharts-{resourcedata['name']}", - "description": "Cut down data for QuickCharts", - } - resource = self.generate_resource_from_rows( - folder, - f"qc_{filename}", - qc["rows"], - qc_resourcedata, - headers=qc["headers"], - encoding=encoding, - ) - retdict["qc_resource"] = resource return True, retdict def generate_resource_from_iterator( @@ -3027,25 +2757,6 @@ def download_generate_resource( the time period and are returned in the results dictionary in keys startdate and enddate. - If the parameter quickcharts is supplied then various QuickCharts - related actions will occur depending upon the keys given in the - dictionary and the returned dictionary will contain the QuickCharts - resource in the key qc_resource. If the keys: hashtag - the HXL hashtag - to examine - and values - the 3 values to look for in that column - are - supplied, then a list of booleans indicating which QuickCharts bites - should be enabled will be returned in the key bites_disabled in the - returned dictionary. For the 3 values, if the key: numeric_hashtag is - supplied then if that column for a given value contains no numbers, - then the corresponding bite will be disabled. If the key: cutdown is - given, if it is 1, then a separate cut down list is created containing - only columns with HXL hashtags and rows with desired values (if hashtag - and values are supplied) for the purpose of driving QuickCharts. It is - returned in the key qcrows in the returned dictionary with the matching - headers in qcheaders. If cutdown is 2, then a resource is created using - the cut down list. If the key cutdownhashtags is supplied, then only - the provided hashtags are used for cutting down otherwise the full list - of HXL tags is used. - Args: downloader: A Download or Retrieve object url: URL to download @@ -3128,25 +2839,6 @@ def download_and_generate_resource( the time period and are returned in the results dictionary in keys startdate and enddate. - If the parameter quickcharts is supplied then various QuickCharts - related actions will occur depending upon the keys given in the - dictionary and the returned dictionary will contain the QuickCharts - resource in the key qc_resource. If the keys: hashtag - the HXL hashtag - to examine - and values - the 3 values to look for in that column - are - supplied, then a list of booleans indicating which QuickCharts bites - should be enabled will be returned in the key bites_disabled in the - returned dictionary. For the 3 values, if the key: numeric_hashtag is - supplied then if that column for a given value contains no numbers, - then the corresponding bite will be disabled. If the key: cutdown is - given, if it is 1, then a separate cut down list is created containing - only columns with HXL hashtags and rows with desired values (if hashtag - and values are supplied) for the purpose of driving QuickCharts. It is - returned in the key qcrows in the returned dictionary with the matching - headers in qcheaders. If cutdown is 2, then a resource is created using - the cut down list. If the key cutdownhashtags is supplied, then only - the provided hashtags are used for cutting down otherwise the full list - of HXL tags is used. - Args: downloader: A Download or Retrieve object url: URL to download @@ -3159,7 +2851,6 @@ def download_and_generate_resource( datecol: Date column for setting time period. Defaults to None (don't set). yearcol: Year column for setting dataset year range. Defaults to None (don't set). date_function: Date function to call for each row. Defaults to None. - quickcharts: Dictionary containing optional keys: hashtag, values, cutdown and/or cutdownhashtags **kwargs: Any additional args to pass to downloader.get_tabular_rows Returns: diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index a06c506..ceba138 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -717,9 +717,7 @@ def test_set_quickchart_resource(self, configuration): dataset = Dataset(datasetdata) assert "dataset_preview" not in dataset assert ( - dataset.set_quickchart_resource("3d777226-96aa-4239-860a-703389d16d1f")[ - "id" - ] + dataset.set_preview_resource("3d777226-96aa-4239-860a-703389d16d1f")["id"] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert dataset["dataset_preview"] == "resource_id" @@ -727,185 +725,45 @@ def test_set_quickchart_resource(self, configuration): assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "True" assert ( - dataset.set_quickchart_resource(resources[0])["id"] + dataset.set_preview_resource(resources[0])["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) assert resources[0]["dataset_preview_enabled"] == "True" assert resources[1]["dataset_preview_enabled"] == "False" assert ( - dataset.set_quickchart_resource(resources[1].data)["id"] + dataset.set_preview_resource(resources[1].data)["id"] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "True" assert ( - dataset.set_quickchart_resource(0)["id"] + dataset.set_preview_resource(0)["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) assert resources[0]["dataset_preview_enabled"] == "True" assert resources[1]["dataset_preview_enabled"] == "False" - assert dataset.set_quickchart_resource("12345") is None + assert dataset.set_preview_resource("12345") is None with pytest.raises(HDXError): - dataset.set_quickchart_resource(True) + dataset.set_preview_resource(True) dataset.preview_off() assert dataset["dataset_preview"] == "no_preview" assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "False" assert ( - dataset.set_quickchart_resource("Resource2")["id"] + dataset.set_preview_resource("Resource2")["id"] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert dataset["dataset_preview"] == "resource_id" assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "True" assert ( - dataset.set_quickchart_resource({"name": "Resource1"})["id"] + dataset.set_preview_resource({"name": "Resource1"})["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) assert dataset["dataset_preview"] == "resource_id" assert resources[0]["dataset_preview_enabled"] == "True" assert resources[1]["dataset_preview_enabled"] == "False" - def test_quickcharts_resource_last(self, configuration): - datasetdata = copy.deepcopy(dataset_data) - resourcesdata = copy.deepcopy(resources_data) - datasetdata["resources"] = resourcesdata - dataset = Dataset(datasetdata) - assert dataset.quickcharts_resource_last() is False - resource = {"name": "QuickCharts-resource"} - dataset._resources.insert(1, resource) - assert dataset.quickcharts_resource_last() is True - assert dataset._resources[3]["name"] == resource["name"] - assert dataset.quickcharts_resource_last() is True - - def test_generate_resource_view( - self, configuration, vocabulary_update, static_resource_view_yaml - ): - datasetdata = copy.deepcopy(dataset_data) - resourcesdata = copy.deepcopy(resources_data) - datasetdata["resources"] = resourcesdata - dataset = Dataset(datasetdata) - assert "dataset_preview" not in dataset - resourceview = dataset.generate_quickcharts(path=static_resource_view_yaml) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert hxl_preview_config["bites"][0]["title"] == "Sum of fatalities" - assert ( - hxl_preview_config["bites"][1]["title"] - == "Sum of fatalities grouped by admin1" - ) - assert ( - hxl_preview_config["bites"][2]["title"] - == "Sum of fatalities grouped by admin2" - ) - resourceview = dataset.generate_quickcharts( - path=static_resource_view_yaml, bites_disabled=[False, True, False] - ) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert hxl_preview_config["bites"][0]["title"] == "Sum of fatalities" - assert ( - hxl_preview_config["bites"][1]["title"] - == "Sum of fatalities grouped by admin2" - ) - resourceview = dataset.generate_quickcharts( - path=static_resource_view_yaml, bites_disabled=[True, True, True] - ) - assert resourceview is None - indicators = [ - { - "code": "1", - "title": "My1", - "unit": "ones", - "description": "This is my one!", - }, - { - "code": "2", - "title": "My2", - "unit": "twos", - "aggregate_col": "Agg2", - }, - { - "code": "3", - "title": "My3", - "description": "This is my three!", - "date_col": "dt3", - "date_format": "%b %Y", - }, - ] - resourceview = dataset.generate_quickcharts(indicators=indicators) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert ( - hxl_preview_config["bites"][0]["ingredient"]["filters"]["filterWith"][0][ - "#indicator+code" - ] - == "1" - ) - assert ( - hxl_preview_config["bites"][0]["ingredient"]["description"] - == "This is my one!" - ) - assert hxl_preview_config["bites"][0]["uiProperties"]["title"] == "My1" - assert ( - hxl_preview_config["bites"][0]["computedProperties"]["dataTitle"] == "ones" - ) - assert ( - hxl_preview_config["bites"][1]["ingredient"]["filters"]["filterWith"][0][ - "#indicator+code" - ] - == "2" - ) - assert hxl_preview_config["bites"][1]["ingredient"]["description"] == "" - assert hxl_preview_config["bites"][1]["uiProperties"]["title"] == "My2" - assert ( - hxl_preview_config["bites"][1]["computedProperties"]["dataTitle"] == "twos" - ) - assert hxl_preview_config["bites"][1]["ingredient"]["aggregateColumn"] == "Agg2" - assert ( - hxl_preview_config["bites"][2]["ingredient"]["filters"]["filterWith"][0][ - "#indicator+code" - ] - == "3" - ) - assert ( - hxl_preview_config["bites"][2]["ingredient"]["description"] - == "This is my three!" - ) - assert hxl_preview_config["bites"][2]["ingredient"]["dateColumn"] == "dt3" - assert hxl_preview_config["bites"][2]["uiProperties"]["title"] == "My3" - assert hxl_preview_config["bites"][2]["computedProperties"]["dataTitle"] == "" - assert hxl_preview_config["bites"][2]["uiProperties"]["dateFormat"] == "%b %Y" - resourceview = dataset.generate_quickcharts( - indicators=indicators, - findreplace={ - "#indicator+code": "#item+code", - "#indicator+value+num": "#value", - }, - ) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert ( - hxl_preview_config["bites"][0]["ingredient"]["filters"]["filterWith"][0][ - "#item+code" - ] - == "1" - ) - assert hxl_preview_config["bites"][0]["ingredient"]["valueColumn"] == "#value" - assert dataset.generate_quickcharts(indicators=[]) is None - assert dataset.generate_quickcharts(indicators=[None, None, None]) is None - assert ( - dataset.generate_quickcharts(resource="123", path=static_resource_view_yaml) - is None - ) - del dataset.get_resources()[0]["id"] - resourceview = dataset.generate_quickcharts(path=static_resource_view_yaml) - assert "id" not in resourceview - assert "resource_id" not in resourceview - assert resourceview["resource_name"] == "Resource1" - with pytest.raises(IOError): - dataset.generate_quickcharts() - def test_remove_dates_from_title(self, configuration): dataset = Dataset() with pytest.raises(HDXError): diff --git a/tests/hdx/data/test_dataset_resource_generation.py b/tests/hdx/data/test_dataset_resource_generation.py index 023bb98..86041b2 100644 --- a/tests/hdx/data/test_dataset_resource_generation.py +++ b/tests/hdx/data/test_dataset_resource_generation.py @@ -562,7 +562,6 @@ def process_row(headers, row): assert results == { "startdate": datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc), "enddate": datetime(2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc), - "bites_disabled": [False, True, False], "resource": { "description": "Conflict data with HXL tags", "format": "csv", @@ -738,85 +737,6 @@ def process_row(headers, row): "lala": "lala", }, ], - "qc_resource": { - "description": "Cut down data for QuickCharts", - "format": "csv", - "name": "QuickCharts-Conflict Data for Algeria", - }, - "qcheaders": [ - "EVENT_ID_CNTY", - "EVENT_DATE", - "YEAR", - "EVENT_TYPE", - "ACTOR1", - "ACTOR2", - "COUNTRY", - "ADMIN1", - "ADMIN2", - "ADMIN3", - "LOCATION", - "LATITUDE", - "LONGITUDE", - "SOURCE", - "NOTES", - "FATALITIES", - ], - "qcrows": [ - { - "EVENT_ID_CNTY": "#event+code", - "EVENT_DATE": "#date+occurred", - "YEAR": "#date+year", - "EVENT_TYPE": "#event+type", - "ACTOR1": "#group+name+first", - "ACTOR2": "#group+name+second", - "COUNTRY": "#country+name", - "ADMIN1": "#adm1+name", - "ADMIN2": "#adm2+name", - "ADMIN3": "#adm3+name", - "LOCATION": "#loc+name", - "LATITUDE": "#geo+lat", - "LONGITUDE": "#geo+lon", - "SOURCE": "#meta+source", - "NOTES": "#description", - "FATALITIES": "#affected+killed", - }, - { - "EVENT_ID_CNTY": "1416RTA", - "EVENT_DATE": "18/04/2001", - "YEAR": "2001", - "EVENT_TYPE": "Violence against civilians", - "ACTOR1": "Police Forces of Algeria (1999-)", - "ACTOR2": "Civilians (Algeria)", - "COUNTRY": "Algeria", - "ADMIN1": "Tizi Ouzou", - "ADMIN2": "Beni-Douala", - "ADMIN3": None, - "LOCATION": "Beni Douala", - "LATITUDE": "36.61954", - "LONGITUDE": "4.08282", - "SOURCE": "Associated Press Online", - "NOTES": "A Berber student was shot while in police custody at a police station in Beni Douala. He later died on Apr.21.", - "FATALITIES": "1", - }, - { - "EVENT_ID_CNTY": "2231RTA", - "EVENT_DATE": "21/04/2001", - "YEAR": "2001", - "EVENT_TYPE": "Riots/Protests", - "ACTOR1": "Rioters (Algeria)", - "ACTOR2": "Police Forces of Algeria (1999-)", - "COUNTRY": "Algeria", - "ADMIN1": "Bejaia", - "ADMIN2": "Amizour", - "ADMIN3": None, - "LOCATION": "Amizour", - "LATITUDE": "36.64022", - "LONGITUDE": "4.90131", - "SOURCE": "Kabylie report", - "NOTES": "Rioters threw molotov cocktails, rocks and burning tires at gendarmerie stations in Beni Douala, El-Kseur and Amizour.", - "FATALITIES": "0", - }, - ], } assert ( dataset["dataset_date"] @@ -830,22 +750,11 @@ def process_row(headers, row): "description": "Conflict data with HXL tags", "format": "csv", }, - { - "name": "QuickCharts-Conflict Data for Algeria", - "description": "Cut down data for QuickCharts", - "format": "csv", - }, ] assert_files_same( fixturesfolder / "download_gen_resource" / filename, folder / filename, ) - qc_filename = f"qc_{filename}" - assert_files_same( - fixturesfolder / "gen_resource" / qc_filename, - folder / qc_filename, - ) - success, results = dataset.download_and_generate_resource( downloader, TestDatasetResourceGeneration.url, @@ -887,7 +796,6 @@ def process_row(headers, row): assert results == { "startdate": datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc), "enddate": datetime(2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc), - "bites_disabled": [False, True, False], "resource": { "description": "Conflict data with HXL tags", "format": "csv", @@ -1063,20 +971,6 @@ def process_row(headers, row): "lala": "lala", }, ], - "qc_resource": { - "description": "Cut down data for QuickCharts", - "format": "csv", - "name": "QuickCharts-Conflict Data for Algeria", - }, - "qcheaders": ["EVENT_ID_CNTY", "FATALITIES"], - "qcrows": [ - { - "EVENT_ID_CNTY": "#event+code", - "FATALITIES": "#affected+killed", - }, - {"EVENT_ID_CNTY": "1416RTA", "FATALITIES": "1"}, - {"EVENT_ID_CNTY": "2231RTA", "FATALITIES": "0"}, - ], } def process_year(row): @@ -1113,10 +1007,6 @@ def process_year(row): dataset["dataset_date"] == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" ) - assert_files_same( - fixturesfolder / "gen_resource" / f"min_{qc_filename}", - folder / qc_filename, - ) with pytest.raises(HDXError): dataset.download_and_generate_resource( From 053526e2f9a5a8cd3534d01a59d9c8d1d966b3be Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 17:43:36 +1300 Subject: [PATCH 3/7] Fix name --- src/hdx/data/dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 3d4c56a..d861662 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -2257,11 +2257,11 @@ def set_preview_resource( dataset_resource.disable_dataset_preview() return preview_resource - def set_quickcharts_resource( + def set_quickchart_resource( self, resource: Union["Resource", dict, str, int] ) -> "Resource": warnings.warn( - "set_quickcharts_resource() is deprecated, use set_preview_resource() instead", + "set_quickchart_resource() is deprecated, use set_preview_resource() instead", DeprecationWarning, ) return self.set_preview_resource(resource) From ccf0037d1371eb7c4f6ef7ec5c6761975af819f1 Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 17:48:19 +1300 Subject: [PATCH 4/7] Fix infer arguments --- src/hdx/facades/infer_arguments.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index 5bdc47a..dacf83c 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -71,6 +71,9 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): case "str" | "Path" | "Path | str": param_type = "str | None" default = None + case "Optional[bool]" | "bool | None": + param_type = "bool | None" + default = None case "bool": default = False case _: From 33fefb8358550e554a425f32994521829b0c1821 Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 17:51:28 +1300 Subject: [PATCH 5/7] Add missing import --- src/hdx/facades/infer_arguments.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index dacf83c..24b714a 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -5,7 +5,7 @@ from collections.abc import Callable from inspect import getdoc from pathlib import Path -from typing import Any +from typing import Any, Optional # noqa: F401 import defopt from hdx.utilities.easy_logging import setup_logging @@ -71,9 +71,6 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): case "str" | "Path" | "Path | str": param_type = "str | None" default = None - case "Optional[bool]" | "bool | None": - param_type = "bool | None" - default = None case "bool": default = False case _: From 5e8e4e70643d86a90a9b517d8064abf9bfe70afb Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 17:53:52 +1300 Subject: [PATCH 6/7] Probably should handle Optional[bool] --- src/hdx/facades/infer_arguments.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index 24b714a..fa24908 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -71,6 +71,9 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): case "str" | "Path" | "Path | str": param_type = "str | None" default = None + case "Optional[bool]" | "bool | None": + param_type = "bool | None" + default = None case "bool": default = False case _: From 9d940dd5700929278391391d6a6011dfddea1779 Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Jan 2026 18:03:31 +1300 Subject: [PATCH 7/7] Actually don't need to handle Optional[bool] here as not one of Configuration's parameters --- src/hdx/facades/infer_arguments.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index fa24908..24b714a 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -71,9 +71,6 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): case "str" | "Path" | "Path | str": param_type = "str | None" default = None - case "Optional[bool]" | "bool | None": - param_type = "bool | None" - default = None case "bool": default = False case _: