diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index f1ed03b..d861662 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -7,7 +7,6 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from copy import deepcopy from datetime import datetime -from os.path import isfile from pathlib import Path from typing import ( TYPE_CHECKING, @@ -29,14 +28,12 @@ from hdx.utilities.dictandlist import merge_two_dictionaries from hdx.utilities.downloader import Download from hdx.utilities.loader import load_json -from hdx.utilities.path import script_dir_plus_file from hdx.utilities.saver import save_iterable, save_json from hdx.utilities.uuid import is_valid_uuid from hxl.input import HXLIOException, InputOptions, munge_url import hdx.data.organization as org_module import hdx.data.resource as res_module -import hdx.data.resource_view as resource_view import hdx.data.showcase as sc_module import hdx.data.user as user import hdx.data.vocabulary as vocabulary @@ -150,7 +147,6 @@ def actions() -> dict[str, str]: "reorder": "package_resource_reorder", "list": "package_list", "autocomplete": "package_autocomplete", - "hxl": "package_hxl_update", "create_default_views": "package_create_default_resource_views", } @@ -403,7 +399,6 @@ def reorder_resources( Args: resource_ids: List of resource ids - hxl_update: Whether to call package_hxl_update. Defaults to True. Returns: None @@ -421,8 +416,6 @@ def reorder_resources( resource = next(x for x in self._resources if x["id"] == resource_id) reordered_resources.append(resource) self._resources = reordered_resources - if hxl_update: - self.hxl_update() def move_resource( self, @@ -767,7 +760,6 @@ def _revise_dataset( resources_to_delete: Sequence[int], filestore_resources: dict[int, str], new_resource_order: Sequence[str] | None, - hxl_update: bool, create_default_views: bool = False, test: bool = False, **kwargs: Any, @@ -781,7 +773,6 @@ def _revise_dataset( resources_to_delete: List of indexes of resources to delete filestore_resources: List of (index of resources, file to upload) new_resource_order: New resource order to use or None - hxl_update: Whether to call package_hxl_update. create_default_views: Whether to create default views. Defaults to False. test: Whether running in a test. Defaults to False. **kwargs: See below @@ -851,14 +842,10 @@ def _revise_dataset( (x["name"], x["format"].lower()) ), ) - self.reorder_resources( - [x["id"] for x in sorted_resources], hxl_update=False - ) + self.reorder_resources([x["id"] for x in sorted_resources]) if create_default_views: self.create_default_views() self._create_preview_resourceview() - if hxl_update: - self.hxl_update() return results def _dataset_update_resources( @@ -1012,7 +999,6 @@ def _dataset_hdx_update( remove_additional_resources: bool, match_resource_order: bool, create_default_views: bool, - hxl_update: bool, **kwargs: Any, ) -> tuple[dict, dict]: """Helper method to compare new and existing dataset data, update @@ -1031,7 +1017,6 @@ def _dataset_hdx_update( remove_additional_resources: Remove additional resources found in dataset (if updating) match_resource_order: Match order of given resources by name create_default_views: Whether to call package_create_default_resource_views. - hxl_update: Whether to call package_hxl_update. Returns: Tuple of (resource status codes, revise call info) @@ -1072,7 +1057,6 @@ def _dataset_hdx_update( resources_to_delete, filestore_resources, new_resource_order, - hxl_update, create_default_views=create_default_views, **kwargs, ) @@ -1110,7 +1094,6 @@ def update_in_hdx( remove_additional_resources: Remove additional resources found in dataset. Defaults to False. match_resource_order: Match order of given resources by name. Defaults to False. create_default_views: Whether to call package_create_default_resource_views. Defaults to True. - hxl_update: Whether to call package_hxl_update. Defaults to True. **kwargs: See below keep_crisis_tags (bool): Whether to keep existing crisis tags. Defaults to True. updated_by_script (str): String to identify your script. Defaults to your user agent. @@ -1140,7 +1123,6 @@ def update_in_hdx( remove_additional_resources=remove_additional_resources, match_resource_order=match_resource_order, create_default_views=create_default_views, - hxl_update=hxl_update, **kwargs, ) logger.info(f"Updated {self.get_hdx_url()}") @@ -1178,7 +1160,6 @@ def create_in_hdx( remove_additional_resources: Remove additional resources found in dataset (if updating). Defaults to False. match_resource_order: Match order of given resources by name. Defaults to False. create_default_views: Whether to call package_create_default_resource_views (if updating). Defaults to True. - hxl_update: Whether to call package_hxl_update. Defaults to True. **kwargs: See below keep_crisis_tags (bool): Whether to keep existing crisis tags. Defaults to True. updated_by_script (str): String to identify your script. Defaults to your user agent. @@ -1208,7 +1189,6 @@ def create_in_hdx( remove_additional_resources=remove_additional_resources, match_resource_order=match_resource_order, create_default_views=create_default_views, - hxl_update=hxl_update, **kwargs, ) logger.info(f"Updated {self.get_hdx_url()}") @@ -1243,7 +1223,6 @@ def create_in_hdx( [], filestore_resources, None, - hxl_update, **kwargs, ) logger.info(f"Created {self.get_hdx_url()}") @@ -1257,14 +1236,6 @@ def delete_from_hdx(self) -> None: """ self._delete_from_hdx("dataset", "id") - def hxl_update(self) -> None: - """Checks dataset for HXL in resources and updates tags and other metadata to trigger HXL preview. - - Returns: - None - """ - self._read_from_hdx("dataset", self.data["id"], action=self.actions()["hxl"]) - @classmethod def search_in_hdx( cls, @@ -2249,10 +2220,10 @@ def preview_resource(self) -> None: """ self.data["dataset_preview"] = "resource_id" - def set_quickchart_resource( + def set_preview_resource( self, resource: Union["Resource", dict, str, int] ) -> "Resource": - """Set the resource that will be used for displaying QuickCharts in dataset preview + """Set the resource that will be used for displaying previews in dataset preview Args: resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position @@ -2286,17 +2257,14 @@ def set_quickchart_resource( dataset_resource.disable_dataset_preview() return preview_resource - def quickcharts_resource_last(self) -> bool: - """Move the QuickCharts resource to be last. Assumes that it's name begins 'QuickCharts-'. - - Returns: - True if QuickCharts resource found, False if not - """ - for i, resource in enumerate(self._resources): - if resource["name"][:12] == "QuickCharts-": - self._resources.append(self._resources.pop(i)) - return True - return False + def set_quickchart_resource( + self, resource: Union["Resource", dict, str, int] + ) -> "Resource": + warnings.warn( + "set_quickchart_resource() is deprecated, use set_preview_resource() instead", + DeprecationWarning, + ) + return self.set_preview_resource(resource) def create_default_views(self, create_datastore_views: bool = False) -> None: """Create default resource views for all resources in dataset @@ -2328,164 +2296,6 @@ def _create_preview_resourceview(self) -> None: self._preview_resourceview = None break - def _generate_resource_view( - self, - resource: Union["Resource", dict, str, int] = 0, - path: Path | str | None = None, - bites_disabled: Sequence[bool] | None = None, - indicators: Sequence[dict] | None = None, - findreplace: dict | None = None, - ) -> resource_view.ResourceView | None: - """Create QuickCharts for the given resource in a dataset. If you do - not supply a path, then the internal indicators resource view template - will be used. You can disable specific bites by providing - bites_disabled, a list of 3 bools where True indicates a specific bite - is disabled and False indicates leave enabled. The parameter indicators - is a list with 3 dictionaries of form: - {"code": "MY_INDICATOR_CODE", "title": "MY_INDICATOR_TITLE", - "unit": "MY_INDICATOR_UNIT"}. Optionally, the following defaults can be - overridden in the parameter indicators: {"code_col": "#indicator+code", - "value_col": "#indicator+value+num", "date_col": "#date+year", - "date_format": "%Y", "aggregate_col": "null"}. - - Creation of the resource view will be delayed until after the next - dataset create or update if a resource id is not yet available. - - Args: - resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position. Defaults to 0. - path: Path to YAML resource view metadata. Defaults to None (config/hdx_resource_view_static.yaml or internal template). - bites_disabled: Which QC bites should be disabled. Defaults to None (all bites enabled). - indicators: Indicator codes, QC titles and units for resource view template. Defaults to None (don't use template). - findreplace: Replacements for anything else in resource view. Defaults to None. - - Returns: - The resource view if QuickCharts created, None is not - """ - if not bites_disabled: - bites_disabled = [False, False, False] - elif all(i for i in bites_disabled): - return None - res = self.set_quickchart_resource(resource) - if res is None: - return None - if "id" in res: - resourceview_data = {"resource_id": res["id"]} - else: - resourceview_data = {"resource_name": res["name"]} - resourceview = resource_view.ResourceView(resourceview_data) - if path is None: - if indicators is None: - path = Path("config", "hdx_resource_view_static.yaml") - if not isfile(path): - path = path.with_suffix(".yml") - else: - path = script_dir_plus_file( - "indicator_resource_view_template.yaml", - NotRequestableError, - ) - resourceview.update_from_yaml(path=path) - - def replace_string(instr, what, withwhat): - return instr.replace(what, str(withwhat)) - - defaults = { - "code_col": "#indicator+code", - "value_col": "#indicator+value+num", - "date_col": "#date+year", - "date_format": "%Y", - "aggregate_col": "null", - } - - def replace_col(hxl_preview_cfg, col_str, ind, col, with_quotes=False): - if with_quotes: - col_str = f'"{col_str}"' - replace = ind.get(col) - if replace: - if with_quotes: - replace = f'"{replace}"' - else: - replace = defaults[col] - return replace_string(hxl_preview_cfg, col_str, replace) - - hxl_preview_config = resourceview["hxl_preview_config"] - if indicators is None: - indicators_notexist = [False, False, False] - else: - len_indicators = len(indicators) - if len_indicators == 0: - return None - indicators_notexist = [True, True, True] - - def replace_indicator(qc_config, index): - indicator = indicators[index] - ind_str = str(index + 1) - qc_config = replace_string( - qc_config, f"CODE_VALUE_{ind_str}", str(indicator["code"]) - ) - replace = indicator.get("description", "") - qc_config = replace_string( - qc_config, f"DESCRIPTION_VALUE_{ind_str}", replace - ) - qc_config = replace_string( - qc_config, f"TITLE_VALUE_{ind_str}", indicator["title"] - ) - replace = indicator.get("unit", "") - qc_config = replace_string(qc_config, f"UNIT_VALUE_{ind_str}", replace) - qc_config = replace_col( - qc_config, f"CODE_COL_{ind_str}", indicator, "code_col" - ) - qc_config = replace_col( - qc_config, f"VALUE_COL_{ind_str}", indicator, "value_col" - ) - qc_config = replace_col( - qc_config, f"DATE_COL_{ind_str}", indicator, "date_col" - ) - qc_config = replace_col( - qc_config, - f"DATE_FORMAT_{ind_str}", - indicator, - "date_format", - ) - qc_config = replace_col( - qc_config, - f"AGGREGATE_COL_{ind_str}", - indicator, - "aggregate_col", - True, - ) - indicators_notexist[index] = False - return qc_config - - if indicators[0]: - hxl_preview_config = replace_indicator(hxl_preview_config, 0) - if len_indicators > 1 and indicators[1]: - hxl_preview_config = replace_indicator(hxl_preview_config, 1) - if len_indicators > 2 and indicators[2]: - hxl_preview_config = replace_indicator(hxl_preview_config, 2) - if indicators_notexist == [True, True, True]: - return None - hxl_preview_config = json.loads(hxl_preview_config) - bites = hxl_preview_config["bites"] - for i, disable in reversed(list(enumerate(bites_disabled))): - if disable or indicators_notexist[i]: - del bites[i] - hxl_preview_config = json.dumps( - hxl_preview_config, indent=None, separators=(",", ":") - ) - if findreplace: - for find in findreplace: - hxl_preview_config = replace_string( - hxl_preview_config, find, findreplace[find] - ) - resourceview["hxl_preview_config"] = hxl_preview_config - - if "resource_id" in resourceview: - resourceview.create_in_hdx() - self._preview_resourceview = None - else: - self._preview_resourceview = resourceview - return resourceview - def generate_quickcharts( self, resource: Union["Resource", dict, str, int] = 0, @@ -2493,22 +2303,8 @@ def generate_quickcharts( bites_disabled: Sequence[bool] | None = None, indicators: Sequence[dict] | None = None, findreplace: dict | None = None, - ) -> resource_view.ResourceView: - """Create QuickCharts for the given resource in a dataset. If you do - not supply a path, then the internal indicators resource view template - will be used. You can disable specific bites by providing - bites_disabled, a list of 3 bools where True indicates a specific bite - is disabled and False indicates leave enabled. The parameter indicators - is a list with 3 dictionaries of form: - {"code": "MY_INDICATOR_CODE", "title": "MY_INDICATOR_TITLE", - "unit": "MY_INDICATOR_UNIT"}. Optionally, the following defaults can be - overridden in the parameter indicators: {"code_col": "#indicator+code", - "value_col": "#indicator+value+num", "date_col": "#date+year", - "date_format": "%Y", "aggregate_col": "null"}. - - Creation of the resource view will be delayed until after the next - dataset create or update if a resource id is not yet available and will - be disabled if there are no valid charts to display. + ) -> None: + """To be removed Args: resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position. Defaults to 0. @@ -2518,14 +2314,9 @@ def generate_quickcharts( findreplace: Replacements for anything else in resource view. Defaults to None. Returns: - The resource view if QuickCharts created, None is not + None """ - resourceview = self._generate_resource_view( - resource, path, bites_disabled, indicators, findreplace=findreplace - ) - if resourceview is None: - self.preview_off() - return resourceview + return None def get_name_or_id(self, prefer_name: bool = True) -> str | None: """Get dataset name or id eg. for use in urls. If prefer_name is True, @@ -2802,25 +2593,6 @@ def generate_resource_from_iterable( the time period and are returned in the results dictionary in keys startdate and enddate. - If the parameter quickcharts is supplied then various QuickCharts - related actions will occur depending upon the keys given in the - dictionary and the returned dictionary will contain the QuickCharts - resource in the key qc_resource. If the keys: hashtag - the HXL hashtag - to examine - and values - the 3 values to look for in that column - are - supplied, then a list of booleans indicating which QuickCharts bites - should be enabled will be returned in the key bites_disabled in the - returned dictionary. For the 3 values, if the key: numeric_hashtag is - supplied then if that column for a given value contains no numbers, - then the corresponding bite will be disabled. If the key: cutdown is - given, if it is 1, then a separate cut down list is created containing - only columns with HXL hashtags and rows with desired values (if hashtag - and values are supplied) for the purpose of driving QuickCharts. It is - returned in the key qcrows in the returned dictionary with the matching - headers in qcheaders. If cutdown is 2, then a resource is created using - the cut down list. If the key cutdownhashtags is supplied, then only - the provided hashtags are used for cutting down otherwise the full list - of HXL tags is used. - Args: headers: Headers iterable: Iterable returning rows @@ -2831,7 +2603,6 @@ def generate_resource_from_iterable( datecol: Date column for setting time period. Defaults to None (don't set). yearcol: Year column for setting dataset year range. Defaults to None (don't set). date_function: Date function to call for each row. Defaults to None. - quickcharts: Dictionary containing optional keys: hashtag, values, cutdown and/or cutdownhashtags encoding: Encoding to use. Defaults to None (infer encoding). Returns: @@ -2848,40 +2619,6 @@ def generate_resource_from_iterable( return False, retdict rows = [Download.hxl_row(headers, hxltags, dict_form=True)] dates = [default_enddate, default_date] - qc = {"cutdown": 0} - bites_disabled = [True, True, True] - if quickcharts is not None: - hashtag = quickcharts.get("hashtag") - if hashtag: - qc["column"] = next( - key for key, value in hxltags.items() if value == hashtag - ) # reverse dict lookup - else: - qc["column"] = None - numeric_hashtag = quickcharts.get("numeric_hashtag") - if numeric_hashtag: - qc["numeric"] = next( - key for key, value in hxltags.items() if value == numeric_hashtag - ) # reverse lookup - else: - qc["numeric"] = None - cutdown = quickcharts.get("cutdown", 0) - if cutdown: - qc["cutdown"] = cutdown - cutdownhashtags = quickcharts.get("cutdownhashtags") - if cutdownhashtags is None: - cutdownhashtags = list(hxltags.keys()) - else: - cutdownhashtags = [ - key - for key, value in hxltags.items() - if value in cutdownhashtags - ] - if numeric_hashtag and qc["numeric"] not in cutdownhashtags: - cutdownhashtags.append(qc["numeric"]) - qc["headers"] = [x for x in headers if x in cutdownhashtags] - qc["rows"] = [Download.hxl_row(qc["headers"], hxltags, dict_form=True)] - if yearcol is not None: def yearcol_function(row): @@ -2923,24 +2660,6 @@ def datecol_function(row): if enddate > dates[1]: dates[1] = enddate rows.append(row) - if quickcharts is not None: - if qc["column"] is None: - if qc["cutdown"]: - qcrow = {x: row[x] for x in qc["headers"]} - qc["rows"].append(qcrow) - else: - value = row[qc["column"]] - for i, lookup in enumerate(quickcharts["values"]): - if value == lookup: - if qc["numeric"]: - try: - float(row[qc["numeric"]]) - except (TypeError, ValueError): - continue - bites_disabled[i] = False - if qc["cutdown"]: - qcrow = {x: row[x] for x in qc["headers"]} - qc["rows"].append(qcrow) if len(rows) == 1: logger.error(f"No data rows in {filename}!") return False, retdict @@ -2963,25 +2682,6 @@ def datecol_function(row): retdict["resource"] = resource retdict["headers"] = headers retdict["rows"] = rows - if quickcharts is not None: - retdict["bites_disabled"] = bites_disabled - if qc["cutdown"]: - retdict["qcheaders"] = qc["headers"] - retdict["qcrows"] = qc["rows"] - if qc["cutdown"] == 2: - qc_resourcedata = { - "name": f"QuickCharts-{resourcedata['name']}", - "description": "Cut down data for QuickCharts", - } - resource = self.generate_resource_from_rows( - folder, - f"qc_{filename}", - qc["rows"], - qc_resourcedata, - headers=qc["headers"], - encoding=encoding, - ) - retdict["qc_resource"] = resource return True, retdict def generate_resource_from_iterator( @@ -3057,25 +2757,6 @@ def download_generate_resource( the time period and are returned in the results dictionary in keys startdate and enddate. - If the parameter quickcharts is supplied then various QuickCharts - related actions will occur depending upon the keys given in the - dictionary and the returned dictionary will contain the QuickCharts - resource in the key qc_resource. If the keys: hashtag - the HXL hashtag - to examine - and values - the 3 values to look for in that column - are - supplied, then a list of booleans indicating which QuickCharts bites - should be enabled will be returned in the key bites_disabled in the - returned dictionary. For the 3 values, if the key: numeric_hashtag is - supplied then if that column for a given value contains no numbers, - then the corresponding bite will be disabled. If the key: cutdown is - given, if it is 1, then a separate cut down list is created containing - only columns with HXL hashtags and rows with desired values (if hashtag - and values are supplied) for the purpose of driving QuickCharts. It is - returned in the key qcrows in the returned dictionary with the matching - headers in qcheaders. If cutdown is 2, then a resource is created using - the cut down list. If the key cutdownhashtags is supplied, then only - the provided hashtags are used for cutting down otherwise the full list - of HXL tags is used. - Args: downloader: A Download or Retrieve object url: URL to download @@ -3158,25 +2839,6 @@ def download_and_generate_resource( the time period and are returned in the results dictionary in keys startdate and enddate. - If the parameter quickcharts is supplied then various QuickCharts - related actions will occur depending upon the keys given in the - dictionary and the returned dictionary will contain the QuickCharts - resource in the key qc_resource. If the keys: hashtag - the HXL hashtag - to examine - and values - the 3 values to look for in that column - are - supplied, then a list of booleans indicating which QuickCharts bites - should be enabled will be returned in the key bites_disabled in the - returned dictionary. For the 3 values, if the key: numeric_hashtag is - supplied then if that column for a given value contains no numbers, - then the corresponding bite will be disabled. If the key: cutdown is - given, if it is 1, then a separate cut down list is created containing - only columns with HXL hashtags and rows with desired values (if hashtag - and values are supplied) for the purpose of driving QuickCharts. It is - returned in the key qcrows in the returned dictionary with the matching - headers in qcheaders. If cutdown is 2, then a resource is created using - the cut down list. If the key cutdownhashtags is supplied, then only - the provided hashtags are used for cutting down otherwise the full list - of HXL tags is used. - Args: downloader: A Download or Retrieve object url: URL to download @@ -3189,7 +2851,6 @@ def download_and_generate_resource( datecol: Date column for setting time period. Defaults to None (don't set). yearcol: Year column for setting dataset year range. Defaults to None (don't set). date_function: Date function to call for each row. Defaults to None. - quickcharts: Dictionary containing optional keys: hashtag, values, cutdown and/or cutdownhashtags **kwargs: Any additional args to pass to downloader.get_tabular_rows Returns: diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index 5bdc47a..24b714a 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -5,7 +5,7 @@ from collections.abc import Callable from inspect import getdoc from pathlib import Path -from typing import Any +from typing import Any, Optional # noqa: F401 import defopt from hdx.utilities.easy_logging import setup_logging diff --git a/tests/hdx/api/test_ckan.py b/tests/hdx/api/test_ckan.py index 6a0e79e..776ad01 100644 --- a/tests/hdx/api/test_ckan.py +++ b/tests/hdx/api/test_ckan.py @@ -168,9 +168,7 @@ def create_resource(): for i in range(10): create_resource() - dataset.create_in_hdx( - hxl_update=False, updated_by_script="hdx_python_api_ignore" - ) + dataset.create_in_hdx(updated_by_script="hdx_python_api_ignore") # check created dataset dataset = Dataset.read_from_hdx(name) @@ -219,7 +217,7 @@ def create_resource(): dataset.add_update_resources(resources) sleep(2) - dataset.update_in_hdx(hxl_update=False, remove_additional_resources=True) + dataset.update_in_hdx(remove_additional_resources=True) sleep(2) # check updated dataset @@ -264,7 +262,6 @@ def create_resource(): sleep(2) dataset.create_in_hdx( - hxl_update=False, remove_additional_resources=True, keys_to_delete=("caveats",), ) diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index 7385aaa..9f1b6ad 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -170,19 +170,6 @@ def mockall(url, datadict): ) -def mockhxlupdate(url, datadict): - if "hxl" not in url: - return MockResponse( - 404, - '{"success": false, "error": {"message": "TEST ERROR: Not HXL Update", "__type": "TEST ERROR: Not HXL Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_hxl_update"}', - ) - result = json.dumps(hxlupdate_list) - return MockResponse( - 200, - f'{{"success": true, "result": {result}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_hxl_update"}}', - ) - - class TestDatasetCore: @pytest.fixture(scope="class") def static_yaml(self, configfolder): @@ -244,8 +231,6 @@ def post(url, data, headers, files, allow_redirects, auth=None): return vocabulary_mockshow(url, datadict) if "show" in url: return dataset_mockshow(url, datadict) - if "hxl" in url: - return mockhxlupdate(url, datadict) if "default" in url: result = json.dumps(resource_view_list) return MockResponse( @@ -331,8 +316,6 @@ def post(url, data, headers, files, allow_redirects, auth=None): return vocabulary_mockshow(url, datadict) if "show" in url: return dataset_mockshow(url, datadict) - if "hxl" in url: - return mockhxlupdate(url, datadict) if "resource" in url: resultdictcopy = copy.deepcopy(resources_data[0]) merge_two_dictionaries(resultdictcopy, datadict) @@ -386,8 +369,6 @@ def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(decodedata) if "show" in url: return dataset_mockshow(url, datadict) - if "hxl" in url: - return mockhxlupdate(url, datadict) if "reorder" not in url: return MockResponse( 404, diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index a06c506..ceba138 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -717,9 +717,7 @@ def test_set_quickchart_resource(self, configuration): dataset = Dataset(datasetdata) assert "dataset_preview" not in dataset assert ( - dataset.set_quickchart_resource("3d777226-96aa-4239-860a-703389d16d1f")[ - "id" - ] + dataset.set_preview_resource("3d777226-96aa-4239-860a-703389d16d1f")["id"] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert dataset["dataset_preview"] == "resource_id" @@ -727,185 +725,45 @@ def test_set_quickchart_resource(self, configuration): assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "True" assert ( - dataset.set_quickchart_resource(resources[0])["id"] + dataset.set_preview_resource(resources[0])["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) assert resources[0]["dataset_preview_enabled"] == "True" assert resources[1]["dataset_preview_enabled"] == "False" assert ( - dataset.set_quickchart_resource(resources[1].data)["id"] + dataset.set_preview_resource(resources[1].data)["id"] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "True" assert ( - dataset.set_quickchart_resource(0)["id"] + dataset.set_preview_resource(0)["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) assert resources[0]["dataset_preview_enabled"] == "True" assert resources[1]["dataset_preview_enabled"] == "False" - assert dataset.set_quickchart_resource("12345") is None + assert dataset.set_preview_resource("12345") is None with pytest.raises(HDXError): - dataset.set_quickchart_resource(True) + dataset.set_preview_resource(True) dataset.preview_off() assert dataset["dataset_preview"] == "no_preview" assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "False" assert ( - dataset.set_quickchart_resource("Resource2")["id"] + dataset.set_preview_resource("Resource2")["id"] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert dataset["dataset_preview"] == "resource_id" assert resources[0]["dataset_preview_enabled"] == "False" assert resources[1]["dataset_preview_enabled"] == "True" assert ( - dataset.set_quickchart_resource({"name": "Resource1"})["id"] + dataset.set_preview_resource({"name": "Resource1"})["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) assert dataset["dataset_preview"] == "resource_id" assert resources[0]["dataset_preview_enabled"] == "True" assert resources[1]["dataset_preview_enabled"] == "False" - def test_quickcharts_resource_last(self, configuration): - datasetdata = copy.deepcopy(dataset_data) - resourcesdata = copy.deepcopy(resources_data) - datasetdata["resources"] = resourcesdata - dataset = Dataset(datasetdata) - assert dataset.quickcharts_resource_last() is False - resource = {"name": "QuickCharts-resource"} - dataset._resources.insert(1, resource) - assert dataset.quickcharts_resource_last() is True - assert dataset._resources[3]["name"] == resource["name"] - assert dataset.quickcharts_resource_last() is True - - def test_generate_resource_view( - self, configuration, vocabulary_update, static_resource_view_yaml - ): - datasetdata = copy.deepcopy(dataset_data) - resourcesdata = copy.deepcopy(resources_data) - datasetdata["resources"] = resourcesdata - dataset = Dataset(datasetdata) - assert "dataset_preview" not in dataset - resourceview = dataset.generate_quickcharts(path=static_resource_view_yaml) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert hxl_preview_config["bites"][0]["title"] == "Sum of fatalities" - assert ( - hxl_preview_config["bites"][1]["title"] - == "Sum of fatalities grouped by admin1" - ) - assert ( - hxl_preview_config["bites"][2]["title"] - == "Sum of fatalities grouped by admin2" - ) - resourceview = dataset.generate_quickcharts( - path=static_resource_view_yaml, bites_disabled=[False, True, False] - ) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert hxl_preview_config["bites"][0]["title"] == "Sum of fatalities" - assert ( - hxl_preview_config["bites"][1]["title"] - == "Sum of fatalities grouped by admin2" - ) - resourceview = dataset.generate_quickcharts( - path=static_resource_view_yaml, bites_disabled=[True, True, True] - ) - assert resourceview is None - indicators = [ - { - "code": "1", - "title": "My1", - "unit": "ones", - "description": "This is my one!", - }, - { - "code": "2", - "title": "My2", - "unit": "twos", - "aggregate_col": "Agg2", - }, - { - "code": "3", - "title": "My3", - "description": "This is my three!", - "date_col": "dt3", - "date_format": "%b %Y", - }, - ] - resourceview = dataset.generate_quickcharts(indicators=indicators) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert ( - hxl_preview_config["bites"][0]["ingredient"]["filters"]["filterWith"][0][ - "#indicator+code" - ] - == "1" - ) - assert ( - hxl_preview_config["bites"][0]["ingredient"]["description"] - == "This is my one!" - ) - assert hxl_preview_config["bites"][0]["uiProperties"]["title"] == "My1" - assert ( - hxl_preview_config["bites"][0]["computedProperties"]["dataTitle"] == "ones" - ) - assert ( - hxl_preview_config["bites"][1]["ingredient"]["filters"]["filterWith"][0][ - "#indicator+code" - ] - == "2" - ) - assert hxl_preview_config["bites"][1]["ingredient"]["description"] == "" - assert hxl_preview_config["bites"][1]["uiProperties"]["title"] == "My2" - assert ( - hxl_preview_config["bites"][1]["computedProperties"]["dataTitle"] == "twos" - ) - assert hxl_preview_config["bites"][1]["ingredient"]["aggregateColumn"] == "Agg2" - assert ( - hxl_preview_config["bites"][2]["ingredient"]["filters"]["filterWith"][0][ - "#indicator+code" - ] - == "3" - ) - assert ( - hxl_preview_config["bites"][2]["ingredient"]["description"] - == "This is my three!" - ) - assert hxl_preview_config["bites"][2]["ingredient"]["dateColumn"] == "dt3" - assert hxl_preview_config["bites"][2]["uiProperties"]["title"] == "My3" - assert hxl_preview_config["bites"][2]["computedProperties"]["dataTitle"] == "" - assert hxl_preview_config["bites"][2]["uiProperties"]["dateFormat"] == "%b %Y" - resourceview = dataset.generate_quickcharts( - indicators=indicators, - findreplace={ - "#indicator+code": "#item+code", - "#indicator+value+num": "#value", - }, - ) - hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) - assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - assert ( - hxl_preview_config["bites"][0]["ingredient"]["filters"]["filterWith"][0][ - "#item+code" - ] - == "1" - ) - assert hxl_preview_config["bites"][0]["ingredient"]["valueColumn"] == "#value" - assert dataset.generate_quickcharts(indicators=[]) is None - assert dataset.generate_quickcharts(indicators=[None, None, None]) is None - assert ( - dataset.generate_quickcharts(resource="123", path=static_resource_view_yaml) - is None - ) - del dataset.get_resources()[0]["id"] - resourceview = dataset.generate_quickcharts(path=static_resource_view_yaml) - assert "id" not in resourceview - assert "resource_id" not in resourceview - assert resourceview["resource_name"] == "Resource1" - with pytest.raises(IOError): - dataset.generate_quickcharts() - def test_remove_dates_from_title(self, configuration): dataset = Dataset() with pytest.raises(HDXError): diff --git a/tests/hdx/data/test_dataset_resource_generation.py b/tests/hdx/data/test_dataset_resource_generation.py index 023bb98..86041b2 100644 --- a/tests/hdx/data/test_dataset_resource_generation.py +++ b/tests/hdx/data/test_dataset_resource_generation.py @@ -562,7 +562,6 @@ def process_row(headers, row): assert results == { "startdate": datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc), "enddate": datetime(2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc), - "bites_disabled": [False, True, False], "resource": { "description": "Conflict data with HXL tags", "format": "csv", @@ -738,85 +737,6 @@ def process_row(headers, row): "lala": "lala", }, ], - "qc_resource": { - "description": "Cut down data for QuickCharts", - "format": "csv", - "name": "QuickCharts-Conflict Data for Algeria", - }, - "qcheaders": [ - "EVENT_ID_CNTY", - "EVENT_DATE", - "YEAR", - "EVENT_TYPE", - "ACTOR1", - "ACTOR2", - "COUNTRY", - "ADMIN1", - "ADMIN2", - "ADMIN3", - "LOCATION", - "LATITUDE", - "LONGITUDE", - "SOURCE", - "NOTES", - "FATALITIES", - ], - "qcrows": [ - { - "EVENT_ID_CNTY": "#event+code", - "EVENT_DATE": "#date+occurred", - "YEAR": "#date+year", - "EVENT_TYPE": "#event+type", - "ACTOR1": "#group+name+first", - "ACTOR2": "#group+name+second", - "COUNTRY": "#country+name", - "ADMIN1": "#adm1+name", - "ADMIN2": "#adm2+name", - "ADMIN3": "#adm3+name", - "LOCATION": "#loc+name", - "LATITUDE": "#geo+lat", - "LONGITUDE": "#geo+lon", - "SOURCE": "#meta+source", - "NOTES": "#description", - "FATALITIES": "#affected+killed", - }, - { - "EVENT_ID_CNTY": "1416RTA", - "EVENT_DATE": "18/04/2001", - "YEAR": "2001", - "EVENT_TYPE": "Violence against civilians", - "ACTOR1": "Police Forces of Algeria (1999-)", - "ACTOR2": "Civilians (Algeria)", - "COUNTRY": "Algeria", - "ADMIN1": "Tizi Ouzou", - "ADMIN2": "Beni-Douala", - "ADMIN3": None, - "LOCATION": "Beni Douala", - "LATITUDE": "36.61954", - "LONGITUDE": "4.08282", - "SOURCE": "Associated Press Online", - "NOTES": "A Berber student was shot while in police custody at a police station in Beni Douala. He later died on Apr.21.", - "FATALITIES": "1", - }, - { - "EVENT_ID_CNTY": "2231RTA", - "EVENT_DATE": "21/04/2001", - "YEAR": "2001", - "EVENT_TYPE": "Riots/Protests", - "ACTOR1": "Rioters (Algeria)", - "ACTOR2": "Police Forces of Algeria (1999-)", - "COUNTRY": "Algeria", - "ADMIN1": "Bejaia", - "ADMIN2": "Amizour", - "ADMIN3": None, - "LOCATION": "Amizour", - "LATITUDE": "36.64022", - "LONGITUDE": "4.90131", - "SOURCE": "Kabylie report", - "NOTES": "Rioters threw molotov cocktails, rocks and burning tires at gendarmerie stations in Beni Douala, El-Kseur and Amizour.", - "FATALITIES": "0", - }, - ], } assert ( dataset["dataset_date"] @@ -830,22 +750,11 @@ def process_row(headers, row): "description": "Conflict data with HXL tags", "format": "csv", }, - { - "name": "QuickCharts-Conflict Data for Algeria", - "description": "Cut down data for QuickCharts", - "format": "csv", - }, ] assert_files_same( fixturesfolder / "download_gen_resource" / filename, folder / filename, ) - qc_filename = f"qc_{filename}" - assert_files_same( - fixturesfolder / "gen_resource" / qc_filename, - folder / qc_filename, - ) - success, results = dataset.download_and_generate_resource( downloader, TestDatasetResourceGeneration.url, @@ -887,7 +796,6 @@ def process_row(headers, row): assert results == { "startdate": datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc), "enddate": datetime(2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc), - "bites_disabled": [False, True, False], "resource": { "description": "Conflict data with HXL tags", "format": "csv", @@ -1063,20 +971,6 @@ def process_row(headers, row): "lala": "lala", }, ], - "qc_resource": { - "description": "Cut down data for QuickCharts", - "format": "csv", - "name": "QuickCharts-Conflict Data for Algeria", - }, - "qcheaders": ["EVENT_ID_CNTY", "FATALITIES"], - "qcrows": [ - { - "EVENT_ID_CNTY": "#event+code", - "FATALITIES": "#affected+killed", - }, - {"EVENT_ID_CNTY": "1416RTA", "FATALITIES": "1"}, - {"EVENT_ID_CNTY": "2231RTA", "FATALITIES": "0"}, - ], } def process_year(row): @@ -1113,10 +1007,6 @@ def process_year(row): dataset["dataset_date"] == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" ) - assert_files_same( - fixturesfolder / "gen_resource" / f"min_{qc_filename}", - folder / qc_filename, - ) with pytest.raises(HDXError): dataset.download_and_generate_resource( diff --git a/tests/hdx/data/test_update_dataset_resources.py b/tests/hdx/data/test_update_dataset_resources.py index 4fa1939..6aaed64 100644 --- a/tests/hdx/data/test_update_dataset_resources.py +++ b/tests/hdx/data/test_update_dataset_resources.py @@ -156,7 +156,6 @@ def test_dataset_update_resources( resources_to_delete, filestore_resources, new_resource_order, - hxl_update=False, create_default_views=False, test=True, ) diff --git a/tests/hdx/data/test_update_logic.py b/tests/hdx/data/test_update_logic.py index cbc1637..ae51218 100644 --- a/tests/hdx/data/test_update_logic.py +++ b/tests/hdx/data/test_update_logic.py @@ -185,7 +185,6 @@ def test_update_logic_1( remove_additional_resources=True, match_resource_order=False, create_default_views=False, - hxl_update=False, test=True, ) assert statuses == { @@ -909,7 +908,6 @@ def test_update_logic_2( remove_additional_resources=True, match_resource_order=False, create_default_views=False, - hxl_update=False, test=True, ) assert statuses == { @@ -1633,7 +1631,6 @@ def test_update_logic_3( remove_additional_resources=True, match_resource_order=False, create_default_views=False, - hxl_update=False, test=True, ) assert statuses == {