diff --git a/.import_linter_config b/.import_linter_config index 03b655cc7..01a924fd8 100644 --- a/.import_linter_config +++ b/.import_linter_config @@ -27,3 +27,4 @@ ignore_imports = # To reduce the effort in using nox sessions (i.e. having to pass the config path # in each CLI usage), we allow the noxconfig to be imported within these modules. exasol.toolbox.nox.* -> noxconfig + exasol.toolbox.tools.template -> noxconfig diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index e275802eb..d91b24026 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -34,21 +34,22 @@ In previous versions, the default value for `poetry-version` was `2.1.2`, and it * Depending on its poetry version, a repository relying on the default behavior of said action may run into breaking changes. This can easily be resolved with explicitly setting the `poetry-version` when calling the GitHub action. It is, however, recommended whenever -possible to update the poetry version of the affected repository. Unfortunately, -there is not a quick and easy way to update all the places where `poetry-version` -could be specified in the GitHub workflows. +possible to update the poetry version of the affected repository. Since this major release, +you can, if needed, alter the `poetry-version` via the `noxconfig.py::PROJECT_CONFIG` +by changing `dependency_manager_version`. If you do this, please create an issue to +update to `2.3.0` at your earliest convenience. * Projects migrating to this version should: -* Update their `pyproject.toml` to have: - ```toml - requires-poetry = ">=2.3.0" - ``` -* Run `poetry check` and resolve any issues -* (optional) Run `poetry lock` to update the lock -* (optional) Update their `pyproject.toml` to fit: - * [PEP-621](https://peps.python.org/pep-0621/) - * [PEP-735](https://peps.python.org/pep-0735/) + * Update their `pyproject.toml` to have: + ```toml + requires-poetry = ">=2.3.0" + ``` + * Run `poetry check` and resolve any issues + * (optional) Run `poetry lock` to update the lock + * (optional) Update their `pyproject.toml` to fit: + * [PEP-621](https://peps.python.org/pep-0621/) + * [PEP-735](https://peps.python.org/pep-0735/) Note that [uvx migrate-to-uv](https://github.com/mkniewallner/migrate-to-uv) seems to do a good job with automating many of the PEP-related changes; though developers should @@ -65,6 +66,16 @@ take care and will need to make manual changes to ensure it still works with * #647: Added summary to changelog template * #657: Updated `release:prepare` to modify cookiecutter template exasol-toolbox version range * #665: Added SECURITY.md to the cookiecutter template +* #667: Switched GitHub workflow templates to be controlled by PROJECT_CONFIG: + * The values in `BaseConfig.github_template_dict` are used to render the following values in + the templates + * `dependency_manager_version` - used for `poetry-version` in the workflows. + The default it `2.3.0`. + * `minimum_python_version` - used for `python-version` in the workflows whenever + `python-version` for actions that are run once. The default is the minimum value + in your project's defined `python_versions` + * `os_version` - used for the GitHub runner in the workflows. The default is + `ubuntu-24.04` ## Refactoring @@ -74,3 +85,5 @@ take care and will need to make manual changes to ensure it still works with * `upload-pages-artifact` from v3 to [v4](https://github.com/actions/upload-pages-artifact/releases/tag/v4.0.0) - breaking change * `download-artifact`from v6 to [v7](https://github.com/actions/download-artifact/releases/tag/v7.0.0) - using Node.js 24 * `upload-artifact` from v5 to [v6](https://github.com/actions/upload-artifact/releases/tag/v6.0.0) - using Node.js 24 +* #667: Added deprecation warnings to `tbx workflow x` endpoints as some are unneeded +(will be removed) and others need updates (will be moved to a nox session) diff --git a/doc/user_guide/dependencies.rst b/doc/user_guide/dependencies.rst index 782b3a856..7cadf486a 100644 --- a/doc/user_guide/dependencies.rst +++ b/doc/user_guide/dependencies.rst @@ -1,9 +1,62 @@ Dependencies ============ -Core dependencies -+++++++++++++++++ +Core Dependencies +----------------- - Python >= 3.10 -- poetry >= 2.3.0 +- `Poetry `__ >= 2.3.0 - `poetry export `__ + +Supported Poetry Versions by PTB +-------------------------------- + +.. list-table:: PTB Poetry Version Compatibility + :header-rows: 1 + + * - PTB Version + - Default in PTB + - Range Allowed + - Migration Information + * - >=1.0.0, <5.0.0 + - 2.1.2 + - >=2.1.0,<3.0 + - None + * - >=5.0.0 + - 2.3.0 + - >=2.3.0,<3.0 + - :ref:`migration_to_2.3.x` + +Migration Information +--------------------- + +.. _migration_to_2.3.x: + +From Poetry ``2.1.x`` to ``2.3.0`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +It is **highly** encouraged that a developer update their ``pyproject.toml`` and +system-wide Poetry installation to most use effectively use Poetry ``2.3.0``: + +#. In your terminal, update your system-wide Poetry version: + + .. code-block:: bash + + poetry self update 2.3.0 + +#. In your project's ``pyproject.toml``, update the ``requires-poetry`` value: + + .. code-block:: toml + + requires-poetry = ">=2.3.0" + +#. In your terminal, execute ``poetry check`` and resolve any listed issues +#. In your terminal, run ``poetry lock`` to update the lock +#. (optional but recommended) In your project's ``pyproject.toml``, update it to fit: + * `PEP-621 `__ + * `PEP-735 `__ + + .. note:: + Note that `uvx migrate-to-uv `__ + seems to do a good job with automating many of the PEP-related changes. + Though, a developer should take care to verify the changes, as some are unneeded + as it completes the migration to ``uv`` which the PTB does NOT yet support. diff --git a/doc/user_guide/features/documentation/index.rst b/doc/user_guide/features/documentation/index.rst index f284775c0..34b9f583a 100644 --- a/doc/user_guide/features/documentation/index.rst +++ b/doc/user_guide/features/documentation/index.rst @@ -9,7 +9,7 @@ Deploying documentation multiversion troubleshooting -The PTB uses ref:`sphinx `__ to build and validate the contents +The PTB uses `sphinx `__ to build and validate the contents of your project's documentation. PTB expects the project's documentation in directory ``doc``, primarily as ``rst`` files. The ``doc/conf.py`` acts as the configuration file for building the documentation. diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 4678e4c9b..4b771e268 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -1,18 +1,23 @@ import inspect +import warnings from collections.abc import Callable from pathlib import Path from typing import ( Annotated, Any, + Literal, ) +from packaging import version from pydantic import ( AfterValidator, BaseModel, ConfigDict, Field, computed_field, + field_validator, ) +from pydantic_core.core_schema import ValidationInfo from exasol.toolbox.nox.plugin import ( METHODS_SPECIFIED_FOR_HOOKS, @@ -93,6 +98,26 @@ def valid_version_string(version_string: str) -> str: } +class DependencyManager(BaseModel): + # Restricted to only allow "poetry" at the moment + name: Literal["poetry"] + version: ValidVersionStr + + @field_validator("version") + @classmethod + def check_minimum_version(cls, v: str, info: ValidationInfo) -> str: + tool = info.data.get("name") + if tool == "poetry": + prefix = "Poetry version " + if version.parse(v) < (min_version := version.parse("2.1.4")): + raise ValueError(prefix + f"must be >= {min_version}") + elif version.parse(v) >= (max_version := version.parse("3.0.0")): + raise ValueError(prefix + f"must be < {max_version}") + elif version.parse(v) > (current_version := version.parse("2.3.0")): + warnings.warn(prefix + f"exceeds last tested version {current_version}") + return v + + class BaseConfig(BaseModel): """ Basic configuration for projects using the PTB @@ -142,6 +167,24 @@ class BaseConfig(BaseModel): possible plugin options are defined in `exasol.toolbox.nox.plugins.NoxTasks`. """, ) + dependency_manager: DependencyManager = Field( + default=DependencyManager(name="poetry", version="2.3.0"), + description=""" + This is used to define which dependency manager is used to install dependencies + in the CI. For more details on which PTB version pairs with which + dependency manager, see: + https://exasol.github.io/python-toolbox/main/user_guide/dependencies.html + """, + ) + os_version: str = Field( + default="ubuntu-24.04", + pattern=r"^ubuntu-.*", + description=""" + This is used to set the OS-runner in the GitHub workflows that are + provided as templates from the PTB. Currently, only ubuntu-based runners + are supported. + """, + ) model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @computed_field # type: ignore[misc] @@ -222,3 +265,16 @@ def version_filepath(self) -> Path: the nox sessions ``version:check`` and ``release:prepare``. """ return self.source_code_path / "version.py" + + @computed_field # type: ignore[misc] + @property + def github_template_dict(self) -> dict[str, Any]: + """ + Dictionary of variables to dynamically render Jinja2 templates into valid YAML + configurations. + """ + return { + "dependency_manager_version": self.dependency_manager.version, + "minimum_python_version": self.minimum_python_version, + "os_version": self.os_version, + } diff --git a/exasol/toolbox/templates/github/workflows/build-and-publish.yml b/exasol/toolbox/templates/github/workflows/build-and-publish.yml index d06d10235..7f0faca79 100644 --- a/exasol/toolbox/templates/github/workflows/build-and-publish.yml +++ b/exasol/toolbox/templates/github/workflows/build-and-publish.yml @@ -10,7 +10,7 @@ jobs: cd-job: name: Continuous Delivery - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: write steps: @@ -19,6 +19,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Build Artifacts run: poetry build diff --git a/exasol/toolbox/templates/github/workflows/check-release-tag.yml b/exasol/toolbox/templates/github/workflows/check-release-tag.yml index 1e27719a5..adb50b90a 100644 --- a/exasol/toolbox/templates/github/workflows/check-release-tag.yml +++ b/exasol/toolbox/templates/github/workflows/check-release-tag.yml @@ -7,7 +7,7 @@ jobs: check-tag-version-job: name: Check Tag Version - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -16,6 +16,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Check Tag Version # make sure the pushed/created tag matched the project version diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml index c25bccaab..8230e04df 100644 --- a/exasol/toolbox/templates/github/workflows/checks.yml +++ b/exasol/toolbox/templates/github/workflows/checks.yml @@ -6,7 +6,7 @@ on: jobs: Version-Check: name: Version - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -17,6 +17,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Check Version(s) run: poetry run -- nox -s version:check @@ -24,7 +27,7 @@ jobs: Documentation: name: Docs needs: [ Version-Check ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -33,6 +36,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Build Documentation run: | @@ -50,7 +56,7 @@ jobs: Changelog: name: Changelog Update Check - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/master' }} @@ -60,6 +66,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Run changelog update check run: poetry run -- nox -s changelog:updated @@ -67,7 +76,7 @@ jobs: Lint: name: Linting (Python-${{ matrix.python-version }}) needs: [ Version-Check, build-matrix ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read strategy: @@ -81,6 +90,7 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@v4 with: python-version: ${{ matrix.python-version }} + poetry-version: "(( dependency_manager_version ))" - name: Run lint run: poetry run -- nox -s lint:code @@ -97,7 +107,7 @@ jobs: Type-Check: name: Type Checking (Python-${{ matrix.python-version }}) needs: [ Version-Check, build-matrix ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read strategy: @@ -112,6 +122,7 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@v4 with: python-version: ${{ matrix.python-version }} + poetry-version: "(( dependency_manager_version ))" - name: Run type-check run: poetry run -- nox -s lint:typing @@ -119,7 +130,7 @@ jobs: Security: name: Security Checks (Python-${{ matrix.python-version }}) needs: [ Version-Check, build-matrix ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read strategy: @@ -134,6 +145,7 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@v4 with: python-version: ${{ matrix.python-version }} + poetry-version: "(( dependency_manager_version ))" - name: Run security linter run: poetry run -- nox -s lint:security @@ -147,7 +159,7 @@ jobs: Format: name: Format Check - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -159,11 +171,14 @@ jobs: - name: Run format check run: poetry run -- nox -s format:check + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" Build-Packages: name: Build Package Check needs: [ Documentation, Lint, Type-Check, Security, Format ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -172,6 +187,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Run Distribution Check run: poetry run -- nox -s package:check @@ -179,7 +197,7 @@ jobs: Tests: name: Unit-Tests (Python-${{ matrix.python-version }}) needs: [ Build-Packages, build-matrix ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read strategy: @@ -194,6 +212,7 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@v4 with: python-version: ${{ matrix.python-version }} + poetry-version: "(( dependency_manager_version ))" - name: Run Tests and Collect Coverage run: poetry run -- nox -s test:unit -- --coverage diff --git a/exasol/toolbox/templates/github/workflows/gh-pages.yml b/exasol/toolbox/templates/github/workflows/gh-pages.yml index b71cd7006..0a77ba8f4 100644 --- a/exasol/toolbox/templates/github/workflows/gh-pages.yml +++ b/exasol/toolbox/templates/github/workflows/gh-pages.yml @@ -7,7 +7,7 @@ on: jobs: build-documentation: - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -18,6 +18,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Build Documentation run: | @@ -38,7 +41,7 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" steps: - name: Deploy to GitHub Pages id: deployment diff --git a/exasol/toolbox/templates/github/workflows/matrix-all.yml b/exasol/toolbox/templates/github/workflows/matrix-all.yml index 8200ed1da..ae75945b4 100644 --- a/exasol/toolbox/templates/github/workflows/matrix-all.yml +++ b/exasol/toolbox/templates/github/workflows/matrix-all.yml @@ -9,7 +9,7 @@ on: jobs: all_versions: - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -18,6 +18,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Generate matrix run: poetry run -- nox -s matrix:all diff --git a/exasol/toolbox/templates/github/workflows/matrix-exasol.yml b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml index 6baab3221..51894e837 100644 --- a/exasol/toolbox/templates/github/workflows/matrix-exasol.yml +++ b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml @@ -9,7 +9,7 @@ on: jobs: exasol_versions: - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -18,6 +18,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Generate matrix run: poetry run -- nox -s matrix:exasol diff --git a/exasol/toolbox/templates/github/workflows/matrix-python.yml b/exasol/toolbox/templates/github/workflows/matrix-python.yml index 3b4aa7468..28a75d3c5 100644 --- a/exasol/toolbox/templates/github/workflows/matrix-python.yml +++ b/exasol/toolbox/templates/github/workflows/matrix-python.yml @@ -9,7 +9,7 @@ on: jobs: python_versions: - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read steps: @@ -18,6 +18,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Generate matrix run: poetry run -- nox -s matrix:python diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index 9caa3ceee..bb54e2b94 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -13,7 +13,7 @@ jobs: run-slow-tests: name: Run Slow Tests - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read @@ -36,7 +36,7 @@ jobs: # This job ensures inputs have been executed successfully. approve-merge: name: Allow Merge - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read # If you need additional jobs to be part of the merge gate, add them below diff --git a/exasol/toolbox/templates/github/workflows/report.yml b/exasol/toolbox/templates/github/workflows/report.yml index af9b32ece..3e5345016 100644 --- a/exasol/toolbox/templates/github/workflows/report.yml +++ b/exasol/toolbox/templates/github/workflows/report.yml @@ -6,7 +6,7 @@ on: jobs: report: - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read env: @@ -20,6 +20,9 @@ jobs: - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" - name: Download Artifacts uses: actions/download-artifact@v7 diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index 964365ba7..634fb78f0 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -13,7 +13,7 @@ jobs: tests: name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) needs: [ build-matrix ] - runs-on: ubuntu-24.04 + runs-on: "(( os_version ))" permissions: contents: read env: @@ -30,6 +30,7 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@v4 with: python-version: ${{ matrix.python-version }} + poetry-version: "(( dependency_manager_version ))" - name: Run Tests and Collect Coverage run: poetry run -- nox -s test:integration -- --coverage --db-version ${{ matrix.exasol-version }} diff --git a/exasol/toolbox/tools/template.py b/exasol/toolbox/tools/template.py index 536bdeae1..14829ea1f 100644 --- a/exasol/toolbox/tools/template.py +++ b/exasol/toolbox/tools/template.py @@ -2,6 +2,7 @@ import io from collections.abc import Mapping from contextlib import ExitStack +from inspect import cleandoc from pathlib import Path from typing import ( Any, @@ -9,15 +10,23 @@ import importlib_resources as resources import typer +import yaml +from jinja2 import Environment from rich.columns import Columns from rich.console import Console from rich.syntax import Syntax +from noxconfig import PROJECT_CONFIG + stdout = Console() stderr = Console(stderr=True) CLI = typer.Typer() +jinja_env = Environment( + variable_start_string="((", variable_end_string="))", autoescape=True +) + def _templates(pkg: str) -> Mapping[str, Any]: def _normalize(name: str) -> str: @@ -56,7 +65,25 @@ def show_templates( raise typer.Exit(code=1) template = templates[template] - stdout.print(Syntax.from_path(path=template, encoding="utf-8", lexer=lexer)) # type: ignore + stdout.print( + Syntax.from_path(path=template, encoding="utf-8", lexer=lexer) + ) # type: ignore + + +def _render_template( + src: str | Path, + stack: ExitStack, +) -> str: + input_file = stack.enter_context(open(src, encoding="utf-8")) + + # dynamically render the template with Jinja2 + template = jinja_env.from_string(input_file.read()) + rendered_string = template.render(PROJECT_CONFIG.github_template_dict) + + # validate that the rendered content is a valid YAML. This is not + # written out as by default it does not give GitHub-safe output. + yaml.safe_load(rendered_string) + return cleandoc(rendered_string) + "\n" def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> None: @@ -75,9 +102,14 @@ def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> No old = stack.enter_context( open(old, encoding="utf-8") if old.exists() else io.StringIO("") ) - new = stack.enter_context(open(new, encoding="utf-8")) - old = old.read().split("\n") - new = new.read().split("\n") + if template_type == "issue": + new = stack.enter_context(open(new, encoding="utf-8")) + old = old.read().split("\n") + new = new.read().split("\n") + elif template_type == "workflow": + new = _render_template(src=new, stack=stack) + old = old.read().split("\n") + new = new.split("\n") diff = difflib.unified_diff(old, new, fromfile="old", tofile="new") stdout.print(Syntax("\n".join(diff), "diff")) @@ -95,9 +127,15 @@ def _install_template( raise FileExistsError(f"{template_type} already exists") with ExitStack() as stack: - input_file = stack.enter_context(open(src, "rb")) + if template_type == "issue": + input_file = stack.enter_context(open(src, "rb")) + output_file = stack.enter_context(open(dest, "wb")) + output_file.write(input_file.read()) + return + output_file = stack.enter_context(open(dest, "wb")) - output_file.write(input_file.read()) + rendered_string = _render_template(src=src, stack=stack) + output_file.write(rendered_string.encode("utf-8")) def _select_templates(template: str, pkg: str, template_type: str) -> Mapping[str, Any]: diff --git a/exasol/toolbox/tools/workflow.py b/exasol/toolbox/tools/workflow.py index aaa094f30..b255462c4 100644 --- a/exasol/toolbox/tools/workflow.py +++ b/exasol/toolbox/tools/workflow.py @@ -1,3 +1,4 @@ +import warnings from pathlib import Path import typer @@ -18,6 +19,11 @@ def list_workflows( ) -> None: """List all available workflows.""" template.list_templates(columns=columns, pkg=PKG) + warnings.warn( + "\033[31m`tbx workflow list` is deprecated; this will be removed after 2026-04-22\033[0m", + category=FutureWarning, + stacklevel=2, + ) @CLI.command(name="show") @@ -28,6 +34,11 @@ def show_workflow( template.show_templates( template=workflow, pkg=PKG, template_type=TEMPLATE_TYPE, lexer=LEXER ) + warnings.warn( + "\033[31m`tbx workflow show` is deprecated; this will be removed after 2026-04-22\033[0m", + category=FutureWarning, + stacklevel=2, + ) @CLI.command(name="diff") @@ -42,6 +53,11 @@ def diff_workflow( template.diff_template( template=workflow, dest=dest, pkg=PKG, template_type=TEMPLATE_TYPE ) + warnings.warn( + "\033[31m`tbx workflow diff` is deprecated; this will be removed after 2026-04-22\033[0m", + category=FutureWarning, + stacklevel=2, + ) @CLI.command(name="install") @@ -59,6 +75,11 @@ def install_workflow( template.install_template( template=workflow, dest=dest, pkg=PKG, template_type=TEMPLATE_TYPE ) + warnings.warn( + "\033[31m`tbx workflow install` is deprecated; this will be replaced by a nox session after 2026-04-22\033[0m", + category=FutureWarning, + stacklevel=2, + ) @CLI.command(name="update") @@ -79,6 +100,11 @@ def update_workflow( pkg=PKG, template_type=TEMPLATE_TYPE, ) + warnings.warn( + "\033[31m`tbx workflow update` is deprecated; this will be replaced by a nox session after 2026-04-22\033[0m", + category=FutureWarning, + stacklevel=2, + ) if __name__ == "__main__": diff --git a/poetry.lock b/poetry.lock index f8599d22d..029cfc7cf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3772,6 +3772,18 @@ rich = ">=10.11.0" shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250915" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6"}, + {file = "types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3"}, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -3911,4 +3923,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "7a9dd85bb043ddc5bed4f76d6ac64a9a77c7f2089b77ff1a9c57246b4ca57ab6" +content-hash = "27476fa095d516f571fec7473aa0018edf057a53bf5a2934182915d937997ef0" diff --git a/pyproject.toml b/pyproject.toml index 01596f377..d0a819736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "pysonar>=1.0.1.1548,<2", "pytest>=7.2.2,<10", "pyupgrade>=2.38.2,<4.0.0", + "pyyaml (>=6.0.3,<7.0.0)", "ruff>=0.14.5,<0.15", "shibuya>=2024.5.14", "sphinx>=5.3,<8", @@ -66,6 +67,7 @@ sphinx-multiversion = "exasol.toolbox.sphinx.multiversion:main" dev = [ "autoimport>=1.4.0,<2", "cookiecutter>=2.6.0,<3", + "types-pyyaml (>=6.0.12.20250915,<7.0.0.0)", ] [tool.poetry] diff --git a/test/integration/tools/workflow_test.py b/test/integration/tools/workflow_test.py index a96c5625b..a49fb7e23 100644 --- a/test/integration/tools/workflow_test.py +++ b/test/integration/tools/workflow_test.py @@ -1,5 +1,7 @@ from unittest.mock import patch +import pytest + from exasol.toolbox.tools.workflow import CLI @@ -46,11 +48,29 @@ def test_show_workflow(cli_runner): assert "name: Checks " in result.output -def test_diff_workflow(cli_runner, tmp_path): +@pytest.mark.parametrize( + "workflow", + [ + "build-and-publish", + "cd", + "check-release-tag", + "checks", + "ci", + "gh-pages", + "matrix-all", + "matrix-exasol", + "matrix-python", + "merge-gate", + "pr-merge", + "report", + "slow-checks", + ], +) +def test_diff_workflow(cli_runner, tmp_path, workflow): # set up with file in tmp_path so checks files are the same - cli_runner.invoke(CLI, ["install", "checks", str(tmp_path)]) + cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)]) - result = cli_runner.invoke(CLI, ["diff", "checks", str(tmp_path)]) + result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)]) assert result.exit_code == 0 # as the files are the same, we expect no difference diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 6c9c0ecb2..c2790dbb4 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -6,6 +6,7 @@ from exasol.toolbox.config import ( DEFAULT_EXCLUDED_PATHS, BaseConfig, + DependencyManager, valid_version_string, ) from exasol.toolbox.nox.plugin import hookimpl @@ -21,6 +22,7 @@ def test_works_as_defined(test_project_config_factory): assert config.model_dump() == { "add_to_excluded_python_paths": (), "create_major_version_tags": False, + "dependency_manager": {"name": "poetry", "version": "2.3.0"}, "documentation_path": root_path / "doc", "exasol_versions": ("7.1.30", "8.29.13", "2025.1.8"), "excluded_python_paths": ( @@ -32,7 +34,13 @@ def test_works_as_defined(test_project_config_factory): "dist", "venv", ), + "github_template_dict": { + "dependency_manager_version": "2.3.0", + "minimum_python_version": "3.10", + "os_version": "ubuntu-24.04", + }, "minimum_python_version": "3.10", + "os_version": "ubuntu-24.04", "plugins_for_nox_sessions": (), "project_name": "test", "python_versions": ("3.10", "3.11", "3.12", "3.13", "3.14"), @@ -88,6 +96,20 @@ def test_expansion_validation_fails_for_invalid_version(): BaseConfigExpansion(python_versions=("1.f.0",)) +class TestOsVersion: + @staticmethod + @pytest.mark.parametrize("os_version", ["ubuntu-24.04", "ubuntu-20.10"]) + def test_works_as_expected(test_project_config_factory, os_version): + test_project_config_factory(os_version=os_version) + + @staticmethod + @pytest.mark.parametrize("os_version", ["ubunt-24.04", "windows-2025", "macos-15"]) + def test_fails_when_pattern_not_matched(test_project_config_factory, os_version): + with pytest.raises(ValidationError) as ex: + test_project_config_factory(os_version=os_version) + assert "String should match pattern '^ubuntu-.*'" in str(ex) + + def test_minimum_python_version(test_project_config_factory): conf = test_project_config_factory(python_versions=("5.5.5", "1.10", "9.9.9")) assert conf.minimum_python_version == "1.10" @@ -166,3 +188,35 @@ def test_raises_exception_without_hook(test_project_config_factory): with pytest.raises(ValidationError) as ex: test_project_config_factory(plugins_for_nox_sessions=(WithoutHook,)) assert "No methods in `WithoutHook`" in str(ex.value) + + +class TestDependencyManager: + @staticmethod + @pytest.mark.parametrize("version", ["2.1.4", "2.3.0", "2.9.9"]) + def test_works_as_expected(version): + DependencyManager(name="poetry", version=version) + + @staticmethod + def test_raises_exception_when_not_supported_name(): + with pytest.raises(ValidationError) as ex: + DependencyManager(name="uv", version="2.3.0") + assert "Input should be 'poetry'" in str(ex.value) + + @staticmethod + def test_raises_exception_when_version_too_high(): + with pytest.raises(ValidationError) as ex: + DependencyManager(name="poetry", version="3.1.0") + assert "Poetry version must be <" in str(ex.value) + + @staticmethod + def test_raises_exception_when_version_too_low(): + with pytest.raises(ValidationError) as ex: + DependencyManager(name="poetry", version="2.1.0") + assert "Poetry version must be >=" in str(ex.value) + + @staticmethod + def test_gives_warning_when_in_ok_range_but_above_last_tested(capsys): + with pytest.warns( + UserWarning, match="Poetry version exceeds last tested version" + ): + DependencyManager(name="poetry", version="2.4.0") diff --git a/test/unit/tools/test_template.py b/test/unit/tools/test_template.py new file mode 100644 index 000000000..c30e75582 --- /dev/null +++ b/test/unit/tools/test_template.py @@ -0,0 +1,101 @@ +from contextlib import ExitStack +from inspect import cleandoc +from pathlib import Path + +import pytest +from yaml.parser import ParserError + +from exasol.toolbox.tools.template import ( + _render_template, + _templates, +) +from exasol.toolbox.tools.workflow import PKG + +RENDERED_TEMPLATE = """ +name: Publish Documentation + +on: + workflow_call: + workflow_dispatch: + +jobs: + + build-documentation: + runs-on: "ubuntu-24.04" + permissions: + contents: read + steps: + - name: SCM Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v4 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Build Documentation + run: | + poetry run -- nox -s docs:multiversion + mv .html-documentation html-documentation + + - name: Upload artifact + uses: actions/upload-pages-artifact@v4 + with: + path: html-documentation + + deploy-documentation: + needs: [ build-documentation ] + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: "ubuntu-24.04" + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + +""" + +BAD_TEMPLATE = """ +name: Publish Documentation + +on: + workflow_call: + workflow_dispatch: + +jobs: + + build-documentation: + runs-on: "ubuntu-24.04" + permissions: + contents: read + steps: + - name: SCM Checkout + uses: actions/checkout@v5 +""" + + +class TestRenderTemplate: + pkg = PKG + template = "gh-pages" + + def test_works_as_expected(self): + src = Path(_templates(self.pkg)[self.template]) + with ExitStack() as stack: + rendered_str = _render_template(src=src, stack=stack) + assert rendered_str == cleandoc(RENDERED_TEMPLATE) + "\n" + + @staticmethod + def test_fails_when_yaml_malformed(tmp_path): + file_path = tmp_path / "test.yaml" + file_path.write_text(BAD_TEMPLATE) + with pytest.raises(ParserError, match="while parsing a block collection"): + with ExitStack() as stack: + _render_template(src=file_path, stack=stack)