diff --git a/.gitignore b/.gitignore index e40e99b..5a0a091 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ htmlcov/ # Documentation builds docs/_build/ +site/ # Logs server_logs/ diff --git a/Makefile b/Makefile index d520313..9057ce2 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,27 @@ else endif docs: - cd docs && $(MAKE) html + @echo "Installing documentation dependencies..." + $(VENV_PIP) install -r requirements-docs.txt + @echo "Building documentation..." + mkdocs build + +docs-serve: + @echo "Installing documentation dependencies..." + $(VENV_PIP) install -r requirements-docs.txt + @echo "Starting documentation server..." + mkdocs serve + +docs-deploy: + @echo "Installing documentation dependencies..." + $(VENV_PIP) install -r requirements-docs.txt + @echo "Deploying documentation..." + mkdocs gh-deploy + +docs-api: + @echo "Generating API documentation..." + $(VENV_PYTHON) generate_api_docs.py + @echo "API documentation generated successfully!" lint: flake8 src tests diff --git a/README.md b/README.md index 2059bdf..c8b9cda 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ python -m build ```bash # Run a stress test using a YAML configuration -stress-run examples/api_test.yaml --users 500 --duration 60 +stress-run --test_config examples/api_test.yaml --request_config examples/requests.yaml # Start the monitoring dashboard stress-dashboard --mode websocket # or --mode bokeh diff --git a/docs/api/Http.md b/docs/api/Http.md new file mode 100644 index 0000000..52a3b1f --- /dev/null +++ b/docs/api/Http.md @@ -0,0 +1,18 @@ +# Http + +This page contains the API documentation for the `gradual.runners.request.Http` module. + +::: gradual.runners.request.Http + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.request.Http] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/SocketIO.md b/docs/api/SocketIO.md new file mode 100644 index 0000000..9906ca8 --- /dev/null +++ b/docs/api/SocketIO.md @@ -0,0 +1,18 @@ +# Socketio + +This page contains the API documentation for the `gradual.runners.request.SocketIO` module. + +::: gradual.runners.request.SocketIO + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.request.SocketIO] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/adapters.md b/docs/api/adapters.md new file mode 100644 index 0000000..f964ae1 --- /dev/null +++ b/docs/api/adapters.md @@ -0,0 +1,18 @@ +# Adapters + +This page contains the API documentation for the `gradual.reporting.adapters` module. + +::: gradual.reporting.adapters + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.reporting.adapters] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/base.md b/docs/api/base.md new file mode 100644 index 0000000..d26f7df --- /dev/null +++ b/docs/api/base.md @@ -0,0 +1,18 @@ +# Base + +This page contains the API documentation for the `gradual.runners.request.base` module. + +::: gradual.runners.request.base + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.request.base] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/configs.md b/docs/api/configs.md new file mode 100644 index 0000000..c435f30 --- /dev/null +++ b/docs/api/configs.md @@ -0,0 +1,18 @@ +# Configs + +This page contains the API documentation for the `gradual.configs` module. + +::: gradual.configs + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_", "!^gradual.configs.parser", "!^gradual.configs.phase", "!^gradual.configs.request", "!^gradual.configs.scenario", "!^gradual.configs.validate"] + preload_modules: [gradual.configs] + merge_init_into_class: true + show_submodules: false + show_if_no_docstring: true diff --git a/docs/api/constants.md b/docs/api/constants.md new file mode 100644 index 0000000..0ef1cdb --- /dev/null +++ b/docs/api/constants.md @@ -0,0 +1,18 @@ +# Constants + +This page contains the API documentation for the `gradual.constants` module. + +::: gradual.constants + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_", "!^gradual.constants.request_types"] + preload_modules: [gradual.constants] + merge_init_into_class: true + show_submodules: false + show_if_no_docstring: true diff --git a/docs/api/exceptions.md b/docs/api/exceptions.md new file mode 100644 index 0000000..827a599 --- /dev/null +++ b/docs/api/exceptions.md @@ -0,0 +1,18 @@ +# Exceptions + +This page contains the API documentation for the `gradual.exceptions` module. + +::: gradual.exceptions + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.exceptions] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/iterators.md b/docs/api/iterators.md new file mode 100644 index 0000000..b1d1bc5 --- /dev/null +++ b/docs/api/iterators.md @@ -0,0 +1,18 @@ +# Iterators + +This page contains the API documentation for the `gradual.runners.iterators` module. + +::: gradual.runners.iterators + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.iterators] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/logger.md b/docs/api/logger.md new file mode 100644 index 0000000..a9ef90c --- /dev/null +++ b/docs/api/logger.md @@ -0,0 +1,18 @@ +# Logger + +This page contains the API documentation for the `gradual.reporting.logger` module. + +::: gradual.reporting.logger + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.reporting.logger] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/logging.md b/docs/api/logging.md new file mode 100644 index 0000000..7593ed6 --- /dev/null +++ b/docs/api/logging.md @@ -0,0 +1,18 @@ +# Logging + +This page contains the API documentation for the `gradual.reporting.adapters.logging` module. + +::: gradual.reporting.adapters.logging + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.reporting.adapters.logging] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/orchestrator.md b/docs/api/orchestrator.md new file mode 100644 index 0000000..52f8c22 --- /dev/null +++ b/docs/api/orchestrator.md @@ -0,0 +1,18 @@ +# Orchestrator + +This page contains the API documentation for the `gradual.base.orchestrator` module. + +::: gradual.base.orchestrator + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.base.orchestrator] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/overview.md b/docs/api/overview.md new file mode 100644 index 0000000..c47abcf --- /dev/null +++ b/docs/api/overview.md @@ -0,0 +1,49 @@ +# API Reference + +This page provides comprehensive API documentation for the gradual package. + +## Available Modules + +The following modules are automatically discovered and documented: + +- [Base](base.md) - `gradual.base` +- [Configs](configs.md) - `gradual.configs` +- [Constants](constants.md) - `gradual.constants` +- [Exceptions](exceptions.md) - `gradual.exceptions` +- [Reporting](reporting.md) - `gradual.reporting` +- [Runners](runners.md) - `gradual.runners` +- [Parser](parser.md) - `gradual.configs.parser` +- [Phase](phase.md) - `gradual.configs.phase` +- [Request](request.md) - `gradual.configs.request` +- [Scenario](scenario.md) - `gradual.configs.scenario` +- [Validate](validate.md) - `gradual.configs.validate` +- [Orchestrator](orchestrator.md) - `gradual.base.orchestrator` +- [Request_Types](request_types.md) - `gradual.constants.request_types` +- [Adapters](adapters.md) - `gradual.reporting.adapters` +- [Logger](logger.md) - `gradual.reporting.logger` +- [Stats](stats.md) - `gradual.reporting.stats` +- [Base](base.md) - `gradual.reporting.adapters.base` +- [Logging](logging.md) - `gradual.reporting.adapters.logging` +- [Iterators](iterators.md) - `gradual.runners.iterators` +- [Phase](phase.md) - `gradual.runners.phase` +- [Request](request.md) - `gradual.runners.request` +- [Runner](runner.md) - `gradual.runners.runner` +- [Scenario](scenario.md) - `gradual.runners.scenario` +- [Session](session.md) - `gradual.runners.session` +- [Http](Http.md) - `gradual.runners.request.Http` +- [Socketio](SocketIO.md) - `gradual.runners.request.SocketIO` +- [Base](base.md) - `gradual.runners.request.base` + + +## Automatic Discovery + +This documentation is automatically generated using a Python script that: +1. Discovers all modules in the package +2. Generates individual documentation files for each module +3. Uses mkdocstrings to extract documentation from docstrings +4. Updates automatically when you run the script + +To regenerate all API documentation, run: +```bash +python generate_api_docs.py +``` diff --git a/docs/api/parser.md b/docs/api/parser.md new file mode 100644 index 0000000..491ab43 --- /dev/null +++ b/docs/api/parser.md @@ -0,0 +1,18 @@ +# Parser + +This page contains the API documentation for the `gradual.configs.parser` module. + +::: gradual.configs.parser + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.configs.parser] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/phase.md b/docs/api/phase.md new file mode 100644 index 0000000..ca365f3 --- /dev/null +++ b/docs/api/phase.md @@ -0,0 +1,18 @@ +# Phase + +This page contains the API documentation for the `gradual.runners.phase` module. + +::: gradual.runners.phase + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.phase] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/reporting.md b/docs/api/reporting.md new file mode 100644 index 0000000..7da05a3 --- /dev/null +++ b/docs/api/reporting.md @@ -0,0 +1,18 @@ +# Reporting + +This page contains the API documentation for the `gradual.reporting` module. + +::: gradual.reporting + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_", "!^gradual.reporting.stats", "!^gradual.reporting.logger", "!^gradual.reporting.adapters"] + preload_modules: [gradual.reporting] + merge_init_into_class: true + show_submodules: false + show_if_no_docstring: true diff --git a/docs/api/request.md b/docs/api/request.md new file mode 100644 index 0000000..4160882 --- /dev/null +++ b/docs/api/request.md @@ -0,0 +1,18 @@ +# Request + +This page contains the API documentation for the `gradual.runners.request` module. + +::: gradual.runners.request + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_", "!^gradual.runners.request.Http", "!^gradual.runners.request.SocketIO"] + preload_modules: [gradual.runners.request] + merge_init_into_class: true + show_submodules: false + show_if_no_docstring: true diff --git a/docs/api/request_types.md b/docs/api/request_types.md new file mode 100644 index 0000000..202e579 --- /dev/null +++ b/docs/api/request_types.md @@ -0,0 +1,18 @@ +# Request_Types + +This page contains the API documentation for the `gradual.constants.request_types` module. + +::: gradual.constants.request_types + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.constants.request_types] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/runner.md b/docs/api/runner.md new file mode 100644 index 0000000..a169e1e --- /dev/null +++ b/docs/api/runner.md @@ -0,0 +1,18 @@ +# Runner + +This page contains the API documentation for the `gradual.runners.runner` module. + +::: gradual.runners.runner + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.runner] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/runners.md b/docs/api/runners.md new file mode 100644 index 0000000..3c53de9 --- /dev/null +++ b/docs/api/runners.md @@ -0,0 +1,18 @@ +# Runners + +This page contains the API documentation for the `gradual.runners` module. + +::: gradual.runners + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_", "!^gradual.runners.scenario", "!^gradual.runners.session", "!^gradual.runners.iterators", "!^gradual.runners.phase", "!^gradual.runners.runner", "!^gradual.runners.request"] + preload_modules: [gradual.runners] + merge_init_into_class: true + show_submodules: false + show_if_no_docstring: true diff --git a/docs/api/scenario.md b/docs/api/scenario.md new file mode 100644 index 0000000..f3729a3 --- /dev/null +++ b/docs/api/scenario.md @@ -0,0 +1,18 @@ +# Scenario + +This page contains the API documentation for the `gradual.runners.scenario` module. + +::: gradual.runners.scenario + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.scenario] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/session.md b/docs/api/session.md new file mode 100644 index 0000000..fdabd2d --- /dev/null +++ b/docs/api/session.md @@ -0,0 +1,18 @@ +# Session + +This page contains the API documentation for the `gradual.runners.session` module. + +::: gradual.runners.session + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.runners.session] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/stats.md b/docs/api/stats.md new file mode 100644 index 0000000..737eae3 --- /dev/null +++ b/docs/api/stats.md @@ -0,0 +1,18 @@ +# Stats + +This page contains the API documentation for the `gradual.reporting.stats` module. + +::: gradual.reporting.stats + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.reporting.stats] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api/validate.md b/docs/api/validate.md new file mode 100644 index 0000000..26cb296 --- /dev/null +++ b/docs/api/validate.md @@ -0,0 +1,18 @@ +# Validate + +This page contains the API documentation for the `gradual.configs.validate` module. + +::: gradual.configs.validate + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [gradual.configs.validate] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true diff --git a/docs/configuration_reference.md b/docs/configuration_reference.md new file mode 100644 index 0000000..26536bb --- /dev/null +++ b/docs/configuration_reference.md @@ -0,0 +1,560 @@ +# Configuration Reference + +This document provides a comprehensive reference for all configuration options available in Gradual. It's designed to be your go-to resource for understanding, configuring, and troubleshooting Gradual stress tests. + +> **šŸ’” This is the dedicated Configuration Reference guide. For a quick overview, see the [User Guide](user_guide.md).** + +## Overview + +Gradual uses a two-file configuration system: + +1. **Test Configuration** (`test_config.yaml`) - Defines the overall test structure, phases, and scenarios +2. **Request Configuration** (`request_config.yaml`) - Defines individual HTTP and WebSocket requests and their parameters + +## Command-Line Usage + +The `stress-run` command is the main entry point for running stress tests. It accepts the following arguments: + +```bash +stress-run --test_config [--request_config ] +``` + +### Arguments + +| Argument | Type | Required | Description | +|----------|------|----------|-------------| +| `test_config` | path | Yes | Path to the main test configuration file (YAML) | +| `request_config` | path | No | Path to the request configuration file (YAML). If not provided, requests must be defined inline in the test configuration. | + +### Examples + +```bash +# Run with both configuration files +stress-run --test_config test_config.yaml --request_config request_config.yaml + +# Run with only test configuration (requests defined inline) +stress-run --test_config test_config.yaml + +# Run with relative paths +stress-run --test_config ./configs/test.yaml --request_config ./configs/requests.yaml +``` + +### Dashboard Command + +The `stress-dashboard` command provides real-time monitoring: + +```bash +# Start Bokeh dashboard (interactive charts) +stress-dashboard --mode bokeh --port 8080 + +# Start WebSocket dashboard (lightweight) +stress-dashboard --mode websocket --port 8081 + +# Start with custom host +stress-dashboard --mode bokeh --host 0.0.0.0 --port 8080 +``` + +> **āš ļø Important:** The CLI does not support command-line overrides for configuration values. All test parameters must be defined in the YAML configuration files. + +### CLI Implementation Details + +The current CLI implementation is intentionally simple and focused: + +- **No command-line overrides**: All test parameters (concurrency, duration, ramp-up, etc.) must be defined in YAML files +- **No output format options**: Results are displayed in the console by default +- **No debug flags**: Logging level is controlled by the configuration files +- **No report generation**: Use the dashboard for real-time monitoring and analysis + +This design ensures that: + +- Tests are reproducible and version-controlled +- Configuration is explicit and documented +- Test behavior is consistent across different environments +- Users focus on proper test design rather than command-line tweaking + +### CLI Source Code + +The CLI is implemented in `scripts/run_stress_test.py` and provides a minimal interface: + +```python +parser = ArgumentParser() +parser.add_argument( + "--test_config", + type=pathlib.Path, + required=True, + help="Path to the test configuration file", +) +parser.add_argument( + "--request_config", + type=pathlib.Path, + help="Path to the request configuration file" +) +``` + +This simplicity allows for: + +- Easy integration with CI/CD pipelines +- Consistent behavior across different environments +- Clear separation of concerns between CLI and configuration +- Focus on test design rather than command-line complexity + +## Test Configuration File + +The test configuration file is the main configuration file that defines how your stress test will be executed. + +### Top-Level Structure + +```yaml +runs: + name: "Test Run Name" + wait_between_phases: 10 + phases: + "phase_name": + # Phase configuration +``` + +### Run Configuration + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Name of the test run | +| `wait_between_phases` | integer | No | Wait time between phases in seconds (default: 0) | + +### Phase Configuration + +Each phase represents a distinct period of testing with specific load characteristics. + +```yaml +"phase_name": + scenarios: + "scenario_name": + # Scenario configuration + run_time: 300 +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `scenarios` | object | Yes | Dictionary of scenarios to run in this phase | +| `run_time` | integer | Yes | Duration of the phase in seconds | + +### Scenario Configuration + +Scenarios define specific test patterns with their own concurrency and ramp-up settings. + +```yaml +"scenario_name": + requests: + - "request1" + - "request2" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: [1, 2, 4, 8] + ramp_up_wait: [5, 5, 5, 5] + iterate_through_requests: true + run_once: false +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `requests` | list[string] or "FROM_REQUEST_YAML_FILE" | Yes | List of request names (must match request config) or "FROM_REQUEST_YAML_FILE" to use all requests from request config | +| `min_concurrency` | integer | Yes | Starting number of concurrent requests | +| `max_concurrency` | integer | Yes | Maximum number of concurrent requests | +| `ramp_up_multiply` | integer or list[integer] | No* | [Multiplicative Ramp-up](#multiplicative-ramp-up): Multipliers for gradually increasing concurrency. | +| `ramp_up_add` | integer or list[integer] | No* | [Additive Ramp-up](#additive-ramp-up): Additive values for gradually increasing concurrency. | +| `ramp_up_wait` | integer or list[integer] | No | Wait time between ramp-up steps in seconds (default: [0.1]). | +| `iterate_through_requests` | boolean | No | Whether to cycle through requests sequentially (default: false) | +| `run_once` | boolean | No | Whether to run the scenario only once (default: false) | + +*Note: Either `ramp_up_multiply` or `ramp_up_add` must be specified, but not both. + +## Request Configuration File + +The request configuration file defines individual HTTP and WebSocket requests that can be referenced by scenarios. The system supports both traditional HTTP requests and WebSocket connections for real-time communication testing. Additionally, a plugin feature is in development that will allow users to run custom I/O requests based on their specific requirements. + +### Structure + +```yaml +requests: + "request_name": + url: "http://example.com/api/endpoint" + method: "GET" + expected_response_time: 1.0 + auth: null + params: + key: "value" +``` + +### Request Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `url` | string | Yes | Full URL for the request (HTTP) or WebSocket endpoint (ws:// or wss://) | +| `method` | string | Yes | HTTP method (GET, POST, PUT, DELETE, etc.) or "WEBSOCKET" for WebSocket connections | +| `expected_response_time` | number | Yes | Expected response time in seconds | +| `auth` | object/null | No | Authentication configuration (null for none) | +| `params` | object | No | Request parameters (for GET) or body data (for POST/PUT) | +| `websocket_config` | object | No | WebSocket-specific configuration (only for WebSocket requests) | + +**Note:** For WebSocket requests, set `method: "WEBSOCKET"` and optionally include `websocket_config` for custom WebSocket behavior. The plugin feature will provide additional request types and custom I/O capabilities. + +### WebSocket Configuration + +For WebSocket requests, you can specify additional configuration options: + +```yaml +requests: + "websocket_test": + url: "ws://localhost:8080/ws" + method: "WEBSOCKET" + expected_response_time: 2.0 + websocket_config: + message_interval: 1.0 # Send message every 1 second + max_messages: 100 # Maximum messages to send + message_template: "Test message {counter}" + close_after: 60 # Close connection after 60 seconds +``` + +### Plugin Feature (Work in Progress) + +The plugin system is currently under development and will enable users to: +- Create custom request types beyond HTTP and WebSocket +- Implement custom I/O operations (file I/O, database operations, etc.) +- Extend the system with domain-specific testing capabilities +- Integrate with external systems and APIs + +This feature will provide a flexible framework for testing various types of systems and protocols. + +## Ramp-up Strategies + +Gradual supports two types of ramp-up strategies to gradually increase load on your system. + +### Multiplicative Ramp-up + +Use `ramp_up_multiply` to multiply the current request count by specified factors: + +```yaml +ramp_up_multiply: [1, 2, 4, 8, 16] +ramp_up_wait: [5, 5, 5, 5, 5] +``` + +**Execution Flow:** + +```mermaid +flowchart LR + A[Start: 1 request] --> B[Wait 5s] --> C[Multiply by 2
Now: 2 requests] --> D[Wait 5s] --> E[Multiply by 4
Now: 8 requests] --> F[Wait 5s] --> G[Multiply by 8
Now: 64 requests] --> H[Wait 5s] --> I[Multiply by 16
Now: 1024 requests] + + style A fill:#e1f5fe,font-size:50px + style I fill:#ffebee,font-size:50px + style B fill:#f3e5f5,font-size:50px + style D fill:#f3e5f5,font-size:50px + style F fill:#f3e5f5,font-size:50px + style H fill:#f3e5f5,font-size:50px + style C fill:#f9f9f9,font-size:50px + style E fill:#f9f9f9,font-size:50px + style G fill:#f9f9f9,font-size:50px +``` + + +### Additive Ramp-up + +Use `ramp_up_add` to add specified numbers of requests: + +```yaml +ramp_up_add: [1, 2, 3, 4, 5] +ramp_up_wait: [2, 2, 2, 2, 2] +``` + +**Execution Flow:** + +```mermaid +flowchart LR + A[Start: 1 request] --> B[Wait 2s] --> C[Add 1
Now: 2 requests] --> D[Wait 2s] --> E[Add 2
Now: 4 requests] --> F[Wait 2s] --> G[Add 3
Now: 7 requests] --> H[Wait 2s] --> I[Add 4
Now: 11 requests] --> J[Wait 2s] --> K[Add 5
Now: 16 requests] + + style A fill:#e1f5fe,font-size:50px + style K fill:#ffebee,font-size:50px + style B fill:#f3e5f5,font-size:50px + style D fill:#f3e5f5,font-size:50px + style F fill:#f3e5f5,font-size:50px + style H fill:#f3e5f5,font-size:50px + style J fill:#f3e5f5,font-size:50px + style C fill:#f9f9f9,font-size:50px + style E fill:#f9f9f9,font-size:50px + style G fill:#f9f9f9,font-size:50px + style I fill:#f9f9f9,font-size:50px +``` + + +### Ramp-up Wait Times + +The `ramp_up_wait` field can be: + +- **Single value**: Applied to all ramp-up steps +- **List of values**: Applied to each ramp-up step individually + +```yaml +# Single value - wait 5 seconds between all steps +ramp_up_wait: 5 + +# List of values - wait different times between steps +ramp_up_wait: [5, 10, 15, 20] +``` + +## Configuration Examples + +### Simple Load Test + +```yaml +# test_config.yaml +runs: + name: "Simple API Test" + wait_between_phases: 0 + phases: + "main_test": + scenarios: + "api_scenario": + requests: + - "health_check" + - "get_data" + min_concurrency: 1 + max_concurrency: 50 + ramp_up_multiply: [1, 2, 5, 10, 25, 50] + ramp_up_wait: [10, 10, 10, 10, 10, 10] + iterate_through_requests: true + run_time: 300 + +# Mixed HTTP and WebSocket test +runs: + name: "Mixed Protocol Test" + wait_between_phases: 5 + phases: + "http_phase": + scenarios: + "http_scenario": + requests: + - "api_endpoint" + - "data_fetch" + min_concurrency: 10 + max_concurrency: 100 + ramp_up_multiply: [1, 2, 4, 8, 16, 32, 64, 100] + ramp_up_wait: 5 + run_time: 120 + "websocket_phase": + scenarios: + "websocket_scenario": + requests: + - "websocket_connection" + - "realtime_data" + min_concurrency: 5 + max_concurrency: 50 + ramp_up_add: [5, 10, 15, 20] + ramp_up_wait: 10 + run_time: 180 +``` + +```yaml +# request_config.yaml +requests: + "health_check": + url: "http://localhost:8000/health" + method: "GET" + expected_response_time: 0.5 + auth: null + "get_data": + url: "http://localhost:8000/api/data" + method: "GET" + expected_response_time: 1.0 + auth: null +``` + +### Multi-Phase Stress Test + +```yaml +# test_config.yaml +runs: + name: "Stress Test with Recovery" + wait_between_phases: 30 + phases: + "warm_up": + scenarios: + "light_load": + requests: + - "health_check" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: [1, 2, 5, 10] + ramp_up_wait: [5, 5, 5, 5] + iterate_through_requests: true + run_time: 120 + "peak_load": + scenarios: + "heavy_load": + requests: + - "complex_operation" + - "data_processing" + min_concurrency: 10 + max_concurrency: 200 + ramp_up_multiply: [10, 25, 50, 100, 200] + ramp_up_wait: [15, 15, 15, 15, 15] + iterate_through_requests: true + run_time: 600 + "recovery": + scenarios: + "recovery_test": + requests: + - "health_check" + min_concurrency: 200 + max_concurrency: 1 + ramp_up_add: [-50, -50, -50, -49] + ramp_up_wait: [10, 10, 10, 10] + iterate_through_requests: true + run_time: 180 +``` + +```yaml +# request_config.yaml +requests: + "health_check": + url: "http://localhost:8000/health" + method: "GET" + expected_response_time: 0.5 + auth: null + "complex_operation": + url: "http://localhost:8000/api/complex" + method: "POST" + expected_response_time: 5.0 + auth: null + params: + operation: "heavy_computation" + data_size: 1000 + "data_processing": + url: "http://localhost:8000/api/process" + method: "POST" + expected_response_time: 3.0 + auth: null + params: + batch_size: 100 + priority: "high" +``` + +### Authentication Example + +```yaml +# request_config.yaml +requests: + "authenticated_endpoint": + url: "http://localhost:8000/api/protected" + method: "GET" + expected_response_time: 1.0 + auth: + type: "bearer" + token: "your_auth_token" + params: {} +``` + +## Advanced Configuration + +### Dynamic Request Loading + +For advanced use cases, you can reference requests from external files: + +```yaml +# In test_config.yaml +scenarios: + "dynamic_scenario": + requests: "FROM_REQUEST_YAML_FILE" + request_file: path/to/external_requests.yaml + # ... other scenario configuration +``` + +### Single vs. List Values + +Many configuration fields accept both single values and lists. When a single value is provided, it's automatically converted to a list: + +```yaml +# These are equivalent: +ramp_up_multiply: 5 +ramp_up_multiply: [5] + +# These are equivalent: +ramp_up_wait: 10 +ramp_up_wait: [10] +``` + +## Validation Rules + +### Required Fields + +- **Test Config**: `runs.name`, `runs.phases`, `phases.scenarios`, `scenarios.requests`, `scenarios.min_concurrency`, `scenarios.max_concurrency`, `phases.run_time` +- **Request Config**: `requests.url`, `requests.method`, `requests.expected_response_time` + +### Value Constraints + +- **Concurrency**: Must be positive integers +- **Timing**: Must be positive numbers +- **Ramp-up**: Must specify either `ramp_up_multiply` or `ramp_up_add`, but not both +- **Request References**: Request names in scenarios must exist in the request configuration + +### Best Practices + +1. **Naming**: Use descriptive names for phases, scenarios, and requests +2. **Structure**: Keep test structure and request definitions separate +3. **Reusability**: Define requests once and reuse across multiple scenarios +4. **Ramp-up**: Start with conservative ramp-up values and adjust based on system performance +5. **Phases**: Use phases to test different load patterns and allow system recovery +6. **Documentation**: Add comments to complex configurations for clarity + +### Configuration-First Approach + +Since the CLI doesn't support command-line overrides, adopt a configuration-first approach: + +- **Version Control**: Keep all test configurations in version control for reproducibility +- **Environment-Specific Configs**: Create separate configuration files for different environments (dev, staging, prod) +- **Configuration Templates**: Use base templates and extend them for specific test scenarios +- **Parameterization**: Use YAML anchors and aliases to avoid duplication +- **Validation**: Always validate your YAML syntax before running tests + +Example of using YAML anchors for reusability: + +```yaml +# test_config.yaml +runs: + name: "Parameterized Test" + wait_between_phases: 10 + phases: + "light_load": &light_load + scenarios: + "light_scenario": + requests: ["health_check", "simple_api"] + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: [1, 2, 5, 10] + ramp_up_wait: [5, 5, 5, 5] + iterate_through_requests: true + run_time: 120 + + "medium_load": + <<: *light_load # Inherit from light_load + scenarios: + "medium_scenario": + <<: *light_load.scenarios.light_scenario # Inherit scenario config + max_concurrency: 50 + ramp_up_multiply: [1, 5, 15, 30, 50] + ramp_up_wait: [10, 10, 10, 10, 10] + run_time: 300 +``` + +## Troubleshooting Configuration Issues + +### Common Errors + +1. **Missing Request Definition**: Ensure all request names in scenarios exist in the request configuration +2. **Invalid Ramp-up**: Check that either `ramp_up_multiply` or `ramp_up_add` is specified, but not both +3. **Type Mismatches**: Verify that numeric fields contain valid numbers +4. **File Paths**: Ensure configuration files are in the correct locations + +### Debug Tips + +- Use the `--debug` flag when running tests to see detailed configuration parsing +- Check YAML syntax with a YAML validator +- Verify file permissions and paths +- Test with minimal configurations first, then add complexity gradually diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..4746227 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,436 @@ +# Contributing to Gradual + +Thank you for your interest in contributing to the Gradual stress testing framework! This guide will help you get started with contributing to the project. + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [Contribution Workflow](#contribution-workflow) +- [Code Standards](#code-standards) +- [Testing Guidelines](#testing-guidelines) +- [Documentation](#documentation) +- [Release Process](#release-process) +- [Community](#community) + +## Code of Conduct + +This project adheres to a Code of Conduct. By participating, you are expected to uphold this code. Please report unacceptable behavior to the project maintainers. + +### Our Standards + +- **Be respectful** and inclusive +- **Be collaborative** and open to feedback +- **Be constructive** in criticism and suggestions +- **Be professional** in all interactions + +## Getting Started + +### Prerequisites + +- Python 3.9 or higher +- Git +- Basic understanding of Python development +- Familiarity with stress testing concepts (helpful but not required) + +### Areas to Contribute + +We welcome contributions in many areas: + +- **Core Framework**: Protocol handlers, assertions, data providers +- **Documentation**: User guides, API reference, examples +- **Testing**: Unit tests, integration tests, performance benchmarks +- **Examples**: Test configurations, use cases, tutorials +- **Bug Fixes**: Issue resolution, performance improvements +- **Features**: New protocols, reporting capabilities, integrations + +### Good First Issues + +Look for issues labeled with: +- `good first issue` - Beginner-friendly +- `help wanted` - General help needed +- `documentation` - Documentation improvements +- `bug` - Bug fixes + +## Development Setup + +### 1. Fork and Clone + +```bash +# Fork the repository on GitHub +# Then clone your fork +git clone https://github.com/YOUR_USERNAME/gradual.git +cd gradual + +# Add the upstream remote +git remote add upstream https://github.com/Gradual-Load-Testing/gradual.git +``` + +### 2. Environment Setup + +```bash +# Create virtual environment +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install development dependencies +pip install -e ".[dev]" + +# Install pre-commit hooks +pre-commit install +``` + +### 3. Verify Setup + +```bash +# Run tests to ensure everything works +make test + +# Run linting +make lint + +# Run type checking +make typecheck +``` + +## Contribution Workflow + +### 1. Create a Feature Branch + +```bash +# Update your main branch +git checkout main +git pull upstream main + +# Create a new feature branch +git checkout -b feature/your-feature-name +``` + +### 2. Make Your Changes + +- Write your code following the [Code Standards](#code-standards) +- Add tests for new functionality +- Update documentation as needed +- Ensure all tests pass + +### 3. Commit Your Changes + +```bash +# Stage your changes +git add . + +# Commit with a descriptive message +git commit -m "feat: add custom protocol handler support + +- Implement BaseProtocolHandler interface +- Add TCP protocol support +- Include comprehensive tests +- Update documentation" +``` + +### 4. Push and Create Pull Request + +```bash +# Push your branch +git push origin feature/your-feature-name + +# Create a Pull Request on GitHub +# Fill out the PR template with details about your changes +``` + +### 5. Code Review + +- Address any feedback from reviewers +- Make requested changes +- Ensure CI checks pass +- Maintainers will merge when ready + +## Code Standards + +### Python Style Guide + +We follow PEP 8 with some modifications: + +- **Line Length**: 88 characters (Black default) +- **Import Sorting**: isort with Black profile +- **Type Hints**: Required for public APIs +- **Docstrings**: Google-style docstrings + +### Code Formatting + +```bash +# Format code with Black +make format +# or +black src/ tests/ + +# Sort imports +make sort-imports +# or +isort src/ tests/ +``` + +### Linting and Type Checking + +```bash +# Run all quality checks +make quality + +# Run specific checks +make lint # flake8 +make typecheck # mypy +make test # pytest +``` + +### Pre-commit Hooks + +The project uses pre-commit hooks to ensure code quality: + +```bash +# Install hooks +pre-commit install + +# Run manually +pre-commit run --all-files +``` + +## Testing Guidelines + +### Test Structure + +```python +import pytest +from gradual.base import BaseProtocolHandler + +class TestCustomProtocolHandler: + """Test the custom protocol handler implementation.""" + + def test_initialization(self): + """Test handler initialization with config.""" + config = {"timeout": 30} + handler = CustomProtocolHandler(config) + assert handler.config == config + + def test_execute_request_success(self): + """Test successful request execution.""" + handler = CustomProtocolHandler({}) + result = handler.execute_request({"message": "test"}) + assert result["status"] == "success" + + def test_execute_request_failure(self): + """Test request execution failure.""" + handler = CustomProtocolHandler({}) + result = handler.execute_request({}) + assert result["status"] == "error" +``` + +### Test Requirements + +- **Coverage**: Aim for >90% code coverage +- **Naming**: Use descriptive test names and docstrings +- **Isolation**: Tests should be independent and repeatable +- **Mocking**: Mock external dependencies appropriately +- **Fixtures**: Use pytest fixtures for common setup + +### Running Tests + +```bash +# Run all tests +make test + +# Run specific test files +pytest tests/test_protocols.py + +# Run with coverage +pytest --cov=src/gradual --cov-report=html + +# Run performance tests +pytest tests/benchmarks/ -m "not slow" +``` + +## Documentation + +### Code Documentation + +- **Docstrings**: Use Google-style docstrings for all public methods +- **Type Hints**: Include type hints for function parameters and return values +- **Examples**: Provide usage examples in docstrings +- **Exceptions**: Document all exceptions that may be raised + +Example: + +```python +def execute_test_scenario(self, scenario: dict, user_data: dict) -> dict: + """Execute a test scenario with user data. + + Args: + scenario: Scenario configuration dictionary containing + steps, assertions, and metadata. + user_data: User-specific data for the scenario execution. + + Returns: + Dictionary containing execution results including + status, response time, and any extracted data. + + Raises: + ScenarioExecutionError: If scenario execution fails due to + configuration or runtime errors. + ValidationError: If response validation fails for any + assertion in the scenario. + + Example: + >>> scenario = {"steps": [{"request": {"method": "GET"}}]} + >>> user_data = {"user_id": "123"} + >>> result = runner.execute_test_scenario(scenario, user_data) + >>> print(result["status"]) + 'success' + """ + pass +``` + +### API Documentation + +- **Completeness**: Document all public APIs +- **Examples**: Include usage examples +- **Breaking Changes**: Document any breaking changes +- **Migration Guides**: Provide migration paths for major changes + +### User Documentation + +- **Clarity**: Write clear, concise instructions +- **Examples**: Provide real-world examples +- **Troubleshooting**: Include common issues and solutions +- **Screenshots**: Add visual aids where helpful + +## Pull Request Guidelines + +### PR Title and Description + +```markdown +## Description +Brief description of what this PR accomplishes. + +## Type of Change +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update + +## Testing +- [ ] Unit tests pass +- [ ] Integration tests pass +- [ ] Manual testing completed +- [ ] Documentation updated + +## Checklist +- [ ] Code follows the style guidelines +- [ ] Self-review of code completed +- [ ] Code is commented, particularly in hard-to-understand areas +- [ ] Corresponding changes to documentation made +- [ ] Changes generate no new warnings +- [ ] Tests added that prove fix is effective or feature works +- [ ] New and existing unit tests pass locally +- [ ] AI assistance (e.g., Cursor, ChatGPT, Claude) was used in making these changes (please specify tools in the PR description) +``` + +### Review Process + +1. **Automated Checks**: Ensure CI checks pass +2. **Code Review**: Address reviewer feedback +3. **Testing**: Verify functionality works as expected +4. **Documentation**: Update relevant documentation +5. **Merge**: Maintainers will merge when ready + +## Release Process + +### Version Management + +We use semantic versioning (MAJOR.MINOR.PATCH): + +- **MAJOR**: Breaking changes +- **MINOR**: New features (backward compatible) +- **PATCH**: Bug fixes (backward compatible) + +### Release Checklist + +- [ ] All tests pass +- [ ] Documentation is up to date +- [ ] Changelog is updated +- [ ] Version is bumped in `pyproject.toml` +- [ ] Release notes are written +- [ ] GitHub release is created +- [ ] PyPI package is published (maintainers only) + +### Creating a Release + +```bash +# Update version +# Edit pyproject.toml + +# Build package +make build + +# Test installation +pip install dist/*.whl + +# Create git tag +git tag v1.0.0 +git push origin v1.0.0 + +# Publish to PyPI (maintainers only) +make publish +``` + +## Community + +### Communication Channels + +- **GitHub Issues**: Bug reports and feature requests +- **GitHub Discussions**: Questions and general discussion +- **GitHub Pull Requests**: Code contributions +- **Email**: Direct contact with maintainers + +### Getting Help + +- **Documentation**: Start with the user guide and API reference +- **Examples**: Check the examples directory for usage patterns +- **Issues**: Search existing issues for similar problems +- **Discussions**: Ask questions in GitHub Discussions + +### Recognition + +Contributors are recognized in several ways: + +- **Contributors List**: GitHub automatically tracks contributors +- **Release Notes**: Contributors are credited in release notes +- **Documentation**: Contributors are listed in documentation +- **Community**: Active contributors are invited to join the maintainer team + +## Code of Conduct Enforcement + +### Reporting Issues + +If you experience or witness unacceptable behavior: + +1. **Document**: Record the incident with details +2. **Report**: Contact project maintainers privately +3. **Escalate**: If needed, escalate to GitHub support + +### Enforcement Actions + +Maintainers may take actions including: + +- **Warning**: Private or public warning +- **Temporary Ban**: Temporary restriction from participation +- **Permanent Ban**: Permanent removal from the project + +## Next Steps + +1. **Read the Documentation**: Familiarize yourself with the project +2. **Join Discussions**: Participate in GitHub Discussions +3. **Pick an Issue**: Start with a good first issue +4. **Make a Contribution**: Submit your first pull request +5. **Stay Engaged**: Continue contributing and helping others + +Thank you for contributing to Gradual! Your contributions help make stress testing more accessible and powerful for everyone. diff --git a/docs/dev_guide.md b/docs/dev_guide.md index e69de29..ef05887 100755 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -0,0 +1,457 @@ +# Development Guide + +This guide helps developers contribute to the Gradual stress testing framework. + +## Development Environment Setup + +### Prerequisites + +- Python 3.9 or higher +- Git +- Make (optional, for using Makefile commands) + +### Initial Setup + +```bash +# Clone the repository +git clone https://github.com/Gradual-Load-Testing/gradual.git +cd gradual + +# Create virtual environment +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install development dependencies +pip install -e ".[dev]" + +# Install pre-commit hooks +pre-commit install +``` + +### Development Dependencies + +The project uses several development tools: + +- **pytest**: Testing framework +- **black**: Code formatting +- **isort**: Import sorting +- **flake8**: Linting +- **mypy**: Type checking +- **pre-commit**: Git hooks + +## Project Structure + +``` +gradual/ +ā”œā”€ā”€ src/gradual/ # Main source code +ā”‚ ā”œā”€ā”€ __init__.py # Package initialization +ā”‚ ā”œā”€ā”€ base/ # Base classes and interfaces +ā”‚ ā”‚ ā””ā”€ā”€ orchestrator.py # Main test orchestration +ā”‚ ā”œā”€ā”€ configs/ # Configuration management +ā”‚ ā”‚ ā”œā”€ā”€ parser.py # Configuration parser +ā”‚ ā”‚ ā”œā”€ā”€ phase.py # Phase configuration +ā”‚ ā”‚ ā”œā”€ā”€ request.py # Request configuration +ā”‚ ā”‚ ā”œā”€ā”€ scenario.py # Scenario configuration +ā”‚ ā”‚ ā””ā”€ā”€ validate.py # Configuration validation +ā”‚ ā”œā”€ā”€ constants/ # Constants and enums +ā”‚ ā”œā”€ā”€ exceptions.py # Custom exceptions +ā”‚ ā”œā”€ā”€ reporting/ # Reporting and metrics +ā”‚ ā””ā”€ā”€ runners/ # Test runners +ā”‚ ā”œā”€ā”€ runner.py # Main test runner +ā”‚ ā”œā”€ā”€ phase.py # Phase execution +ā”‚ ā”œā”€ā”€ scenario.py # Scenario execution +ā”‚ ā”œā”€ā”€ session.py # Session management +ā”‚ ā””ā”€ā”€ iterators.py # Concurrency iterators +ā”œā”€ā”€ tests/ # Test suite +ā”œā”€ā”€ docs/ # Documentation +ā”œā”€ā”€ examples/ # Example configurations +ā”œā”€ā”€ benchmarks/ # Performance benchmarks +ā””ā”€ā”€ scripts/ # Utility scripts +``` + +## Code Style and Standards + +### Python Style Guide + +We follow PEP 8 with some modifications: + +- **Line Length**: 88 characters (Black default) +- **Import Sorting**: isort with Black profile +- **Type Hints**: Required for public APIs + +### Code Formatting + +```bash +# Format code with Black +make format +# or +black src/ tests/ + +# Sort imports +make sort-imports +# or +isort src/ tests/ +``` + +### Linting and Type Checking + +```bash +# Run all quality checks +make quality + +# Run specific checks +make lint # flake8 +make typecheck # mypy +make test # pytest +``` + +## Testing + +### Running Tests + +```bash +# Run all tests +make test + +# Run specific test files +pytest tests/test_runners.py + +# Run with coverage +pytest --cov=src/gradual --cov-report=html + +# Run performance tests +pytest tests/benchmarks/ -m "not slow" +``` + +### Writing Tests + +Follow these guidelines: + +1. **Test Structure**: Use descriptive test names and docstrings +2. **Fixtures**: Use pytest fixtures for common setup +3. **Mocking**: Mock external dependencies +4. **Coverage**: Aim for >90% code coverage + +Example test: + +```python +import pytest +from gradual.base.orchestrator import Orchestrator + +class TestOrchestrator: + """Test the orchestrator class.""" + + def test_initialization(self): + """Test orchestrator initialization with config paths.""" + test_config = "test_config.yaml" + request_config = "request_config.yaml" + orchestrator = Orchestrator(test_config, request_config) + assert orchestrator.test_config_file_path == test_config + assert orchestrator.request_configs_path == request_config + + def test_parser_initialization(self): + """Test that parser is properly initialized.""" + orchestrator = Orchestrator("test.yaml", "request.yaml") + assert orchestrator.parser is not None +``` + +## Architecture Overview + +### Core Components + +```mermaid +graph TB + A[Test Configuration] --> B[Configuration Parser] + B --> C[Orchestrator] + C --> D[Phase Runner] + D --> E[Scenario Runner] + E --> F[Request Execution] + + G[Reporting] --> H[Metrics Collection] + G --> I[Results Output] + + J[Configuration Files] --> B + K[YAML Parser] --> B + L[Validation] --> B +``` + +### Key Design Principles + +1. **Extensibility**: Easy to add new test scenarios and phases +2. **Modularity**: Clear separation of concerns between phases, scenarios, and requests +3. **Performance**: Efficient resource usage with gevent-based concurrency +4. **Reliability**: Robust error handling and configuration validation + +## Adding New Features + +### 1. Configuration Extensions + +To add new configuration options: + +```python +from gradual.configs.parser import Parser + +class ExtendedParser(Parser): + """Extended parser with additional configuration options.""" + + def __init__(self, test_config_file_path: str, request_configs_path: str): + super().__init__(test_config_file_path, request_configs_path) + # Add custom initialization logic + + def read_configs(self): + """Read and parse all configuration files with extensions.""" + super().read_configs() + # Add custom configuration parsing logic +``` + +### 2. Custom Runners + +To add new runner types: + +```python +from gradual.runners.runner import Runner + +class CustomRunner(Runner): + """Custom runner with additional functionality.""" + + def __init__(self, scenarios): + super().__init__(scenarios) + # Initialize custom runner features + + def start_test(self): + """Start test execution with custom logic.""" + # Add custom pre-execution logic + super().start_test() + # Add custom post-execution logic +``` + +### 3. Enhanced Reporting + +To add new reporting features: + +```python +from gradual.reporting.base import BaseReporter + +class CustomReporter(BaseReporter): + """Custom reporter implementation.""" + + def __init__(self, config): + super().__init__(config) + # Initialize custom reporting features + + def generate_report(self, results): + """Generate custom report format.""" + # Implementation here + pass +``` + +## Configuration Management + +### Configuration Schema + +The framework uses a YAML-based configuration system: + +```python +from gradual.configs.parser import Parser + +class Parser: + """Manages test configuration loading and validation.""" + + def __init__(self, test_config_file_path: str, request_configs_path: str): + self.test_config_file_path = test_config_file_path + self.request_configs_path = request_configs_path + self.phases = [] + self.run_name = None + self.phase_wait = 0 + + def read_configs(self): + """Load and parse configuration files.""" + # Implementation here + pass +``` + +### Configuration File Structure + +The framework expects two main configuration files: + +1. **Test Configuration**: Defines test phases, scenarios, and timing +2. **Request Configuration**: Defines individual HTTP and WebSocket requests and parameters + +Example test configuration: + +```yaml +runs: + name: "example_test" + wait_between_phases: 10 + phases: + phase1: + run_time: 300 + scenarios: + scenario1: + min_concurrency: 10 + max_concurrency: 100 + ramp_up_add: 10 + ramp_up_wait: 1 + requests: "FROM_REQUEST_YAML_FILE" + request_file: "requests.yaml" +``` + +## Performance Considerations + +### Memory Management + +- Use generators for large datasets +- Implement proper cleanup in runners +- Monitor memory usage during tests + +### Concurrency + +- Leverage gevent for efficient I/O +- Use connection pooling where appropriate +- Implement rate limiting to prevent overwhelming targets + +### Monitoring + +```python +import psutil +import time + +class PerformanceMonitor: + """Monitor system performance during tests.""" + + def __init__(self): + self.start_time = time.time() + self.start_memory = psutil.virtual_memory().used + + def get_stats(self): + """Get current performance statistics.""" + current_memory = psutil.virtual_memory().used + elapsed_time = time.time() - self.start_time + + return { + "elapsed_time": elapsed_time, + "memory_used": current_memory - self.start_memory, + "cpu_percent": psutil.cpu_percent() + } +``` + +## Documentation + +### Code Documentation + +- Use Google-style docstrings +- Include type hints for all public methods +- Document exceptions and edge cases + +Example: + +```python +def execute_test_scenario(self, scenario: dict, user_data: dict) -> dict: + """Execute a test scenario with user data. + + Args: + scenario: Scenario configuration dictionary + user_data: User-specific data for the scenario + + Returns: + Dictionary containing execution results + + Raises: + ScenarioExecutionError: If scenario execution fails + ValidationError: If response validation fails + """ + pass +``` + +### API Documentation + +- Keep API documentation up to date +- Include usage examples +- Document breaking changes + +## Contributing Guidelines + +### Pull Request Process + +1. **Fork** the repository +2. **Create** a feature branch +3. **Make** your changes +4. **Test** thoroughly +5. **Update** documentation +6. **Submit** a pull request + +### Commit Messages + +Use conventional commit format: + +``` +feat: add custom runner support +fix: resolve memory leak in connection pooling +docs: update API documentation +test: add performance benchmarks +``` + +### Code Review + +- All changes require review +- Address feedback promptly +- Maintain test coverage +- Follow project standards + +## Debugging and Troubleshooting + +### Development Tools + +```bash +# Enable debug logging +export GRADUAL_LOG_LEVEL="DEBUG" + +# Run with profiling +python -m cProfile -o profile.stats stress_run.py + +# Analyze profile results +python -c "import pstats; p = pstats.Stats('profile.stats'); p.sort_stats('cumulative').print_stats(20)" +``` + +### Common Issues + +1. **Import Errors**: Check PYTHONPATH and virtual environment +2. **Configuration Issues**: Validate YAML syntax and schema +3. **Performance Problems**: Use profiling tools to identify bottlenecks +4. **Memory Issues**: Monitor memory usage and implement cleanup + +## Release Process + +### Version Management + +- Use semantic versioning (MAJOR.MINOR.PATCH) +- Update version in `pyproject.toml` +- Create release notes +- Tag releases in Git + +### Distribution + +```bash +# Build package +make build + +# Test installation +pip install dist/*.whl + +# Upload to PyPI (maintainers only) +make publish +``` + +## Getting Help + +- **Issues**: GitHub Issues for bugs and feature requests +- **Discussions**: GitHub Discussions for questions and ideas +- **Documentation**: This guide and API reference +- **Code**: Source code and examples + +## Next Steps + +- Read the [API Reference](/gradual/api/overview/) for detailed technical information +- Check out [Examples](examples.md) for usage patterns +- Review the [User Guide](user_guide.md) for end-user documentation +- Join the [community discussions](https://github.com/Gradual-Load-Testing/gradual/discussions) diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..3cf0380 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,655 @@ +# Examples + +This page provides real-world examples of using the Gradual stress testing framework. + +## Sample Configuration Examples + +These examples are based on the actual configuration files included with Gradual. + +### Basic FastAPI Test Configuration + +This example shows a simple test configuration for a FastAPI application with two scenarios. + +**Test Configuration** (`test_config.yaml`): +```yaml +runs: + name: "Test Run" + wait_between_phases: 10 + phases: + "phase1": + scenarios: + "scenario1": + requests: + - "request1" + - "request2" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: + - 1 + - 2 + - 3 + ramp_up_wait: + - 1 + - 2 + - 3 + iterate_through_requests: true + "scenario2": + requests: + - "request3" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: 1 + ramp_up_wait: 1 + iterate_through_requests: true + run_time: 10 +``` + +**Request Configuration** (`request_config.yaml`): +```yaml +requests: + "request1": + url: "http://localhost:8000/ping" + method: "GET" + expected_response_time: 1 + auth: null + "request2": + url: "http://localhost:8000/data" + method: "GET" + expected_response_time: 1 + auth: null + "request3": + url: "http://localhost:8000/submit" + method: "POST" + expected_response_time: 1 + auth: null + params: + name: "John Doe" + email: "john.doe@example.com" + age: 30 + city: "New York" + country: "USA" +``` + +**What This Test Does:** +1. **Phase 1** runs for 10 seconds with two scenarios: + - **Scenario 1**: Tests two GET endpoints (`/ping` and `/data`) with ramp-up from 1 to 10 concurrent requests +- **Scenario 2**: Tests a POST endpoint (`/submit`) with a single concurrent request +2. **Ramp-up Strategy**: Concurrent requests multiply by 1, 2, then 3, with waits of 1, 2, then 3 seconds +3. **Request Iteration**: Both scenarios iterate through their requests sequentially + +### Running the Example + +```bash +# Navigate to the example directory +cd examples/fastapi_app + +# Run the stress test +stress-run stress_test_configs/test_config.yaml + +# Or specify both config files explicitly +stress-run stress_test_configs/test_config.yaml --request-config stress_test_configs/request_config.yaml +``` + +## Basic Examples + +### Simple HTTP API Test + +```yaml +# simple_api_test.yaml +runs: + name: "Simple API Test" + wait_between_phases: 0 + phases: + "api_test": + run_time: 120 + scenarios: + "get_posts": + requests: + - "get_post" + min_concurrency: 1 + max_concurrency: 50 + ramp_up_add: 10 + ramp_up_wait: 30 + iterate_through_requests: false +``` + +**Request Configuration** (`requests.yaml`): +```yaml +requests: + "get_post": + url: "https://jsonplaceholder.typicode.com/posts/1" + method: "GET" + expected_response_time: 1000 + auth: null + params: {} +``` + +### Multi-Scenario Test + +```yaml +# multi_scenario_test.yaml +runs: + name: "E-commerce API Test" + wait_between_phases: 5 + phases: + "browse_phase": + run_time: 300 + scenarios: + "browse_products": + requests: + - "get_products" + min_concurrency: 10 + max_concurrency: 100 + ramp_up_add: 20 + ramp_up_wait: 60 + iterate_through_requests: false + + "cart_phase": + run_time: 300 + scenarios: + "add_to_cart": + requests: + - "add_cart_item" + min_concurrency: 5 + max_concurrency: 50 + ramp_up_add: 10 + ramp_up_wait: 60 + iterate_through_requests: false +``` + +**Request Configuration** (`ecommerce_requests.yaml`): +```yaml +requests: + "get_products": + url: "https://api.ecommerce.com/products" + method: "GET" + expected_response_time: 2000 + auth: null + params: + category: "electronics" + page: "1" + + "add_cart_item": + url: "https://api.ecommerce.com/cart/add" + method: "POST" + expected_response_time: 3000 + auth: "Bearer {{auth_token}}" + params: + product_id: "prod_001" + quantity: 1 +``` + +## Advanced Examples + +### Authentication Testing + +```yaml +# auth_test.yaml +runs: + name: "Authentication Test" + wait_between_phases: 10 + phases: + "login_test": + run_time: 180 + scenarios: + "valid_login": + requests: + - "login_valid" + min_concurrency: 5 + max_concurrency: 50 + ramp_up_add: 10 + ramp_up_wait: 30 + iterate_through_requests: false + + "invalid_login": + requests: + - "login_invalid" + min_concurrency: 2 + max_concurrency: 20 + ramp_up_add: 5 + ramp_up_wait: 30 + iterate_through_requests: false + + "protected_endpoints": + run_time: 180 + scenarios: + "user_profile": + requests: + - "get_profile" + min_concurrency: 10 + max_concurrency: 100 + ramp_up_add: 20 + ramp_up_wait: 30 + iterate_through_requests: false +``` + +**Request Configuration** (`auth_requests.yaml`): +```yaml +requests: + "login_valid": + url: "https://api.example.com/auth/login" + method: "POST" + expected_response_time: 2000 + auth: null + params: + username: "user1" + password: "pass1" + + "login_invalid": + url: "https://api.example.com/auth/login" + method: "POST" + expected_response_time: 2000 + auth: null + params: + username: "invalid" + password: "wrong" + + "get_profile": + url: "https://api.example.com/users/profile" + method: "GET" + expected_response_time: 1500 + auth: "Bearer {{auth_token}}" + params: {} +``` + +### Database Testing + +```yaml +# database_test.yaml +runs: + name: "Database Performance Test" + wait_between_phases: 15 + phases: + "read_operations": + run_time: 300 + scenarios: + "user_reads": + requests: + - "get_user" + min_concurrency: 20 + max_concurrency: 200 + ramp_up_add: 40 + ramp_up_wait: 60 + iterate_through_requests: false + + "write_operations": + run_time: 300 + scenarios: + "user_creation": + requests: + - "create_user" + min_concurrency: 5 + max_concurrency: 50 + ramp_up_add: 10 + ramp_up_wait: 60 + iterate_through_requests: false +``` + +**Request Configuration** (`db_requests.yaml`): +```yaml +requests: + "get_user": + url: "https://api.example.com/users/{{user_id}}" + method: "GET" + expected_response_time: 500 + auth: "Bearer {{auth_token}}" + params: {} + + "create_user": + url: "https://api.example.com/users" + method: "POST" + expected_response_time: 1000 + auth: "Bearer {{admin_token}}" + params: + username: "newuser1" + email: "new1@example.com" + role: "user" +``` + +## Performance Testing Examples + +### Load Testing with Ramp-up + +```yaml +# ramp_up_test.yaml +runs: + name: "Ramp-up Load Test" + wait_between_phases: 0 + phases: + "health_check": + run_time: 1800 # 30 minutes + scenarios: + "api_health": + requests: + - "health_endpoint" + min_concurrency: 10 + max_concurrency: 1000 + ramp_up_add: 100 + ramp_up_wait: 900 # 15 minutes ramp-up + iterate_through_requests: false +``` + +**Request Configuration** (`health_requests.yaml`): +```yaml +requests: + "health_endpoint": + url: "https://api.example.com/health" + method: "GET" + expected_response_time: 100 + auth: null + params: {} +``` + +### Stress Testing + +```yaml +# stress_test.yaml +runs: + name: "Stress Test" + wait_between_phases: 0 + phases: + "heavy_operations": + run_time: 600 # 10 minutes + scenarios: + "complex_processing": + requests: + - "process_data" + min_concurrency: 50 + max_concurrency: 500 + ramp_up_add: 100 + ramp_up_wait: 60 + iterate_through_requests: false +``` + +**Request Configuration** (`stress_requests.yaml`): +```yaml +requests: + "process_data": + url: "https://api.example.com/process" + method: "POST" + expected_response_time: 10000 # 10 seconds + auth: "Bearer {{auth_token}}" + params: + data: "large_dataset" + operation: "complex_calculation" +``` + +## Custom Extensions Examples + +### Custom Configuration Parser + +```python +# custom_parser.py +from gradual.configs.parser import Parser +import yaml + +class ExtendedParser(Parser): + """Extended parser with additional configuration options.""" + + def __init__(self, test_config_file_path: str, request_configs_path: str): + super().__init__(test_config_file_path, request_configs_path) + self.custom_options = {} + + def read_configs(self): + """Read and parse configuration files with custom extensions.""" + super().read_configs() + + # Read custom configuration options + with open(self.test_config_file_path, "r") as config_file: + config_data = yaml.safe_load(config_file) + self.custom_options = config_data.get("custom_options", {}) + + def get_custom_option(self, key: str, default=None): + """Get custom configuration option.""" + return self.custom_options.get(key, default) +``` + +### Custom Runner + +```python +# custom_runner.py +from gradual.runners.runner import Runner +import logging + +class LoggingRunner(Runner): + """Custom runner with enhanced logging.""" + + def __init__(self, scenarios): + super().__init__(scenarios) + self.logger = logging.getLogger(__name__) + self.request_count = 0 + + def start_test(self): + """Start test execution with enhanced logging.""" + self.logger.info(f"Starting test with {len(self.scenarios)} scenarios") + self.logger.info(f"Total expected requests: {self._calculate_total_requests()}") + + super().start_test() + + self.logger.info(f"Test completed. Total requests processed: {self.request_count}") + + def _calculate_total_requests(self): + """Calculate total expected requests across all scenarios.""" + total = 0 + for scenario in self.scenarios: + # This is a simplified calculation - actual implementation would be more complex + total += scenario.max_concurrency * scenario.runtime + return total +``` + +## Integration Examples + +### CI/CD Pipeline Integration + +```yaml +# .github/workflows/stress-test.yml +name: "Stress Testing" + +on: + push: + branches: [main, develop] + pull_request: + branches: [main] + +jobs: + stress-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -e ".[dev]" + + - name: Run stress tests + run: | + stress-run --test_config tests/stress/api_test.yaml --request_config tests/stress/requests.yaml + + - name: Generate report + run: | + stress-run --test_config tests/stress/api_test.yaml --request_config tests/stress/requests.yaml + + - name: Upload report + uses: actions/upload-artifact@v3 + with: + name: stress-test-report + path: ./reports +``` + +### Docker Integration + +```dockerfile +# Dockerfile +FROM python:3.10-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY . . + +# Install Gradual +RUN pip install -e ".[dev]" + +# Expose ports +EXPOSE 8080 + +# Default command +CMD ["python", "-m", "gradual.main"] +``` + +```yaml +# docker-compose.yml +version: '3.8' + +services: + gradual: + build: . + ports: + - "8080:8080" + volumes: + - ./tests:/app/tests + - ./results:/app/results + environment: + - GRADUAL_LOG_LEVEL=INFO + command: ["python", "-m", "gradual.main"] +``` + +## Monitoring and Reporting + +### Custom Metrics Collection + +```python +# custom_metrics.py +import time +from dataclasses import dataclass +from typing import Dict, Any + +@dataclass +class TestMetrics: + """Custom metrics collection for test runs.""" + start_time: float + end_time: float = 0 + total_requests: int = 0 + successful_requests: int = 0 + failed_requests: int = 0 + total_response_time: float = 0 + + def record_request(self, success: bool, response_time: float): + """Record a single request result.""" + self.total_requests += 1 + if success: + self.successful_requests += 1 + else: + self.failed_requests += 1 + self.total_response_time += response_time + + def finalize(self): + """Finalize metrics collection.""" + self.end_time = time.time() + + def get_summary(self) -> Dict[str, Any]: + """Get metrics summary.""" + duration = self.end_time - self.start_time + avg_response_time = self.total_response_time / self.total_requests if self.total_requests > 0 else 0 + + return { + "duration_seconds": duration, + "total_requests": self.total_requests, + "successful_requests": self.successful_requests, + "failed_requests": self.failed_requests, + "success_rate": self.successful_requests / self.total_requests if self.total_requests > 0 else 0, + "average_response_time": avg_response_time, + "requests_per_second": self.total_requests / duration if duration > 0 else 0 + } +``` + +### Simple Dashboard + +```python +# simple_dashboard.py +import time +from typing import Dict, Any +from custom_metrics import TestMetrics + +class SimpleDashboard: + """Simple real-time dashboard for monitoring test progress.""" + + def __init__(self): + self.metrics = TestMetrics(start_time=time.time()) + self.running = True + + def update_metrics(self, success: bool, response_time: float): + """Update metrics with new request data.""" + self.metrics.record_request(success, response_time) + + def display_status(self): + """Display current test status.""" + if not self.running: + return + + summary = self.metrics.get_summary() + print(f"\n=== Test Status ===") + print(f"Duration: {summary['duration_seconds']:.1f}s") + print(f"Total Requests: {summary['total_requests']}") + print(f"Success Rate: {summary['success_rate']:.2%}") + print(f"Avg Response Time: {summary['average_response_time']:.3f}s") + print(f"Requests/sec: {summary['requests_per_second']:.1f}") + print("=" * 20) + + def stop(self): + """Stop the dashboard.""" + self.running = False + self.metrics.finalize() + print("\n=== Final Results ===") + final_summary = self.metrics.get_summary() + for key, value in final_summary.items(): + if isinstance(value, float): + print(f"{key}: {value:.3f}") + else: + print(f"{key}: {value}") +``` + +## Best Practices + +### Test Design + +1. **Realistic Scenarios**: Model real user behavior +2. **Data Variation**: Use different data sets +3. **Ramp-up Strategy**: Start with low concurrency and gradually increase +4. **Error Handling**: Test both success and failure scenarios + +### Environment Setup + +1. **Isolation**: Use dedicated test environments +2. **Monitoring**: Set up comprehensive monitoring +3. **Cleanup**: Implement data cleanup procedures +4. **Backup**: Have rollback plans + +### Analysis + +1. **Baselines**: Establish performance baselines +2. **Trends**: Monitor performance over time +3. **Bottlenecks**: Identify and document bottlenecks +4. **Documentation**: Document findings and recommendations + +## Next Steps + +- Explore the [API Reference](/gradual/api/overview/) for detailed technical information +- Read the [User Guide](user_guide.md) for comprehensive usage instructions +- Check the [Development Guide](dev_guide.md) for contribution guidelines +- Join the [community discussions](https://github.com/Gradual-Load-Testing/gradual/discussions) diff --git a/docs/getting_started.md b/docs/getting_started.md new file mode 100644 index 0000000..5f103d8 --- /dev/null +++ b/docs/getting_started.md @@ -0,0 +1,308 @@ +# Getting Started Guide + +Welcome to Gradual! This guide will help you run your first stress test and understand the basic concepts. By the end of this guide, you'll have successfully executed a load test and viewed the results. + +## Prerequisites + +Before you begin, ensure you have: + +- āœ… **Gradual installed** - Follow the [Installation Guide](installation.md) if you haven't installed it yet +- āœ… **A target application** - A web service, API, or application to test +- āœ… **Basic understanding** - Familiarity with HTTP requests and YAML configuration + +## Quick Start: Your First Test + +### Step 1: Create a Simple Test Configuration + +Create a file named `first_test.yaml` with the following content: + +```yaml +name: "My First Stress Test" +description: "A simple test to get started with Gradual" + +target: + base_url: "https://httpbin.org" + protocol: "http" + +scenarios: + - name: "Get Request" + weight: 100 + steps: + - request: + method: "GET" + path: "/get" + assertions: + - status_code: 200 + - response_time: "< 2000ms" + +load: + users: 10 + duration: 60 + ramp_up: 10 +``` + +**Note**: We're using `httpbin.org` as a test target since it's a public service designed for testing HTTP requests. + +### Step 2: Run Your First Test + +Open a terminal and run: + +```bash +stress-run first_test.yaml +``` + +You should see output similar to: + +``` +šŸš€ Starting stress test: My First Stress Test +šŸ“Š Target: https://httpbin.org +šŸ‘„ Users: 10 | Duration: 60s | Ramp-up: 10s +ā±ļø Test started at 2024-01-15 10:30:00 + +[ā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆ] 100% | 60s elapsed +āœ… Test completed successfully! + +šŸ“ˆ Results Summary: + Total Requests: 1,200 + Successful: 1,200 (100.0%) + Failed: 0 (0.0%) + Average Response Time: 245ms + 95th Percentile: 456ms + Requests/Second: 20.0 +``` + +### Step 3: View Real-time Results + +In another terminal, start the dashboard to monitor your test in real-time: + +```bash +stress-dashboard --mode bokeh --port 8080 +``` + +Open your browser and navigate to `http://localhost:8080` to see live metrics. + +## Understanding the Test Configuration + +Let's break down the key components of your test configuration: + +### Test Metadata +```yaml +name: "My First Stress Test" # Name of your test +description: "A simple test..." # Description for documentation +``` + +### Target Configuration +```yaml +target: + base_url: "https://httpbin.org" # The system you're testing + protocol: "http" # Protocol to use +``` + +### Test Scenarios +```yaml +scenarios: + - name: "Get Request" # Name of this test scenario + weight: 100 # Relative frequency (100 = 100%) + steps: # List of actions to perform + - request: # HTTP request configuration + method: "GET" # HTTP method + path: "/get" # Path relative to base_url +``` + +### Load Configuration +```yaml +load: + users: 10 # Number of concurrent users + duration: 60 # Test duration in seconds + ramp_up: 10 # Time to gradually increase users +``` + +## Creating Your Own Test + +Now let's create a test for your own application. Here's a template: + +```yaml +name: "My Application Test" +description: "Testing my web application" + +target: + base_url: "http://localhost:3000" # Change this to your app's URL + protocol: "http" + +scenarios: + - name: "Homepage Load" + weight: 50 + steps: + - request: + method: "GET" + path: "/" + assertions: + - status_code: 200 + - response_time: "< 1000ms" + + - name: "API Endpoint" + weight: 50 + steps: + - request: + method: "POST" + path: "/api/data" + headers: + Content-Type: "application/json" + body: + key: "value" + assertions: + - status_code: 200 + - response_time: "< 500ms" + +load: + users: 20 + duration: 120 + ramp_up: 30 +``` + +### Running Tests + +Run your stress tests using the CLI: + +```bash +# Run with both configuration files +stress-run --test_config my_test.yaml --request_config requests.yaml + +# Run with only test configuration (requests defined inline) +stress-run --test_config my_test.yaml + +# Run with relative paths +stress-run --test_config ./configs/test.yaml --request_config ./configs/requests.yaml +``` + +> **āš ļø Note:** All test parameters must be defined in the YAML configuration files. The CLI does not support command-line overrides. + +## Monitoring Your Tests + +### Real-time Dashboard + +The dashboard provides live monitoring: + +```bash +# Start Bokeh dashboard (interactive charts) +stress-dashboard --mode bokeh --port 8080 + +# Start WebSocket dashboard (lightweight) +stress-dashboard --mode websocket --port 8081 + +# Start with custom host +stress-dashboard --mode bokeh --host 0.0.0.0 --port 8080 +``` + +### Key Metrics to Watch + +- **Response Time**: How fast your application responds +- **Throughput**: Requests per second +- **Error Rate**: Percentage of failed requests +- **Active Users**: Current load on your system + +## Common Test Patterns + +### 1. Smoke Test (Light Load) +```yaml +load: + users: 5 + duration: 60 + ramp_up: 10 +``` + +### 2. Load Test (Normal Load) +```yaml +load: + users: 50 + duration: 300 + ramp_up: 60 +``` + +### 3. Stress Test (High Load) +```yaml +load: + users: 200 + duration: 600 + ramp_up: 120 +``` + +### 4. Spike Test (Sudden Load) +```yaml +load: + users: 100 + duration: 180 + ramp_up: 10 # Quick ramp-up for spike effect +``` + +## Best Practices for Beginners + +### 1. Start Small +- Begin with a small number of users (5-10) +- Use short durations (1-2 minutes) +- Gradually increase load + +### 2. Test in Isolation +- Use dedicated test environments +- Avoid testing production systems initially +- Clean up test data after runs + +### 3. Monitor Everything +- Watch your target application's performance +- Monitor system resources (CPU, memory, network) +- Use the dashboard for real-time insights + +### 4. Validate Results +- Check that assertions are passing +- Verify response times are reasonable +- Ensure error rates are acceptable + +## Troubleshooting Common Issues + +### Test Won't Start +```bash +# Check if Gradual is installed +stress-run --version + +# Verify your YAML syntax +python -c "import yaml; yaml.safe_load(open('test.yaml'))" +``` + +### High Error Rates +```bash +# Check if target is accessible +curl http://localhost:3000 + +# Run with debug mode +stress-run test.yaml --debug --verbose +``` + +### Dashboard Not Loading +```bash +# Check if port is available +netstat -tulpn | grep 8080 + +# Try different port +stress-dashboard --mode bokeh --port 8081 +``` + +## Next Steps + +Congratulations! You've successfully run your first stress test. Here's what to explore next: + +1. **Read the [User Guide](user_guide.md)** for advanced features and detailed configuration +2. **Check out [Examples](examples.md)** for real-world test scenarios +3. **Learn about [Custom Protocol Handlers](user_guide.md#custom-protocol-handlers)** for specialized testing +4. **Explore [Distributed Testing](user_guide.md#distributed-testing)** for large-scale tests +5. **Dive into the [Configuration Reference](configuration_reference.md)** for comprehensive configuration options and examples + +> **šŸ’” Tip:** Use the **Configuration** tab in the navigation for quick access to all configuration options and best practices. + +## Getting Help + +- **Documentation**: Browse the complete documentation +- **Examples**: Study the example configurations +- **Community**: Join discussions on GitHub +- **Issues**: Report bugs or request features + +Happy testing! šŸš€ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..9130a1c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,71 @@ +# Welcome to Gradual + +A comprehensive Python stress testing framework for applications and systems. + +## What is Gradual? + +Gradual is a powerful, extensible stress testing framework designed to help developers and DevOps engineers validate the performance and reliability of their applications under various load conditions. Built with Python, it provides a flexible and intuitive way to create, execute, and analyze stress tests. + +## Key Features + +- **šŸš€ High Performance**: Built on gevent for efficient concurrency +- **šŸ”Œ Protocol Support**: HTTP, WebSocket, and custom protocol support +- **šŸ“Š Real-time Monitoring**: Live dashboard with Bokeh visualizations +- **šŸ“ˆ Comprehensive Metrics**: Detailed performance analytics and reporting +- **šŸ” Authentication**: Support for Kerberos, NTLM, OAuth, and more +- **šŸŒ Distributed Testing**: Scale tests across multiple machines +- **šŸ“ YAML Configuration**: Simple, declarative test configuration +- **šŸŽÆ CLI Interface**: Easy-to-use command-line tools + +## Quick Start + +```bash +# Install Gradual +pip install -e ".[dev,bokeh,websockets]" + +# Run a stress test +stress-run --test_config examples/api_test.yaml --request_config examples/requests.yaml + +# Start the monitoring dashboard +stress-dashboard --mode bokeh +``` + +## Architecture Overview + + +```mermaid +graph TB + A[Test Configuration] --> B[Configuration Parser] + B --> C[Orchestrator] + C --> D[Phase Runner] + D --> E[Scenario Runner] + E --> F[Request Execution] + + G[Reporting] --> H[Metrics Collection] + G --> I[Results Output] + + J[Configuration Files] --> B + K[YAML Parser] --> B + L[Validation] --> B +``` + +## Documentation Sections + +- **[Quick Start Guide](quick_start.md)**: Get up and running in minutes +- **[User Guide](user_guide.md)**: Learn how to use Gradual for stress testing +- **[Configuration Reference](configuration_reference.md)**: Complete configuration options and examples +- **[Development Guide](dev_guide.md)**: Contribute to the framework development +- **[API Reference](api/overview)**: Complete API documentation +- **[Examples](examples.md)**: Real-world usage examples +- **[Contributing](contributing.md)**: How to contribute to the project + +## Getting Help + +- šŸ“– **Documentation**: This site contains comprehensive guides +- šŸ› **Issues**: Report bugs on [GitHub Issues](https://github.com/Gradual-Load-Testing/gradual/issues) +- šŸ’¬ **Discussions**: Join the conversation on [GitHub Discussions](https://github.com/Gradual-Load-Testing/gradual/discussions) +- šŸ“§ **Contact**: Reach out to the maintainers + +## License + +Gradual is licensed under the MIT License. See the [LICENSE](https://github.com/Gradual-Load-Testing/gradual/blob/main/LICENSE) file for details. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..49858dd --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,238 @@ +# Installation Guide + +This guide covers everything you need to install and set up Gradual for stress testing your applications and systems. + +## Prerequisites + +Before installing Gradual, ensure you have the following: + +- **Python 3.9 or higher** - Gradual requires Python 3.9+ for modern language features +- **pip package manager** - For installing Python packages +- **Git** - For cloning the repository (if installing from source) +- **System dependencies** - May vary based on your operating system + +## Installation Methods + +### Method 1: Install from Source (Recommended for Development) + +```bash +# Clone the repository +git clone https://github.com/Gradual-Load-Testing/gradual.git +cd gradual + +# Install in development mode with core features +pip install -e ".[dev,bokeh,websockets]" +``` + +### Method 2: Install from PyPI (Coming Soon) + +```bash +# Install the latest stable version +pip install gradual-load-testing + +# Install with all optional dependencies +pip install gradual-load-testing[all] +``` + +### Method 3: Install in Virtual Environment (Recommended for Production) + +```bash +# Create a virtual environment +python -m venv gradual-env + +# Activate the virtual environment +# On Linux/macOS: +source gradual-env/bin/activate +# On Windows: +gradual-env\Scripts\activate + +# Install Gradual +pip install -e ".[dev,bokeh,websockets]" +``` + +## Dependency Management + +### Core Dependencies + +The following dependencies are automatically installed with the core package: + +- **gevent** ā€“ High-performance asynchronous I/O and concurrency +- **requests** ā€“ User-friendly HTTP client for Python + +### Optional Dependencies + +Install additional features by specifying extra dependencies: + +```bash +# For kerb and oidc (coming soon) authentication support +pip install -e ".[auth]" + +# For distributed testing across multiple machines +pip install -e ".[distributed]" + +# For advanced metrics collection and analysis +pip install -e ".[metrics]" + +# For all optional features +pip install -e ".[all]" +``` + +### Development Dependencies + +For contributing to Gradual or running tests: + +```bash +# Install development dependencies +pip install -e ".[dev]" + +# This includes: +# - pytest for testing +# - black for code formatting +# - flake8 for linting +# - mypy for type checking +# - pre-commit for git hooks +``` + +## Platform-Specific Installation + +### Linux (Ubuntu/Debian) + +```bash +# Update package list +sudo apt update + +# Install system dependencies +sudo apt install python3-pip python3-venv git + +# Install Gradual +pip3 install -e ".[dev,bokeh,websockets]" +``` + +### Linux (CentOS/RHEL/Fedora) + +```bash +# Install system dependencies +sudo yum install python3-pip python3-venv git +# or for newer versions: +sudo dnf install python3-pip python3-venv git + +# Install Gradual +pip3 install -e ".[dev,bokeh,websockets]" +``` + +### macOS + +```bash +# Install Homebrew if not already installed +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + +# Install Python and Git +brew install python git + +# Install Gradual +pip3 install -e ".[dev,bokeh,websockets]" +``` + +### Windows + +```bash +# Install Python from https://python.org +# Install Git from https://git-scm.com + +# Open Command Prompt or PowerShell +pip install -e ".[dev,bokeh,websockets]" +``` + +## Docker Installation + +If you prefer using Docker: + +```bash +# Pull the official image (when available) +docker pull gradual/gradual:latest + +# Run a container +docker run -it --rm gradual/gradual:latest stress-run --help + +# Mount your test configurations +docker run -it --rm -v $(pwd):/tests gradual/gradual:latest stress-run --test_config /tests/my_test.yaml --request_config /tests/requests.yaml +``` + +## Verification + +After installation, verify that Gradual is working correctly: + +```bash +# Check if stress-run command is available +stress-run --version + +# Check if stress-dashboard command is available +stress-dashboard --version + +# Run a simple test to verify functionality +stress-run --help +``` + +## Troubleshooting Installation + +### Common Issues + +#### Permission Errors + +```bash +# Use --user flag to install in user directory +pip install --user -e ".[dev,bokeh,websockets]" + +# Or use sudo (not recommended for production) +sudo pip install -e ".[dev,bokeh,websockets]" +``` + +#### Python Version Issues + +```bash +# Check your Python version +python --version +python3 --version + +# Use specific Python version +python3.9 -m pip install -e ".[dev,bokeh,websockets]" +``` + +#### Dependency Conflicts + +```bash +# Create a fresh virtual environment +python -m venv fresh-env +source fresh-env/bin/activate # Linux/macOS +# fresh-env\Scripts\activate # Windows + +# Install with --no-deps to avoid conflicts +pip install -e ".[dev,bokeh,websockets]" --no-deps +``` + +#### Network Issues + +```bash +# Use alternative package index +pip install -e ".[dev,bokeh,websockets]" -i https://pypi.org/simple/ + +# Or use a mirror +pip install -e ".[dev,bokeh,websockets]" -i https://pypi.tuna.tsinghua.edu.cn/simple/ +``` + +## Next Steps + +After successful installation: + +1. **Read the [Getting Started Guide](getting_started.md)** to run your first test +2. **Explore the [User Guide](user_guide.md)** for detailed usage information +3. **Check out [Examples](examples.md)** for real-world use cases +4. **Join the community** for support and discussions + +## Support + +If you encounter installation issues: + +- Check the [GitHub Issues](https://github.com/Gradual-Load-Testing/gradual/issues) +- Review the [Troubleshooting section](user_guide.md#troubleshooting) +- Join the [community discussions](https://github.com/Gradual-Load-Testing/gradual/discussions) diff --git a/docs/quick_start.md b/docs/quick_start.md new file mode 100644 index 0000000..f658d8b --- /dev/null +++ b/docs/quick_start.md @@ -0,0 +1,263 @@ +# Quick Start Guide + +Get up and running with Gradual stress testing in minutes using the included sample configurations. + +## Prerequisites + +- Python 3.9 or higher +- pip package manager +- Basic knowledge of YAML configuration files + +## Installation + +```bash +# Clone the repository +git clone https://github.com/Gradual-Load-Testing/gradual.git +cd gradual + +# Install Gradual with development dependencies +pip install -e ".[dev,bokeh,websockets]" +``` + +## Your First Stress Test + +Gradual comes with sample configuration files that you can use immediately to test your understanding. + +### 1. Explore the Sample Configuration + +Navigate to the examples directory: + +```bash +cd examples/fastapi_app/stress_test_configs +``` + +You'll find two files: + +- `test_config.yaml` - Defines the test structure +- `request_config.yaml` - Defines the HTTP and WebSocket requests + +### 2. Understand the Configuration + +**Test Configuration** (`test_config.yaml`): +```yaml +runs: + name: "Test Run" + wait_between_phases: 10 + phases: + "phase1": + scenarios: + "scenario1": + requests: + - "request1" + - "request2" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: [1, 2, 3] + ramp_up_wait: [1, 2, 3] + iterate_through_requests: true + run_time: 10 +``` + +**Request Configuration** (`request_config.yaml`): +```yaml +requests: + "request1": + url: "http://localhost:8000/ping" + method: "GET" + expected_response_time: 1 + auth: null + "request2": + url: "http://localhost:8000/data" + method: "GET" + expected_response_time: 1 + auth: null +``` + +### 3. Run Your First Test + +```bash +# From the examples/fastapi_app directory +stress-run --test_config stress_test_configs/test_config.yaml +``` + +**What Happens:** + +1. Gradual reads the test configuration +2. Starts with 1 concurrent request +3. Ramp-up: 1 ā†’ 2 ā†’ 3 concurrent requests (waiting 1, 2, 3 seconds between steps) +4. Runs for 10 seconds +5. Tests two endpoints: `/ping` and `/data` + +### 4. Monitor the Test + +Open another terminal and start the monitoring dashboard: + +```bash +stress-dashboard --mode bokeh --port 8080 +``` + +Then open your browser to `http://localhost:8080` to see real-time metrics. + +## Understanding the Results + +After the test completes, you'll see: + +- **Total Requests**: Number of requests made +- **Success Rate**: Percentage of successful requests +- **Response Times**: Average, 95th percentile, 99th percentile +- **Throughput**: Requests per second +- **Error Details**: Any failed requests and their reasons + +## Customizing Your Test + +### Modify the Sample Configuration + +1. **Change the target URL** in `request_config.yaml`: +```yaml +requests: + "request1": + url: "https://your-api.com/health" # Change this + method: "GET" + expected_response_time: 1 + auth: null +``` + +2. **Adjust the load** in `test_config.yaml`: +```yaml +scenarios: + "scenario1": + min_concurrency: 1 + max_concurrency: 50 # Increase this + ramp_up_multiply: [1, 5, 10, 25, 50] # More gradual ramp-up + ramp_up_wait: [10, 10, 10, 10, 10] # Longer waits +``` + +3. **Add more requests**: +```yaml +requests: + "request1": + url: "https://your-api.com/health" + method: "GET" + expected_response_time: 1 + auth: null + "request2": + url: "https://your-api.com/endpoints" + method: "GET" + expected_response_time: 2 + auth: null + "request3": + url: "https://your-api.com/submit" + method: "POST" + expected_response_time: 3 + auth: null + params: + name: "Test User" + email: "test@example.com" +``` + +## Test Different Load Patterns + +### Spike Test (sudden load increase): +```yaml +scenarios: + "spike_test": + min_concurrency: 1 + max_concurrency: 100 + ramp_up_multiply: [1, 100] # Jump from 1 to 100 concurrent requests + ramp_up_wait: [0, 0] # No wait between steps + iterate_through_requests: true +``` + +### Gradual Ramp-up (steady increase): +```yaml +scenarios: + "gradual_test": + min_concurrency: 1 + max_concurrency: 50 + ramp_up_multiply: [1, 2, 5, 10, 25, 50] + ramp_up_wait: [30, 30, 30, 30, 30, 30] # 30 seconds between steps + iterate_through_requests: true +``` + +### Steady State (constant load): +```yaml +scenarios: + "steady_test": + min_concurrency: 50 + max_concurrency: 50 + ramp_up_multiply: [50] # Start immediately with 50 concurrent requests + ramp_up_wait: [0] # No ramp-up wait + iterate_through_requests: true +``` + +## Common Commands + +```bash +# Run a test with configuration files +stress-run --test_config config.yaml --request_config requests.yaml + +# Run with only test configuration (requests defined inline) +stress-run --test_config config.yaml + +# Run with relative paths +stress-run --test_config ./configs/test.yaml --request_config ./configs/requests.yaml +``` + +## Next Steps + +1. **Read the [User Guide](user_guide.md)** for comprehensive usage information +2. **Explore the [Configuration Reference](configuration_reference.md)** for all available options +3. **Check out [Examples](examples.md)** for more complex test scenarios +4. **Learn about [Development](dev_guide.md)** if you want to contribute + +## Troubleshooting + +### Common Issues + +**"No module named 'gradual'"** +```bash +# Make sure you're in the project directory and have installed it +cd gradual +pip install -e ".[dev,bokeh,websockets]" +``` + +**"Configuration file not found"** +```bash +# Check your current directory and file paths +pwd +ls -la +``` + +**"Invalid configuration"** +```bash +# Validate your YAML syntax +python -c "import yaml; yaml.safe_load(open('config.yaml'))" +``` + +**"Connection refused"** +- Ensure your target application is running +- Check the URL and port in your configuration +- Verify firewall settings + +### Getting Help + +- šŸ“– **Documentation**: Check the [User Guide](user_guide.md) and [Configuration Reference](configuration_reference.md) +- šŸ› **Issues**: Report bugs on [GitHub Issues](https://github.com/Gradual-Load-Testing/gradual/issues) +- šŸ’¬ **Discussions**: Ask questions on [GitHub Discussions](https://github.com/Gradual-Load-Testing/gradual/discussions) + +## Congratulations! + +You've successfully run your first stress test with Gradual! You now understand: + +- How to configure tests using YAML files +- How to run stress tests from the command line +- How to monitor tests in real-time +- How to customize test parameters + +Continue exploring the documentation to learn about advanced features like: + +- Multi-phase testing +- Authentication +- Custom protocol handlers +- Distributed testing +- Advanced reporting diff --git a/docs/user_guide.md b/docs/user_guide.md index e69de29..ee15395 100755 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -0,0 +1,410 @@ +# User Guide + +Learn how to use Gradual for stress testing your applications and systems. + +## Getting Started + +If you're new to Gradual, start with these guides: + +- **[Installation Guide](installation.md)** - Complete setup and installation instructions +- **[Getting Started Guide](getting_started.md)** - Run your first test and learn the basics + +## Quick Reference + +For experienced users, here's a quick reference of the main commands: + +```bash +# Run a stress test +stress-run --test_config test_config.yaml + +# Run with request configuration +stress-run --test_config test_config.yaml --request_config request_config.yaml + +# Start the monitoring dashboard +stress-dashboard --mode bokeh --port 8080 +``` + +## Configuration + +Gradual uses a two-file configuration system that provides flexibility and clarity: + +1. **Test Configuration** (`test_config.yaml`) - Defines test structure, phases, and scenarios +2. **Request Configuration** (`request_config.yaml`) - Defines individual HTTP and WebSocket requests + +> **šŸ“– šŸ’” Need detailed configuration options? Check out the dedicated [Configuration Reference](configuration_reference.md) tab in the navigation for a complete guide to all available settings, examples, and best practices.** + +This section provides a quick overview of the configuration structure. For comprehensive configuration details, advanced examples, and troubleshooting tips, use the **Configuration** tab in the left navigation. + +### Test Configuration Structure + +The main test configuration file defines the overall test structure: + +```yaml +runs: + name: "Test Run Name" # Name of the test run + wait_between_phases: 10 # Wait time between phases (seconds) + phases: # Test phases + "phase1": # Phase name + scenarios: # Scenarios within this phase + "scenario1": # Scenario name + requests: # List of request names + - "request1" + - "request2" + min_concurrency: 1 # Minimum concurrent requests + max_concurrency: 10 # Maximum concurrent requests + ramp_up_multiply: # Ramp-up multipliers (can be list or single value) + - 1 + - 2 + - 3 + ramp_up_wait: # Wait time between ramp-up steps (seconds) + - 1 + - 2 + - 3 + iterate_through_requests: true # Whether to iterate through requests + "scenario2": # Another scenario + requests: + - "request3" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: 1 # Single value (will be converted to list) + ramp_up_wait: 1 # Single value (will be converted to list) + iterate_through_requests: true + run_time: 10 # Phase duration (seconds) +``` + +### Request Configuration Structure + +The request configuration file defines individual HTTP and WebSocket requests: + +```yaml +requests: + "request1": # Request name (referenced in test config) + url: "http://localhost:8000/ping" # Full URL + method: "GET" # HTTP method + expected_response_time: 1 # Expected response time (seconds) + auth: null # Authentication (null for none) + "request2": + url: "http://localhost:8000/data" + method: "GET" + expected_response_time: 1 + auth: null + "request3": + url: "http://localhost:8000/submit" + method: "POST" + expected_response_time: 1 + auth: null + params: # Request parameters/body + name: "John Doe" + email: "john.doe@example.com" + age: 30 + city: "New York" + country: "USA" +``` + +## Configuration Options + +### Phase Configuration + +- **`name`**: Unique identifier for the phase +- **`scenarios`**: Dictionary of scenarios to run in this phase +- **`run_time`**: Duration of the phase in seconds + +### Scenario Configuration + +- **`requests`**: List of request names (must match names in request config) +- **`min_concurrency`**: Starting number of concurrent requests +- **`max_concurrency`**: Maximum number of concurrent requests +- **`ramp_up_multiply`**: Multipliers for gradual request increase (list or single value) +- **`ramp_up_wait`**: Wait time between ramp-up steps in seconds (list or single value) +- **`iterate_through_requests`**: Whether to cycle through requests sequentially +- **`run_once`**: Whether to run the scenario only once (optional) + +### Request Configuration + +- **`url`**: Full URL for the request +- **`method`**: HTTP method (GET, POST, PUT, DELETE, etc.) +- **`expected_response_time`**: Expected response time in seconds +- **`auth`**: Authentication configuration (null for none) +- **`params`**: Request parameters (for GET) or body data (for POST/PUT) + +## Ramp-up Configuration + +Gradual supports two types of ramp-up strategies: + +### Multiplicative Ramp-up + +```yaml +ramp_up_multiply: [1, 2, 4, 8, 16] # Multiply requests by these factors +ramp_up_wait: [5, 5, 5, 5, 5] # Wait 5 seconds between each step +``` + +This will: +1. Start with 1 concurrent request +2. Wait 5 seconds, then increase to 2 concurrent requests +3. Wait 5 seconds, then increase to 4 concurrent requests +4. Wait 5 seconds, then increase to 8 concurrent requests +5. Wait 5 seconds, then increase to 16 concurrent requests + +### Additive Ramp-up + +```yaml +ramp_up_add: [1, 2, 3, 4, 5] # Add these numbers of requests +ramp_up_wait: [2, 2, 2, 2, 2] # Wait 2 seconds between each step +``` + +This will: +1. Start with 1 concurrent request +2. Wait 2 seconds, then add 1 request (total: 2) +3. Wait 2 seconds, then add 2 requests (total: 4) +4. Wait 2 seconds, then add 3 requests (total: 7) +5. Wait 2 seconds, then add 4 requests (total: 11) +6. Wait 2 seconds, then add 5 requests (total: 16) + +## Advanced Features + +### Request Iteration + +When `iterate_through_requests: true`, Gradual will cycle through the requests in the scenario sequentially. This is useful for simulating realistic request workflows. + +### Phase Sequencing + +Phases run sequentially with the specified `wait_between_phases` delay. This allows you to: + +- Test different load patterns +- Implement complex test scenarios +- Allow system recovery between phases + +### Dynamic Request Loading + +For advanced use cases, you can reference requests from external files: + +```yaml +requests: "FROM_REQUEST_YAML_FILE" +request_file: path/to/requests.yaml +``` + +## Best Practices + +### 1. Test Design + +- **Start Small**: Begin with low concurrency and gradually increase +- **Realistic Scenarios**: Model realistic request patterns +- **Phase Planning**: Use phases to test different load patterns +- **Request Sequencing**: Use `iterate_through_requests` for realistic workflows + +### 2. Ramp-up Strategy + +- **Gradual Increase**: Use ramp-up to avoid overwhelming the system +- **Monitor Performance**: Watch for performance degradation during ramp-up +- **Realistic Timing**: Set appropriate wait times between ramp-up steps + +### 3. Configuration Management + +- **Separate Concerns**: Keep test structure and request definitions separate +- **Reusable Requests**: Define requests once and reuse across scenarios +- **Clear Naming**: Use descriptive names for phases, scenarios, and requests + +## Example Configurations + +### Basic Load Test + +```yaml +# test_config.yaml +runs: + name: "Basic Load Test" + wait_between_phases: 5 + phases: + "ramp_up": + scenarios: + "basic_scenario": + requests: + - "health_check" + - "api_endpoint" + min_concurrency: 1 + max_concurrency: 50 + ramp_up_multiply: [1, 2, 5, 10, 25, 50] + ramp_up_wait: [10, 10, 10, 10, 10, 10] + iterate_through_requests: true + run_time: 300 +``` + +```yaml +# request_config.yaml +requests: + "health_check": + url: "http://localhost:8000/health" + method: "GET" + expected_response_time: 0.5 + auth: null + "api_endpoint": + url: "http://localhost:8000/api/data" + method: "GET" + expected_response_time: 1.0 + auth: null +``` + +### Multi-Phase Test + +```yaml +# test_config.yaml +runs: + name: "Multi-Phase Test" + wait_between_phases: 10 + phases: + "warm_up": + scenarios: + "warm_up_scenario": + requests: + - "light_request" + min_concurrency: 1 + max_concurrency: 10 + ramp_up_multiply: [1, 2, 5, 10] + ramp_up_wait: [5, 5, 5, 5] + iterate_through_requests: true + run_time: 120 + "peak_load": + scenarios: + "peak_scenario": + requests: + - "heavy_request" + - "data_processing" + min_concurrency: 10 + max_concurrency: 100 + ramp_up_multiply: [10, 25, 50, 75, 100] + ramp_up_wait: [10, 10, 10, 10, 10] + iterate_through_requests: true + run_time: 300 + "cool_down": + scenarios: + "cool_down_scenario": + requests: + - "light_request" + min_concurrency: 100 + max_concurrency: 1 + ramp_up_add: [-20, -20, -20, -20, -19] + ramp_up_wait: [5, 5, 5, 5, 5] + iterate_through_requests: true + run_time: 120 +``` + +## Custom Protocol Handlers + +Gradual supports custom protocol handlers for specialized testing scenarios. You can extend the framework to handle custom protocols beyond HTTP. + +### Creating Custom Handlers + +```python +from gradual.base import BaseProtocolHandler + +class CustomProtocolHandler(BaseProtocolHandler): + def __init__(self, config): + super().__init__(config) + # Custom initialization + + def execute_request(self, request_data): + # Custom protocol implementation + # Return response data + return {"status": "success", "data": request_data} + + def validate_response(self, response): + # Custom validation logic + return True +``` + +### Registering Custom Handlers + +```python +from gradual.runners import TestRunner + +# Register custom handler +runner = TestRunner(config) +runner.register_protocol("custom", CustomProtocolHandler) + +# Use in configuration +scenario = { + "protocol": "custom", + "config": {...} +} +``` + +## Distributed Testing + +Gradual supports distributed testing for large-scale load testing across multiple machines. + +### Distributed Architecture + +```yaml +# Distributed test configuration +runs: + name: "Distributed Load Test" + distributed: + enabled: true + coordinator: "192.168.1.100:6379" # Redis coordinator + workers: + - "192.168.1.101:8080" + - "192.168.1.102:8080" + - "192.168.1.103:8080" + phases: + "distributed_phase": + scenarios: + "distributed_scenario": + requests: + - "api_request" + min_concurrency: 100 + max_concurrency: 1000 + ramp_up_multiply: [100, 250, 500, 750, 1000] + ramp_up_wait: [30, 30, 30, 30, 30] + iterate_through_requests: true + run_time: 600 +``` + +### Coordinator Setup + +```bash +# Start Redis coordinator +redis-server --port 6379 + +# Start worker nodes +stress-run --worker --coordinator redis://192.168.1.100:6379 --port 8080 +``` + +### Benefits of Distributed Testing + +- **Higher Load Capacity**: Distribute load across multiple machines +- **Geographic Distribution**: Test from different locations +- **Resource Scaling**: Scale horizontally without single machine limits +- **Realistic Scenarios**: Simulate real-world distributed traffic + +## Troubleshooting + +### Common Issues + +#### Configuration Errors + +- **Missing Fields**: Ensure all required fields are present +- **Invalid Values**: Check that numeric values are positive +- **Request References**: Verify request names match between config files + +#### Performance Issues + +- **Ramp-up Too Fast**: Increase wait times between ramp-up steps +- **High Error Rates**: Reduce concurrency or check target system health +- **Memory Issues**: Monitor system resources during tests + +### Running Tests + +```bash +# Run with configuration files +stress-run --test_config my_test.yaml --request_config requests.yaml + +# Run with only test configuration (requests defined inline) +stress-run --test_config my_test.yaml +``` + +## Next Steps + +- **New to Gradual?** Start with the [Installation Guide](installation.md) and [Getting Started Guide](getting_started.md) +- **Ready for more?** Explore the [API Reference](/gradual/api/overview/) for detailed technical information +- **Looking for examples?** Check out [Examples](examples.md) for real-world use cases +- **Want to contribute?** Learn about [Development](dev_guide.md) +- **Need help?** Join the [community discussions](https://github.com/Gradual-Load-Testing/gradual/discussions) diff --git a/generate_api_docs.py b/generate_api_docs.py new file mode 100644 index 0000000..29f8e46 --- /dev/null +++ b/generate_api_docs.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +Automatic API Documentation Generator + +This script automatically discovers all modules in the gradual package +and generates the corresponding API documentation files. +""" + +import os +import importlib +import pkgutil +from pathlib import Path + + +def discover_modules(package_name): + """Discover all modules in a package recursively.""" + try: + package = importlib.import_module(package_name) + package_path = Path(package.__file__).parent + + modules = [] + + # Get all Python files in the package + for item in pkgutil.iter_modules([str(package_path)]): + if not item.name.startswith("_"): + modules.append(f"{package_name}.{item.name}") + + # Recursively check subdirectories + for item in os.listdir(package_path): + item_path = package_path / item + if item_path.is_dir() and not item.startswith("_"): + init_file = item_path / "__init__.py" + if init_file.exists(): + sub_package = f"{package_name}.{item}" + sub_modules = discover_modules(sub_package) + modules.extend(sub_modules) + + return modules + except ImportError: + return [] + + +def generate_api_file(module_name, output_dir): + """Generate an API documentation file for a module.""" + filename = f"{module_name.split('.')[-1]}.md" + filepath = output_dir / filename + + content = f"""# {module_name.split('.')[-1].title()} + +This page contains the API documentation for the `{module_name}` module. + +::: {module_name} + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + filters: ["!^_"] + preload_modules: [{module_name}] + merge_init_into_class: true + show_submodules: true + show_if_no_docstring: true +""" + + with open(filepath, "w") as f: + f.write(content) + + return filename + + +def main(): + """Main function to generate all API documentation.""" + package_name = "gradual" + docs_dir = Path("docs/api") + + # Create docs directory if it doesn't exist + docs_dir.mkdir(parents=True, exist_ok=True) + + # Discover all modules + print(f"Discovering modules in {package_name}...") + modules = discover_modules(package_name) + + if not modules: + print(f"No modules found in {package_name}") + return + + print(f"Found {len(modules)} modules:") + for module in modules: + print(f" - {module}") + + # Generate documentation files + print("\nGenerating API documentation files...") + generated_files = [] + + for module in modules: + filename = generate_api_file(module, docs_dir) + generated_files.append(filename) + print(f" āœ“ Generated {filename}") + + # Generate overview file + overview_content = f"""# API Reference + +This page provides comprehensive API documentation for the {package_name} package. + +## Available Modules + +The following modules are automatically discovered and documented: + +""" + + for module in modules: + module_short = module.split(".")[-1] + overview_content += ( + f"- [{module_short.title()}]({module_short}.md) - `{module}`\n" + ) + + overview_content += """ + +## Automatic Discovery + +This documentation is automatically generated using a Python script that: +1. Discovers all modules in the package +2. Generates individual documentation files for each module +3. Uses mkdocstrings to extract documentation from docstrings +4. Updates automatically when you run the script + +To regenerate all API documentation, run: +```bash +python generate_api_docs.py +``` +""" + + overview_file = docs_dir / "overview.md" + with open(overview_file, "w") as f: + f.write(overview_content) + + print(f"\nāœ“ Generated overview.md") + print(f"\nTotal files generated: {len(generated_files) + 1}") + print(f"Run 'mkdocs build' to build the documentation") + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..15dffb4 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,169 @@ +site_name: Gradual - Stress Testing Framework +site_description: A comprehensive Python stress testing framework for applications and systems +site_author: Gradual Load Testing Team +site_url: https://gradual-load-testing.github.io/gradual + +repo_name: Gradual-Load-Testing/gradual +repo_url: https://github.com/Gradual-Load-Testing/gradual +edit_uri: edit/main/docs/ + +theme: + name: material + language: en + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.top + - navigation.footer + - search.highlight + - search.share + - content.code.copy + - content.code.annotate + - content.code.select + - content.tabs.link + - content.tabs.remember + - content.tabs.tabs + - content.tabs.tabs.sticky + - content.tabs.tabs.sticky.primary + - content.tabs.tabs.sticky.secondary + - content.tabs.tabs.sticky.tertiary + - content.tabs.tabs.sticky.quaternary + - content.tabs.tabs.sticky.quinary + - content.tabs.tabs.sticky.senary + - content.tabs.tabs.sticky.septenary + - content.tabs.tabs.sticky.octonary + - content.tabs.tabs.sticky.nonary + - content.tabs.tabs.sticky.denary + palette: + - scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/toggle-switch + name: Switch to dark mode + - scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/toggle-switch-off-outline + name: Switch to light mode + icon: + repo: fontawesome/brands/github + edit: material/pencil + view: material/eye + +plugins: + - search + - mkdocstrings: + default_handler: python + handlers: + python: + paths: [src] + options: + show_source: true + show_root_heading: true + show_signature_annotations: true + show_category_heading: true + heading_level: 2 + members_order: source + docstring_style: google + preload_modules: [gradual] + filters: ["!^_"] + # - mermaid # Using Material theme's built-in Mermaid support instead + +markdown_extensions: + - admonition + - codehilite + - footnotes + - meta + - pymdownx.betterem + - pymdownx.caret + - pymdownx.details + - pymdownx.emoji + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.magiclink + - pymdownx.mark + - pymdownx.smartsymbols + - pymdownx.snippets: + check_paths: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + +nav: + - Home: index.md + - Getting Started: + - Installation: installation.md + - Getting Started: getting_started.md + - User Guide: + - User Guide: user_guide.md + - Configuration: configuration_reference.md + - Development Guide: dev_guide.md + - API Reference: + - Overview: api/overview.md + - Core Modules: + - Base Classes: api/base.md + - Configuration: api/configs.md + - Constants: api/constants.md + - Exceptions: api/exceptions.md + - Reporting: api/reporting.md + - Runners: api/runners.md + - Configuration: + - Parser: api/parser.md + - Phase: api/phase.md + - Request: api/request.md + - Scenario: api/scenario.md + - Validation: api/validate.md + - Base: + - Orchestrator: api/orchestrator.md + - Constants: + - Request Types: api/request_types.md + - Reporting: + - Adapters: api/adapters.md + - Logger: api/logger.md + - Statistics: api/stats.md + - Base Adapters: api/base.md + - Logging Adapters: api/logging.md + - Runners: + - Iterators: api/iterators.md + - Phase Runner: api/phase.md + - Request Runner: api/request.md + - Main Runner: api/runner.md + - Scenario Runner: api/scenario.md + - Session: api/session.md + - HTTP Request: api/Http.md + - SocketIO Request: api/SocketIO.md + - Base Request: api/base.md + - Examples: examples.md + - Contributing: contributing.md + +# extra: +# social: +# - icon: fontawesome/brands/github +# link: https://github.com/Gradual-Load-Testing/gradual +# - icon: fontawesome/brands/twitter +# link: https://twitter.com/gradual_testing +# analytics: +# provider: google +# property: !ENV GOOGLE_ANALYTICS_KEY +# version: +# provider: mike + +extra_css: + - stylesheets/extra.css + +extra_javascript: + - https://polyfill.io/v3/polyfill.min.js?features=es6 diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..88a1dad --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,16 @@ +# Documentation dependencies +mkdocs>=1.5.0 +mkdocs-material>=9.4.0 +mkdocstrings>=0.24.0 +mkdocstrings-python>=1.7.0 +pymdown-extensions>=10.0.0 +mkdocs-material-extensions>=1.3.0 +mkdocs-git-revision-date-localized-plugin>=1.2.0 +mkdocs-git-authors-plugin>=0.7.0 +mkdocs-awesome-pages-plugin>=2.9.0 +mkdocs-section-index>=0.3.0 +mkdocs-redirects>=1.2.0 +mkdocs-print-site-plugin>=0.4.0 +mkdocs-versioning>=0.2.0 +mkdocs-macros-plugin>=1.0.0 +mkdocs-roamlinks-plugin>=0.1.0 diff --git a/scripts/setup_docs.sh b/scripts/setup_docs.sh new file mode 100755 index 0000000..cbfd886 --- /dev/null +++ b/scripts/setup_docs.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Setup script for Gradual documentation +# This script installs dependencies and sets up the documentation environment + +set -e + +echo "šŸš€ Setting up Gradual documentation environment..." + +# Check if Python is available +if ! command -v python3 &> /dev/null; then + echo "āŒ Python 3 is required but not installed." + exit 1 +fi + +# Check Python version +PYTHON_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +REQUIRED_VERSION="3.9" + +if [ "$(printf '%s\n' "$REQUIRED_VERSION" "$PYTHON_VERSION" | sort -V | head -n1)" != "$REQUIRED_VERSION" ]; then + echo "āŒ Python $REQUIRED_VERSION or higher is required. Found: $PYTHON_VERSION" + exit 1 +fi + +echo "āœ… Python $PYTHON_VERSION found" + +# Create virtual environment if it doesn't exist +if [ ! -d "venv" ]; then + echo "šŸ“¦ Creating virtual environment..." + python3 -m venv venv +else + echo "šŸ“¦ Using existing virtual environment..." +fi + +# Activate virtual environment +echo "šŸ”§ Activating virtual environment..." +source venv/bin/activate + +# Upgrade pip +echo "ā¬†ļø Upgrading pip..." +pip install --upgrade pip setuptools wheel + +# Install documentation dependencies +echo "šŸ“š Installing documentation dependencies..." +pip install -r requirements-docs.txt + +# Install Gradual in development mode +echo "šŸ”§ Installing Gradual in development mode..." +pip install -e ".[dev]" + +# Verify installation +echo "āœ… Verifying installation..." +python -c "import mkdocs; print(f'MkDocs version: {mkdocs.__version__}')" +python -c "import gradual; print('Gradual package imported successfully')" + +echo "" +echo "šŸŽ‰ Documentation environment setup complete!" +echo "" +echo "šŸ“– Available commands:" +echo " make docs - Build the documentation" +echo " make docs-serve - Start local documentation server" +echo " make docs-deploy - Deploy to GitHub Pages" +echo "" +echo "šŸ”§ API Documentation Generation:" +echo " python generate_api_docs.py - Generate/update API documentation" +echo " make docs-api - Generate/update API documentation (Makefile target)" +echo " # This script automatically discovers all modules in the gradual package" +echo " # and generates corresponding API documentation files in docs/api/" +echo "" +echo "šŸŒ To view documentation locally:" +echo " make docs-serve" +echo " # Then open http://127.0.0.1:8000 in your browser" +echo "" +echo "šŸ“š Documentation files are in the docs/ directory" +echo " - index.md - Main documentation page" +echo " - user_guide.md - User guide" +echo " - dev_guide.md - Development guide" +echo " - api/ - API reference (auto-generated)" +echo " - overview.md - API overview and module listing" +echo " - *.md - Individual module documentation" +echo " - examples.md - Usage examples" +echo " - contributing.md - Contributing guide" +echo "" +echo "šŸ”„ To keep API docs up-to-date:" +echo " # Run this after making changes to the codebase" +echo " python generate_api_docs.py" +echo " # OR use the Makefile target:" +echo " make docs-api" +echo " make docs-serve # View updated documentation" diff --git a/src/gradual/__init__.py b/src/gradual/__init__.py index e69de29..4392787 100755 --- a/src/gradual/__init__.py +++ b/src/gradual/__init__.py @@ -0,0 +1,9 @@ +""" +Gradual - A comprehensive Python stress testing framework for applications and systems. + +This package provides high-level interfaces for configuring and running stress tests +with support for multiple protocols, distributed testing, and comprehensive reporting. +""" + +__version__ = "0.1.0" +__author__ = "Gradual Load Testing Team" diff --git a/src/gradual/base/__init__.py b/src/gradual/base/__init__.py index e69de29..c1bc71a 100755 --- a/src/gradual/base/__init__.py +++ b/src/gradual/base/__init__.py @@ -0,0 +1,6 @@ +""" +Base classes and interfaces for extending the Gradual framework. + +This module provides the foundational classes that can be extended to add +custom protocol handlers, assertions, and other functionality. +""" diff --git a/src/gradual/configs/__init__.py b/src/gradual/configs/__init__.py index e69de29..d42e294 100755 --- a/src/gradual/configs/__init__.py +++ b/src/gradual/configs/__init__.py @@ -0,0 +1,6 @@ +""" +Configuration management and validation for the Gradual framework. + +This module handles parsing, validation, and management of test configurations +including scenarios, phases, and request definitions. +""" diff --git a/src/gradual/constants/__init__.py b/src/gradual/constants/__init__.py index e69de29..2e7bcad 100644 --- a/src/gradual/constants/__init__.py +++ b/src/gradual/constants/__init__.py @@ -0,0 +1,6 @@ +""" +Framework constants and enumerations for the Gradual framework. + +This module defines constants used throughout the framework including +request types, configuration keys, and other standardized values. +""" diff --git a/src/gradual/reporting/__init__.py b/src/gradual/reporting/__init__.py index e69de29..e9bacd7 100644 --- a/src/gradual/reporting/__init__.py +++ b/src/gradual/reporting/__init__.py @@ -0,0 +1,6 @@ +""" +Reporting and metrics collection for the Gradual framework. + +This module handles collection, storage, and reporting of test metrics +including response times, error rates, and custom statistics. +""" diff --git a/src/gradual/reporting/adapters/__init__.py b/src/gradual/reporting/adapters/__init__.py index e69de29..0fae410 100644 --- a/src/gradual/reporting/adapters/__init__.py +++ b/src/gradual/reporting/adapters/__init__.py @@ -0,0 +1,6 @@ +""" +Reporting adapters for the Gradual framework. + +This module provides adapters for different output formats and destinations +including logging, databases, and external monitoring systems. +""" diff --git a/src/gradual/runners/__init__.py b/src/gradual/runners/__init__.py index e69de29..0271c8e 100755 --- a/src/gradual/runners/__init__.py +++ b/src/gradual/runners/__init__.py @@ -0,0 +1,6 @@ +""" +Test execution and load generation for the Gradual framework. + +This module handles the execution of stress tests including scenario management, +request iteration, and concurrency control. +""" diff --git a/src/gradual/runners/request/__init__.py b/src/gradual/runners/request/__init__.py index e69de29..e882536 100644 --- a/src/gradual/runners/request/__init__.py +++ b/src/gradual/runners/request/__init__.py @@ -0,0 +1,6 @@ +""" +Request handling and execution for the Gradual framework. + +This module provides request runners for different protocols including +HTTP, WebSocket, and custom protocols. +"""