diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index b59c5e95b..ffc945cef 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.21" +version = "2.13.22" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/samples/simulate-component-agent/main.py b/packages/uipath/samples/simulate-component-agent/main.py index 0acd8f0cb..499e92c92 100644 --- a/packages/uipath/samples/simulate-component-agent/main.py +++ b/packages/uipath/samples/simulate-component-agent/main.py @@ -9,6 +9,9 @@ Run with per-component simulation (routes each tool call to the API): uipath run main -f input.json --simulation "$(cat simulation.json)" + +Debug with per-component simulation: + uipath debug main -f input.json --simulation "$(cat simulation.json)" """ from pydantic import BaseModel diff --git a/packages/uipath/src/uipath/_cli/cli_debug.py b/packages/uipath/src/uipath/_cli/cli_debug.py index 7f542e871..7d7eceba2 100644 --- a/packages/uipath/src/uipath/_cli/cli_debug.py +++ b/packages/uipath/src/uipath/_cli/cli_debug.py @@ -3,13 +3,14 @@ from typing import Any, cast, get_args import click +from pydantic import ValidationError from uipath._cli._chat._bridge import get_chat_bridge from uipath._cli._debug._bridge import DebugAttachMode, get_debug_bridge from uipath._cli._utils._debug import setup_debugging from uipath._cli._utils._studio_project import StudioClient from uipath._cli._utils._tracing import create_trace_manager -from uipath.eval.mocks import UiPathMockRuntime +from uipath.eval.mocks import SimulationConfig, UiPathMockRuntime, build_mocking_context from uipath.eval.mocks._mock_runtime import load_simulation_config from uipath.platform.common import ( ExecutionSourceContext, @@ -79,6 +80,12 @@ "'console' for local runs." ), ) +@click.option( + "--simulation", + required=False, + default=None, + help="Simulation config as a JSON object (same schema as simulation.json)", +) @track_command("debug") def debug( entrypoint: str | None, @@ -90,6 +97,7 @@ def debug( debug: bool, debug_port: int, attach: str | None, + simulation: str | None, ) -> None: """Debug the project.""" input_file = file or input_file @@ -97,6 +105,14 @@ def debug( if not setup_debugging(debug, debug_port): console.error(f"Failed to start debug server on port {debug_port}") + simulation_config: SimulationConfig | None = None + if simulation: + try: + simulation_config = SimulationConfig.model_validate_json(simulation) + except (ValidationError, ValueError) as e: + console.error(f"Invalid --simulation config: {e}") + return + attach_mode: DebugAttachMode | None = ( cast(DebugAttachMode, attach.lower()) if attach else None ) @@ -223,22 +239,35 @@ async def execute_debug_runtime(): if schema.metadata and "settings" in schema.metadata: agent_model = schema.metadata["settings"].get("model") - mocking_context = load_simulation_config( - agent_model=agent_model - ) - - mock_runtime = UiPathMockRuntime( - delegate=debug_runtime, - mocking_context=mocking_context, + delegate_runtime: UiPathDebugRuntime | UiPathMockRuntime = ( + debug_runtime ) + if simulation_config: + mocking_context = build_mocking_context( + simulation_config, agent_model + ) + if mocking_context: + delegate_runtime = UiPathMockRuntime( + delegate=debug_runtime, + mocking_context=mocking_context, + ) + else: + mocking_context = load_simulation_config( + agent_model=agent_model + ) + delegate_runtime = UiPathMockRuntime( + delegate=debug_runtime, + mocking_context=mocking_context, + ) try: - ctx.result = await mock_runtime.execute( + ctx.result = await delegate_runtime.execute( ctx.get_input(), options=UiPathExecuteOptions(resume=resume), ) finally: - await mock_runtime.dispose() + if delegate_runtime is not debug_runtime: + await delegate_runtime.dispose() await debug_runtime.dispose() if chat_runtime: await chat_runtime.dispose() diff --git a/packages/uipath/testcases/debug-simulation-testcase/pyproject.toml b/packages/uipath/testcases/debug-simulation-testcase/pyproject.toml new file mode 100644 index 000000000..e20c1b585 --- /dev/null +++ b/packages/uipath/testcases/debug-simulation-testcase/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "debug-simulation-testcase" +version = "0.0.1" +description = "debug-simulation-testcase" +authors = [{ name = "UiPath", email = "python-sdk@uipath.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } diff --git a/packages/uipath/testcases/debug-simulation-testcase/run.sh b/packages/uipath/testcases/debug-simulation-testcase/run.sh new file mode 100755 index 000000000..7880fa101 --- /dev/null +++ b/packages/uipath/testcases/debug-simulation-testcase/run.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +TESTCASE_DIR="$(cd "$(dirname "$0")" && pwd)" +SAMPLE_DIR="$(cd "$TESTCASE_DIR/../../samples/runtime-simulations-agent" && pwd)" + +echo "Syncing testcase dependencies (local editable uipath)..." +uv sync --project "$TESTCASE_DIR" + +UIPATH_BIN="$TESTCASE_DIR/.venv/bin/uipath" + +# Run auth and agent from the sample dir so credentials are stored and read +# from the same location. +cd "$SAMPLE_DIR" + +echo "Authenticating with UiPath..." +"$UIPATH_BIN" auth \ + --client-id="$CLIENT_ID" \ + --client-secret="$CLIENT_SECRET" \ + --base-url="$BASE_URL" + +echo "Running agent with debug + simulation..." +"$UIPATH_BIN" debug main \ + -f input.json \ + --attach none \ + --simulation "$(cat simulation.json)" 2>&1 | tee "$TESTCASE_DIR/run.log" + +# Copy the runtime output file back to the testcase dir for assert.py +mkdir -p "$TESTCASE_DIR/__uipath" +cp "$SAMPLE_DIR/__uipath/output.json" "$TESTCASE_DIR/__uipath/output.json" diff --git a/packages/uipath/testcases/debug-simulation-testcase/src/assert.py b/packages/uipath/testcases/debug-simulation-testcase/src/assert.py new file mode 100644 index 000000000..fd7e89697 --- /dev/null +++ b/packages/uipath/testcases/debug-simulation-testcase/src/assert.py @@ -0,0 +1,57 @@ +import json +import os + +# ── 1. Verify agent output exists and succeeded ────────────────────────────── +output_file = "__uipath/output.json" +assert os.path.isfile(output_file), "Agent output file not found" + +with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + +status = output_data.get("status") +assert status == "successful", f"Agent execution failed with status: {status}" + +output = output_data.get("output", {}) + +assert "syntax" in output, "Missing 'syntax' in output" +assert "style" in output, "Missing 'style' in output" +assert "improvements" in output, "Missing 'improvements' in output" +assert "summary" in output, "Missing 'summary' in output" + +assert isinstance(output["syntax"]["valid"], bool), "'syntax.valid' must be a bool" +assert isinstance(output["syntax"]["errors"], list), "'syntax.errors' must be a list" + +score = output["style"]["score"] +assert isinstance(score, int), "'style.score' must be an int" +assert 0 <= score <= 100, f"'style.score' out of range: {score}" +assert isinstance(output["style"]["violations"], list), ( + "'style.violations' must be a list" +) + +assert isinstance(output["improvements"]["suggestions"], list), ( + "'improvements.suggestions' must be a list" +) +assert isinstance(output["improvements"]["refactored_snippet"], str), ( + "'improvements.refactored_snippet' must be a str" +) + +# ── 2. Verify simulation produced non-default values ───────────────────────── +# Real tool impls always return: score=100, violations=[], suggestions=[]. +# The LLM simulation should detect issues in the input code and return richer output. +simulated_something = ( + score < 100 + or len(output["style"]["violations"]) > 0 + or len(output["improvements"]["suggestions"]) > 0 +) +assert simulated_something, ( + "Output matches hardcoded real-tool defaults — simulation may not have run. " + f"style.score={score}, violations={output['style']['violations']}, " + f"suggestions={output['improvements']['suggestions']}" +) + +print( + f"Simulation confirmed: score={score}, " + f"violations={len(output['style']['violations'])}, " + f"suggestions={len(output['improvements']['suggestions'])}" +) +print("All assertions passed.") diff --git a/packages/uipath/tests/cli/test_debug_simulation.py b/packages/uipath/tests/cli/test_debug_simulation.py index 9e66a1a24..9185d98c6 100644 --- a/packages/uipath/tests/cli/test_debug_simulation.py +++ b/packages/uipath/tests/cli/test_debug_simulation.py @@ -368,6 +368,225 @@ def test_simulation_config_enables_tool_mocking( # Clean up clear_execution_context() + +_SIMULATION_JSON = { + "enabled": True, + "toolsToSimulate": [{"name": "check_syntax"}, {"name": "check_style"}], + "instructions": "Simulate.", +} + +_COMPONENT_SIMULATION_JSON = { + "enabled": True, + "components": [ + { + "componentId": "get_current_weather", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Return realistic weather data", + }, + { + "componentId": "get_forecast", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Return a multi-day forecast", + }, + ], +} + + +class TestDebugSimulationFlag: + """Tests for the --simulation flag on the debug command.""" + + def _make_debug_patches(self): + """Create common mock objects for debug command tests.""" + mock_runtime = Mock() + mock_runtime.dispose = AsyncMock() + mock_runtime.get_schema = AsyncMock(return_value=Mock(metadata=None)) + + mock_factory = Mock() + mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) + mock_factory.get_settings = AsyncMock(return_value=Mock(trace_settings=None)) + mock_factory.dispose = AsyncMock() + + mock_debug_runtime = Mock() + mock_debug_runtime.dispose = AsyncMock() + + return mock_factory, mock_runtime, mock_debug_runtime + + def test_invalid_simulation_json_exits_with_error( + self, runner: CliRunner, temp_dir: str + ): + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + result = runner.invoke( + cli, ["debug", "main", "--simulation", "{ not valid json }"] + ) + assert result.exit_code == 1 + assert "Invalid" in result.output + + def test_simulation_flag_wraps_runtime_with_mock_runtime( + self, runner: CliRunner, temp_dir: str + ): + mock_factory, mock_runtime, mock_debug_runtime = self._make_debug_patches() + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_debug.Middlewares.next", + return_value=MiddlewareResult( + should_continue=True, + info_message=None, + error_message=None, + should_include_stacktrace=False, + ), + ), + patch( + "uipath._cli.cli_debug.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch("uipath._cli.cli_debug.get_debug_bridge"), + patch( + "uipath._cli.cli_debug.UiPathDebugRuntime", + return_value=mock_debug_runtime, + ), + patch("uipath._cli.cli_debug.UiPathMockRuntime") as mock_cls, + ): + mock_cls.return_value = Mock( + execute=AsyncMock( + return_value=Mock(status="SUCCESSFUL", output={}) + ), + dispose=AsyncMock(), + ) + runner.invoke( + cli, + [ + "debug", + "main", + "{}", + "--simulation", + json.dumps(_SIMULATION_JSON), + ], + ) + + assert mock_cls.called + assert mock_cls.call_args.kwargs["mocking_context"] is not None + assert mock_cls.call_args.kwargs["delegate"] is mock_debug_runtime + + def test_simulation_flag_disabled_does_not_wrap_runtime( + self, runner: CliRunner, temp_dir: str + ): + mock_factory, mock_runtime, mock_debug_runtime = self._make_debug_patches() + disabled = {**_SIMULATION_JSON, "enabled": False} + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_debug.Middlewares.next", + return_value=MiddlewareResult( + should_continue=True, + info_message=None, + error_message=None, + should_include_stacktrace=False, + ), + ), + patch( + "uipath._cli.cli_debug.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch("uipath._cli.cli_debug.get_debug_bridge"), + patch( + "uipath._cli.cli_debug.UiPathDebugRuntime", + return_value=mock_debug_runtime, + ), + patch("uipath._cli.cli_debug.UiPathMockRuntime") as mock_cls, + patch( + "uipath._cli.cli_debug.load_simulation_config", + return_value=None, + ), + ): + mock_debug_runtime.execute = AsyncMock( + return_value=Mock(status="SUCCESSFUL", output={}) + ) + runner.invoke( + cli, + ["debug", "main", "{}", "--simulation", json.dumps(disabled)], + ) + + assert not mock_cls.called + + def test_simulation_flag_with_component_format( + self, runner: CliRunner, temp_dir: str + ): + """Test that --simulation with new component format sets components on MockingContext.""" + mock_factory, mock_runtime, mock_debug_runtime = self._make_debug_patches() + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_debug.Middlewares.next", + return_value=MiddlewareResult( + should_continue=True, + info_message=None, + error_message=None, + should_include_stacktrace=False, + ), + ), + patch( + "uipath._cli.cli_debug.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch("uipath._cli.cli_debug.get_debug_bridge"), + patch( + "uipath._cli.cli_debug.UiPathDebugRuntime", + return_value=mock_debug_runtime, + ), + patch("uipath._cli.cli_debug.UiPathMockRuntime") as mock_cls, + ): + mock_cls.return_value = Mock( + execute=AsyncMock( + return_value=Mock(status="SUCCESSFUL", output={}) + ), + dispose=AsyncMock(), + ) + runner.invoke( + cli, + [ + "debug", + "main", + "{}", + "--simulation", + json.dumps(_COMPONENT_SIMULATION_JSON), + ], + ) + + assert mock_cls.called + mocking_context = mock_cls.call_args.kwargs["mocking_context"] + assert mocking_context is not None + assert mocking_context.components is not None + assert len(mocking_context.components) == 2 + assert mocking_context.components[0].component_id == "get_current_weather" + assert mocking_context.components[1].component_id == "get_forecast" + assert mocking_context.strategy is None + def test_middleware_short_circuits_before_mock_runtime( self, runner: CliRunner, diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 3eba8c870..eca6b2c62 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = ">=3.11" [options] -exclude-newer = "2026-07-29T07:23:41.8582141Z" +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P2D" [options.exclude-newer-package] @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.21" +version = "2.13.22" source = { editable = "." } dependencies = [ { name = "applicationinsights" },