From 03debb7c3edcf6f508d31800b53a93461b0da2d3 Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:06:09 -0600 Subject: [PATCH 1/7] Provide base tests/conftest.py Signed-off-by: Teo --- tests/conftest.py | 115 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..c3413f35 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,115 @@ +from typing import TYPE_CHECKING +import vcr +from vcr.record_mode import RecordMode +import pytest +import os +from collections import defaultdict + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + +# Store VCR stats across the session +vcr_session_stats = defaultdict(lambda: {"total": 0, "played": 0, "new": 0}) + +# Configure VCR +vcr_config = vcr.VCR( + record_mode=RecordMode.ONCE, + match_on=["method", "scheme", "host", "port", "path", "query"], + filter_headers=[ + "authorization", + "x-api-key", + "api-key", + "bearer", + "openai-api-key", + "anthropic-api-key", + # Additional common API authentication headers + "x-api-token", + "api-token", + "x-auth-token", + "x-session-token", + # LLM specific headers + "cohere-api-key", + "x-cohere-api-key", + "ai21-api-key", + "x-ai21-api-key", + "replicate-api-token", + "huggingface-api-key", + "x-huggingface-api-key", + "claude-api-key", + "x-claude-api-key", + # OpenAI specific headers + "openai-organization", + "x-request-id", + "__cf_bm", + "_cfuvid", + "cf-ray", + # Rate limit headers that may expose account details + "x-ratelimit-limit-requests", + "x-ratelimit-limit-tokens", + "x-ratelimit-remaining-requests", + "x-ratelimit-remaining-tokens", + "x-ratelimit-reset-requests", + "x-ratelimit-reset-tokens", + ], + # Filter out any api keys from query parameters + filter_query_parameters=["api_key", "api-key", "token"], + # Ignore requests to internal/package management APIs + ignore_hosts=["api.agentops.ai", "pypi.org", "files.pythonhosted.org", "upload.pypi.org", "test.pypi.org"], +) + + +@pytest.fixture +def vcr_cassette(request): + """Provides VCR cassette with standard LLM API filtering""" + # Get the directory of the test file + test_dir = os.path.dirname(request.module.__file__) + cassette_dir = os.path.join(test_dir, ".cassettes") + os.makedirs(cassette_dir, exist_ok=True) + + cassette_path = os.path.join(cassette_dir, f"{request.node.name}.yaml") + + # Override the cassette dir for this specific cassette + with vcr.use_cassette( + cassette_path, + record_mode=vcr_config.record_mode, + match_on=vcr_config.match_on, + filter_headers=vcr_config.filter_headers, + filter_query_parameters=vcr_config.filter_query_parameters, + ignore_hosts=vcr_config.ignore_hosts, + ) as cassette: + yield cassette + + if len(cassette.requests) > 0: + new_recordings = len(cassette.responses) - cassette.play_count + if new_recordings > 0: + # Update session stats + vcr_session_stats[test_dir]["total"] += len(cassette.requests) + vcr_session_stats[test_dir]["played"] += cassette.play_count + vcr_session_stats[test_dir]["new"] += new_recordings + + +def pytest_sessionfinish(session): + """Print VCR stats at the end of the session""" + if vcr_session_stats: + print("\n=== VCR Session Stats ===") + for test_dir, stats in vcr_session_stats.items(): + if stats["new"] > 0: # Only show directories with new recordings + print(f"\nšŸ“ {os.path.basename(test_dir)}") + print(f" ā”œā”€ Total Requests: {stats['total']}") + print(f" ā”œā”€ Played: {stats['played']}") + print(f" ā””ā”€ New: {stats['new']}") + print("=======================\n") + + +@pytest.fixture(scope="function") +def llm_event_spy(agentops_client, mocker: "MockerFixture"): + """Fixture that provides spies on both providers' response handling""" + from agentops.llms.providers.anthropic import AnthropicProvider + from agentops.llms.providers.litellm import LiteLLMProvider + from agentops.llms.providers.openai import OpenAiProvider + + return { + "litellm": mocker.spy(LiteLLMProvider(agentops_client), "handle_response"), + "openai": mocker.spy(OpenAiProvider(agentops_client), "handle_response"), + "anthropic": mocker.spy(AnthropicProvider(agentops_client), "handle_response"), + } From 5d785ded3004facd631d2e24ae0f3c1fd1de806c Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:06:41 -0600 Subject: [PATCH 2/7] docs: update testing documentation and add README.md --- CONTRIBUTING.md | 183 ++---------------------------------------------- tests/README.md | 121 ++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 176 deletions(-) create mode 100644 tests/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1db574e5..0ea9eb93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,184 +91,15 @@ Even if you're not ready to contribute code, we'd love to hear your thoughts. Dr ## Testing -We use a comprehensive testing stack to ensure code quality and reliability. Our testing framework includes pytest and several specialized testing tools. - -### Testing Dependencies - -Install all testing dependencies: -```bash -pip install -e ".[dev]" -``` - -We use the following testing packages: -- `pytest==7.4.0`: Core testing framework -- `pytest-depends`: Manage test dependencies -- `pytest-asyncio`: Test async code -- `pytest-vcr`: Record and replay HTTP interactions -- `pytest-mock`: Mocking functionality -- `pyfakefs`: Mock filesystem operations -- `requests_mock==1.11.0`: Mock HTTP requests -- `tach~=0.9`: Performance testing and dependency tracking to prevent circular dependencies - -### Using Tox - -We use tox to automate and standardize testing. Tox: -- Creates isolated virtual environments for testing -- Tests against multiple Python versions (3.7-3.12) -- Runs all test suites consistently -- Ensures dependencies are correctly specified -- Verifies the package installs correctly - -Run tox: -```bash -tox -``` - -This will: -1. Create fresh virtual environments -2. Install dependencies -3. Run pytest with our test suite -4. Generate coverage reports - -### Running Tests - -1. **Run All Tests**: - ```bash - tox - ``` - -2. **Run Specific Test File**: - ```bash - pytest tests/llms/test_anthropic.py -v - ``` - -3. **Run with Coverage**: - ```bash - coverage run -m pytest - coverage report - ``` - -### Writing Tests - -1. **Test Structure**: - ```python - import pytest - from pytest_mock import MockerFixture - from unittest.mock import Mock, patch - - @pytest.mark.asyncio # For async tests - async def test_async_function(): - # Test implementation - - @pytest.mark.depends(on=['test_prerequisite']) # Declare test dependencies - def test_dependent_function(): - # Test implementation - ``` - -2. **Recording HTTP Interactions**: - ```python - @pytest.mark.vcr() # Records HTTP interactions - def test_api_call(): - response = client.make_request() - assert response.status_code == 200 - ``` - -3. **Mocking Filesystem**: - ```python - def test_file_operations(fs): # fs fixture provided by pyfakefs - fs.create_file('/fake/file.txt', contents='test') - assert os.path.exists('/fake/file.txt') - ``` - -4. **Mocking HTTP Requests**: - ```python - def test_http_client(requests_mock): - requests_mock.get('http://api.example.com', json={'key': 'value'}) - response = make_request() - assert response.json()['key'] == 'value' - ``` - -### Testing Best Practices - -1. **Test Categories**: - - Unit tests: Test individual components - - Integration tests: Test component interactions - - End-to-end tests: Test complete workflows - - Performance tests: Test response times and resource usage - -2. **Fixtures**: - Create reusable test fixtures in `conftest.py`: - ```python - @pytest.fixture - def mock_llm_client(): - client = Mock() - client.chat.completions.create.return_value = Mock() - return client - ``` - -3. **Test Data**: - - Store test data in `tests/data/` - - Use meaningful test data names - - Document data format and purpose - -4. **VCR Cassettes**: - - Store in `tests/cassettes/` - - Sanitize sensitive information - - Update cassettes when API changes - -5. **Performance Testing**: - ```python - from tach import Tach +We maintain comprehensive testing documentation in [tests/README.md](tests/README.md). This includes: - def test_performance(): - with Tach('operation_name'): - perform_operation() - ``` - -### CI Testing Strategy - -We use Jupyter notebooks as integration tests for LLM providers. This approach: -- Tests real-world usage patterns -- Verifies end-to-end functionality -- Ensures examples stay up-to-date -- Tests against actual LLM APIs - -1. **Notebook Tests**: - - Located in `examples/` directory - - Each LLM provider has example notebooks - - CI runs notebooks on PR merges to main - - Tests run against multiple Python versions - -2. **Test Workflow**: - The `test-notebooks.yml` workflow: - ```yaml - name: Test Notebooks - on: - pull_request: - paths: - - "agentops/**" - - "examples/**" - - "tests/**" - ``` - - Runs on PR merges and manual triggers - - Sets up environment with provider API keys - - Installs AgentOps from main branch - - Executes each notebook - - Excludes specific notebooks that require manual testing - -3. **Provider Coverage**: - Each provider should have notebooks demonstrating: - - Basic completion calls - - Streaming responses - - Async operations (if supported) - - Error handling - - Tool usage (if applicable) +- Test structure and organization +- How to run tests +- Using VCR.py for HTTP interaction testing +- Writing new tests +- Test dependencies and setup -4. **Adding Provider Tests**: - - Create notebook in `examples/provider_name/` - - Include all provider functionality - - Add necessary secrets to GitHub Actions - - Update `exclude_notebooks` in workflow if manual testing needed +For detailed testing instructions and best practices, please refer to the testing documentation. ## Adding LLM Providers diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..3038eb01 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,121 @@ +# Testing AgentOps + +This directory contains the test suite for AgentOps. We use a comprehensive testing stack including pytest and several specialized testing tools. + +## Running Tests + +1. **Run All Tests**: + ```bash + pytest + ``` + +2. **Run Specific Test File**: + ```bash + pytest tests/providers/test_openai_integration.py + ``` + +3. **Run with Coverage**: + ```bash + coverage run -m pytest + coverage report + ``` + +## Writing Tests + +1. **Test Structure**: + ```python + import pytest + from pytest_mock import MockerFixture + from unittest.mock import Mock, patch + + @pytest.mark.asyncio # For async tests + async def test_async_function(): + # Test implementation + + @pytest.mark.depends(on=['test_prerequisite']) # Declare test dependencies + def test_dependent_function(): + # Test implementation + ``` + +2. **Using Fixtures**: + ```python + def test_with_mocks(llm_event_spy): + # Use the spy to track LLM events + pass + ``` + +3. **Using VCR**: + ```python + def test_api_call(vcr_cassette): + # Make API calls - they will be recorded/replayed automatically + response = client.make_api_call() + ``` + +## Test Categories + +### Core Tests +- Unit tests for core functionality +- Integration tests for SDK features +- Performance benchmarks + +### Provider Tests +Tests for LLM provider integrations. See [providers/README.md](providers/README.md) for details on: +- VCR.py configuration for recording API interactions +- Provider-specific test configuration +- Recording and managing API fixtures + +### Manual Tests +Located in `core_manual_tests/`: +- API server tests +- Multi-session scenarios +- Provider-specific canary tests +- Time travel debugging tests + +## Test Dependencies + +Required packages are included in the dev dependencies: +```bash +pip install -e ".[dev]" +``` + +Key testing packages: +- `pytest`: Core testing framework +- `pytest-depends`: Manage test dependencies +- `pytest-asyncio`: Test async code +- `pytest-vcr`: Record and replay HTTP interactions +- `pytest-mock`: Mocking functionality +- `pyfakefs`: Mock filesystem operations +- `requests_mock`: Mock HTTP requests + +## Best Practices + +1. **Recording API Fixtures**: + - Use VCR.py to record API interactions + - Fixtures are stored in `.cassettes` directories + - VCR automatically filters sensitive headers and API keys + - New recordings are summarized at the end of test runs + +2. **Test Isolation**: + - Use fresh sessions for each test + - Clean up resources in test teardown + - Avoid test interdependencies + +3. **Async Testing**: + - Use `@pytest.mark.asyncio` for async tests + - Handle both sync and async variants + - Test streaming responses properly + +## VCR Configuration + +The VCR setup automatically: +- Records API interactions on first run +- Replays recorded responses on subsequent runs +- Filters sensitive information (API keys, tokens) +- Ignores AgentOps API and package management calls +- Creates `.cassettes` directories as needed +- Reports new recordings in the test summary + +To update existing cassettes: +1. Delete the relevant `.cassette` file +2. Run the tests +3. Verify the new recordings in the VCR summary From fdb9d36e313177650d1b1509788ed6ed1211182f Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:06:46 -0600 Subject: [PATCH 3/7] test: move providers tests and add cassettes for OpenAI provider tests --- .../.cassettes/test_openai_integration.yaml | 577 ++++++++++++++++++ .../test_provider_openai.py} | 3 + 2 files changed, 580 insertions(+) create mode 100644 tests/providers/.cassettes/test_openai_integration.yaml rename tests/{openai_handlers/test_openai_integration.py => providers/test_provider_openai.py} (98%) diff --git a/tests/providers/.cassettes/test_openai_integration.yaml b/tests/providers/.cassettes/test_openai_integration.yaml new file mode 100644 index 00000000..7e06591c --- /dev/null +++ b/tests/providers/.cassettes/test_openai_integration.yaml @@ -0,0 +1,577 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Hello from sync no stream"}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '90' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.58.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.58.1 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4xTy27bMBC86ys2vPQiBbbi2okvgWGkaND2UKBAD00h0NRKYkTtEiRl1w3y7wVl + x3LQFOiFIGZ2h7MPPiUAQpdiCUI1MqjOmmzVfNvy/MN2sf3q2u/rL5+m9vfdqq1Xj/n6TqQxgzeP + qMJL1qXizhoMmulAK4cyYFSdLq5mN/n8fT4biI5LNDGttiGbcdZp0lk+yWfZZJFNr4/ZDWuFXizh + RwIA8DSc0SeV+EssYZK+IB16L2sUy1MQgHBsIiKk99oHSUGkI6mYAtJg/SMawxdwH8BzT6UHo1uE + PffvHILDCh2S0lSD5w5DM9wsKl1pdQlr7k0Zg0EZ6XS1h10jwwB0KAk2e3gQfk8KiMEHh7J7ELew + csMLYJjbKFixA00Vu07G9gETSAioGtJKGtDe95iCBMWk0IYU2J3ZQePxFj5jgA6hJd5Bwzu4ByUJ + DtXHxy7O63dY9V7GGVBvzBF/PjXUcG0db/yRP+GVJu2bwqH0TLF5PrAVA/ucAPwcBte/moWwjjsb + isAtUhSc5gc5Ma7LSM5ujmTgIM2Iz6fpG2pFiUFq488GL5RUDZZj5rglsi81nxHJWc1/m3lL+1C3 + pvp/5EdCxZlhWViHpVavCx7DHMbP9K+wU48Hw8LvfcCuqDTV6KzTh1WubDGR8rq8wnyyEclz8gcA + AP//AwDTjqJw2AMAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f657439cff78788-DFW + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 23 Dec 2024 04:02:05 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=JnEiTAqjLExJCFHShFG.Xhs8w8olXu1zUPI0bhcTBLQ-1734926525-1.0.1.1-knsMI40s_q1GDjUiykGdVDIaYZB79IVFD6NUYKqcuqBoHDLbaxvjVgT70SdzCbRlC7Qi7uUv1FZ3AVpNs9EDWw; + path=/; expires=Mon, 23-Dec-24 04:32:05 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=j_JsEcigucw66WqIGz7wo7I6uN..u6AiPacVFPxy4TM-1734926525200-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - team-cobra-xebc3o + openai-processing-ms: + - '676' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999976' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_0600f3fa0dad9578892b055a559be71a + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"Hello from sync streaming"}],"model":"gpt-4o-mini","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '104' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.58.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.58.1 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":" + How"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":" + can"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":" + I"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":" + assist"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":" + you"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":" + today"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvpS67dRpMfNBXiil0P1M1ulGou","object":"chat.completion.chunk","created":1734926525,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_d02d531b47","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f65743fced8e78e-DFW + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 23 Dec 2024 04:02:05 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=lWWCV2SM6ubBoVO4TflUiGYKbsBuRGTfLwAS_0RlEkI-1734926525-1.0.1.1-nLG0xD4MaCh.FF950emmCoQwiTvYUj_kEjAkzZlvei7KuKmK71qg5wN2Dcer2evDqjqhQbCKk7iTKdqGW3XZnA; + path=/; expires=Mon, 23-Dec-24 04:32:05 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=I97zJKxPMhEA7KDnhHvxJCcR9FzkYruUZDGvI9Sxt2E-1734926525619-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - team-cobra-xebc3o + openai-processing-ms: + - '136' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999976' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_5248dfa27425bb259f1a994f757580c4 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"Hello from async no stream"}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '91' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.58.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.58.1 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4xTsW7bMBDd/RUXLlmkwJbtuPFSFOhgo0CHNkCHohBo6iQxoXgK79RaDfzvBWXH + ctAU6MLh3r3Hd4/H5wmAsoVagzK1FtO0Lv1Q3/98Kn+vvsw/rr7q7b43i0+b+8+o78zqQSWRQbsH + NPLCujHUtA7Fkj/CJqAWjKqz1Xxxl90us9sBaKhAF2lVK+mC0sZ6m2bTbJFOV+ns3YldkzXIag3f + JwAAz8MZffoC92oN0+Sl0iCzrlCtz00AKpCLFaWZLYv2opIRNOQF/WB9g87RFWwFGLFh6Km7DggB + SwzojfUVaA+ae2/qQJ46hoG9F6AAGlgC6iZ1yAzWCwZtYgQ3sKFfYLSHLRwtRGUQKnT/Hr7VKDUG + sHLNoOGpQ44k0DvqBNpAVdBNY32VgKCpPTmq+iReyNSg1NEVOsYESkQHZUAEIdD8eHU5ZsCyYx2j + 9p1zp/rhnJujqg204xN+rpfWW67zgJrJx4xYqFUDepgA/Bjep3sVuWoDNa3kQo/oo+AsO8qpcStG + cDE7gUKi3VhfzpM31PICRVvHF++rjDY1FiNzXAbdFZYugMnFzH+beUv7OLf11f/Ij4Ax2AoWeRuw + sOb1wGNbwPhn/tV2zngwrLhnwSYvra8wtMEeN7Zs82KaFcv5bLdYqclh8gcAAP//AwAQ0AR/vwMA + AA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f6574431ad1eafe-DFW + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 23 Dec 2024 04:02:06 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=Ghh0qPlBvccsesqoYaIrjd8LFLkZAe1aCvfaVg0hboA-1734926526-1.0.1.1-enJWQJ_BaBt5AOG6b6mEMt.5JIBZWgCd_RVxqE4G5_z8HVOT.jBuXEWYjJZEXOquVdouVHrMhi70.WrHYYi8Mw; + path=/; expires=Mon, 23-Dec-24 04:32:06 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=zVW7qbZq61pX53gwzZDJybA_75vOLJAuMlDlo2H6BkM-1734926526572-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - team-cobra-xebc3o + openai-processing-ms: + - '563' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999976' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_02a30509d9f530c6dc355cbc02bb0bc1 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"Hello from async streaming"}],"model":"gpt-4o-mini","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '105' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.58.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.58.1 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + It"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + sounds"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + like"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + you''re"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + working"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + with"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + async"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + streaming"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + How"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + can"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + I"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + assist"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + you"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + with"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + that"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + Whether"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + it''s"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + questions"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + about"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + implementation"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + best"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + practices"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + or"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + something"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + else"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + feel"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + free"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + to"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":" + share"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AhTvqX3VRL4EYV55fFT0Zzge6yuIe","object":"chat.completion.chunk","created":1734926526,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_0aa8d3e20b","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f6574481a5ee9c6-DFW + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 23 Dec 2024 04:02:06 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=zaiy4gEEydxEOZUEU5t4hskGaTqak_Q_N84xsC2sjtc-1734926526-1.0.1.1-C5rso22nY.keB.gEfbBOT_jjMuuw7wPLEbpADRxm05LN73.4RwxaCs2REdhXQQRzq02gKkKOf6Gu5A7PxaSrWQ; + path=/; expires=Mon, 23-Dec-24 04:32:06 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=Xx_sGTrYcqnOW0pAkLa.ZMd.FIFrO2DLYkIYgc.CCn0-1734926526936-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - team-cobra-xebc3o + openai-processing-ms: + - '134' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999976' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_bfeaa834eabecbbd7fcebf3fe0f3b3f6 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/openai_handlers/test_openai_integration.py b/tests/providers/test_provider_openai.py similarity index 98% rename from tests/openai_handlers/test_openai_integration.py rename to tests/providers/test_provider_openai.py index 8b2a0fec..0fd149ee 100644 --- a/tests/openai_handlers/test_openai_integration.py +++ b/tests/providers/test_provider_openai.py @@ -8,6 +8,9 @@ load_dotenv() +pytestmark = [pytest.mark.vcr] + + @pytest.mark.integration def test_openai_integration(): """Integration test demonstrating all four OpenAI call patterns: From 2b3723bc5ef7d4ccf6382a366181681a1a5186b5 Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:07:43 -0600 Subject: [PATCH 4/7] pyproject.toml housekeeping Signed-off-by: Teo --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f8e0a770..ed846fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,8 @@ langchain = [ "langchain==0.2.14; python_version >= '3.8.1'" ] +[project.scripts] +agentops = "agentops.cli:main" [project.urls] Homepage = "https://github.com/AgentOps-AI/agentops" @@ -59,12 +61,10 @@ Issues = "https://github.com/AgentOps-AI/agentops/issues" [tool.autopep8] max_line_length = 120 -[project.scripts] -agentops = "agentops.cli:main" [tool.pytest.ini_options] asyncio_mode = "strict" -asyncio_default_fixture_loop_scope = "function" +asyncio_default_fixture_loop_scope = "function" # WARNING: Changing this may break tests. A `module`-scoped session might be faster, but also unstable. test_paths = [ "tests", ] From cb3b9312c3c0cdf7210603be7bf08c14bcf78d1a Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:07:57 -0600 Subject: [PATCH 5/7] Update maintainers list Signed-off-by: Teo --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ed846fba..28b54145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,8 @@ authors = [ { name="Alex Reibman", email="areibman@gmail.com" }, { name="Shawn Qiu", email="siyangqiu@gmail.com" }, { name="Braelyn Boynton", email="bboynton97@gmail.com" }, - { name="Howard Gil", email="howardbgil@gmail.com" } + { name="Howard Gil", email="howardbgil@gmail.com" }, + { name="Constantin Teodorescu", email="teocns@gmail.com"} ] description = "Observability and DevTool Platform for AI Agents" readme = "README.md" From 3757e75513603129aa633b1028a5decab5357fc1 Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:34:27 -0600 Subject: [PATCH 6/7] fix test_host_env: apaptable to psutil deps --- tests/test_host_env.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/test_host_env.py b/tests/test_host_env.py index c22796f3..39d10136 100644 --- a/tests/test_host_env.py +++ b/tests/test_host_env.py @@ -1,15 +1,27 @@ from unittest.mock import patch from agentops import host_env +import psutil # noinspection PyProtectedMember from psutil._common import sdiskpart, sdiskusage def mock_partitions(): - return [ - sdiskpart(device="/dev/sda1", mountpoint="/", fstype="ext4", opts="rw,relatime"), - sdiskpart(device="z:\\", mountpoint="z:\\", fstype="ntfs", opts="rw,relatime"), - ] + # Try to create with new fields first, fall back to old format if it fails + try: + return [ + sdiskpart( # noqa: E501 + device="/dev/sda1", + mountpoint="/", + fstype="ext4", + opts="rw,relatime", + maxfile=255, # type: ignore + maxpath=4096, # type: ignore + ) + ] + except TypeError: + # Fallback for older versions that don't have maxfile/maxpath + return [sdiskpart(device="/dev/sda1", mountpoint="/", fstype="ext4", opts="rw,relatime")] def mock_disk_usage(partition): From 6309a0a4709cfccd93e35e7f6d118c51cf442d90 Mon Sep 17 00:00:00 2001 From: Teo Date: Sun, 22 Dec 2024 22:58:02 -0600 Subject: [PATCH 7/7] refactor(tests): move VCR configuration to tests/providers --- tests/conftest.py | 92 ---------------------- tests/providers/conftest.py | 153 ++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 92 deletions(-) create mode 100644 tests/providers/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py index c3413f35..9c292100 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,98 +8,6 @@ if TYPE_CHECKING: from pytest_mock import MockerFixture -# Store VCR stats across the session -vcr_session_stats = defaultdict(lambda: {"total": 0, "played": 0, "new": 0}) - -# Configure VCR -vcr_config = vcr.VCR( - record_mode=RecordMode.ONCE, - match_on=["method", "scheme", "host", "port", "path", "query"], - filter_headers=[ - "authorization", - "x-api-key", - "api-key", - "bearer", - "openai-api-key", - "anthropic-api-key", - # Additional common API authentication headers - "x-api-token", - "api-token", - "x-auth-token", - "x-session-token", - # LLM specific headers - "cohere-api-key", - "x-cohere-api-key", - "ai21-api-key", - "x-ai21-api-key", - "replicate-api-token", - "huggingface-api-key", - "x-huggingface-api-key", - "claude-api-key", - "x-claude-api-key", - # OpenAI specific headers - "openai-organization", - "x-request-id", - "__cf_bm", - "_cfuvid", - "cf-ray", - # Rate limit headers that may expose account details - "x-ratelimit-limit-requests", - "x-ratelimit-limit-tokens", - "x-ratelimit-remaining-requests", - "x-ratelimit-remaining-tokens", - "x-ratelimit-reset-requests", - "x-ratelimit-reset-tokens", - ], - # Filter out any api keys from query parameters - filter_query_parameters=["api_key", "api-key", "token"], - # Ignore requests to internal/package management APIs - ignore_hosts=["api.agentops.ai", "pypi.org", "files.pythonhosted.org", "upload.pypi.org", "test.pypi.org"], -) - - -@pytest.fixture -def vcr_cassette(request): - """Provides VCR cassette with standard LLM API filtering""" - # Get the directory of the test file - test_dir = os.path.dirname(request.module.__file__) - cassette_dir = os.path.join(test_dir, ".cassettes") - os.makedirs(cassette_dir, exist_ok=True) - - cassette_path = os.path.join(cassette_dir, f"{request.node.name}.yaml") - - # Override the cassette dir for this specific cassette - with vcr.use_cassette( - cassette_path, - record_mode=vcr_config.record_mode, - match_on=vcr_config.match_on, - filter_headers=vcr_config.filter_headers, - filter_query_parameters=vcr_config.filter_query_parameters, - ignore_hosts=vcr_config.ignore_hosts, - ) as cassette: - yield cassette - - if len(cassette.requests) > 0: - new_recordings = len(cassette.responses) - cassette.play_count - if new_recordings > 0: - # Update session stats - vcr_session_stats[test_dir]["total"] += len(cassette.requests) - vcr_session_stats[test_dir]["played"] += cassette.play_count - vcr_session_stats[test_dir]["new"] += new_recordings - - -def pytest_sessionfinish(session): - """Print VCR stats at the end of the session""" - if vcr_session_stats: - print("\n=== VCR Session Stats ===") - for test_dir, stats in vcr_session_stats.items(): - if stats["new"] > 0: # Only show directories with new recordings - print(f"\nšŸ“ {os.path.basename(test_dir)}") - print(f" ā”œā”€ Total Requests: {stats['total']}") - print(f" ā”œā”€ Played: {stats['played']}") - print(f" ā””ā”€ New: {stats['new']}") - print("=======================\n") - @pytest.fixture(scope="function") def llm_event_spy(agentops_client, mocker: "MockerFixture"): diff --git a/tests/providers/conftest.py b/tests/providers/conftest.py new file mode 100644 index 00000000..712f0c6a --- /dev/null +++ b/tests/providers/conftest.py @@ -0,0 +1,153 @@ +from typing import TYPE_CHECKING +import vcr +from vcr.record_mode import RecordMode +import pytest +import os +from collections import defaultdict + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + + +# INFO: When we are ready, we could move this to global `tests/conftest.py` + +# Store VCR stats across the session +vcr_session_stats = defaultdict(lambda: {"total": 0, "played": 0, "new": 0}) + +# Configure VCR +vcr_config = vcr.VCR( + record_mode=RecordMode.ONCE, + match_on=["method", "scheme", "host", "port", "path", "query"], + filter_headers=[ + "authorization", + "x-api-key", + "api-key", + "bearer", + "openai-api-key", + "anthropic-api-key", + # Additional common API authentication headers + "x-api-token", + "api-token", + "x-auth-token", + "x-session-token", + # LLM specific headers + "cohere-api-key", + "x-cohere-api-key", + "ai21-api-key", + "x-ai21-api-key", + "replicate-api-token", + "huggingface-api-key", + "x-huggingface-api-key", + "claude-api-key", + "x-claude-api-key", + # OpenAI specific headers + "openai-organization", + "x-request-id", + "__cf_bm", + "_cfuvid", + "cf-ray", + # Rate limit headers that may expose account details + "x-ratelimit-limit-requests", + "x-ratelimit-limit-tokens", + "x-ratelimit-remaining-requests", + "x-ratelimit-remaining-tokens", + "x-ratelimit-reset-requests", + "x-ratelimit-reset-tokens", + ], + # Filter out any api keys from query parameters + filter_query_parameters=["api_key", "api-key", "token"], + # Ignore requests to internal/package management APIs + ignore_hosts=["api.agentops.ai", "pypi.org", "files.pythonhosted.org", "upload.pypi.org", "test.pypi.org"], +) + + +@pytest.fixture +def vcr_cassette(request): + """Provides VCR cassette with standard LLM API filtering""" + # Get the directory of the test file + test_dir = os.path.dirname(request.module.__file__) + cassette_dir = os.path.join(test_dir, ".cassettes") + os.makedirs(cassette_dir, exist_ok=True) + + cassette_path = os.path.join(cassette_dir, f"{request.node.name}.yaml") + + # Override the cassette dir for this specific cassette + with vcr.use_cassette( + cassette_path, + record_mode=vcr_config.record_mode, + match_on=vcr_config.match_on, + filter_headers=vcr_config.filter_headers, + filter_query_parameters=vcr_config.filter_query_parameters, + ignore_hosts=vcr_config.ignore_hosts, + ) as cassette: + yield cassette + + if len(cassette.requests) > 0: + new_recordings = len(cassette.responses) - cassette.play_count + if new_recordings > 0: + # Update session stats + vcr_session_stats[test_dir]["total"] += len(cassette.requests) + vcr_session_stats[test_dir]["played"] += cassette.play_count + vcr_session_stats[test_dir]["new"] += new_recordings + + +def pytest_sessionfinish(session): + """Print VCR stats at the end of the session""" + if vcr_session_stats: + print("\n=== VCR Session Stats ===") + for test_dir, stats in vcr_session_stats.items(): + if stats["new"] > 0: # Only show directories with new recordings + print(f"\nšŸ“ {os.path.basename(test_dir)}") + print(f" ā”œā”€ Total Requests: {stats['total']}") + print(f" ā”œā”€ Played: {stats['played']}") + print(f" ā””ā”€ New: {stats['new']}") + print("=======================\n") + + + +# Store VCR stats across the session +vcr_session_stats = defaultdict(lambda: {"total": 0, "played": 0, "new": 0}) + +@pytest.fixture +def vcr_cassette(request): + """Provides VCR cassette with standard LLM API filtering""" + # Get the directory of the test file + test_dir = os.path.dirname(request.module.__file__) + cassette_dir = os.path.join(test_dir, ".cassettes") + os.makedirs(cassette_dir, exist_ok=True) + + cassette_path = os.path.join(cassette_dir, f"{request.node.name}.yaml") + + # Override the cassette dir for this specific cassette + with vcr.use_cassette( + cassette_path, + record_mode=vcr_config.record_mode, + match_on=vcr_config.match_on, + filter_headers=vcr_config.filter_headers, + filter_query_parameters=vcr_config.filter_query_parameters, + ignore_hosts=vcr_config.ignore_hosts, + ) as cassette: + yield cassette + + if len(cassette.requests) > 0: + new_recordings = len(cassette.responses) - cassette.play_count + if new_recordings > 0: + # Update session stats + vcr_session_stats[test_dir]["total"] += len(cassette.requests) + vcr_session_stats[test_dir]["played"] += cassette.play_count + vcr_session_stats[test_dir]["new"] += new_recordings + + +def pytest_sessionfinish(session): + """Print VCR stats at the end of the session""" + if vcr_session_stats: + print("\n=== VCR Session Stats ===") + for test_dir, stats in vcr_session_stats.items(): + if stats["new"] > 0: # Only show directories with new recordings + print(f"\nšŸ“ {os.path.basename(test_dir)}") + print(f" ā”œā”€ Total Requests: {stats['total']}") + print(f" ā”œā”€ Played: {stats['played']}") + print(f" ā””ā”€ New: {stats['new']}") + print("=======================\n") + +