From 1b8e179580b90456016b351869007a2ff864c2bf Mon Sep 17 00:00:00 2001 From: ATC964 <82515918+atc964@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:39:26 -0400 Subject: [PATCH 1/3] Remove hardcoded CPM fallbacks that fabricate pricing (Layer 2a) When sellers provide no pricing, the buyer agent was silently substituting hardcoded values ($20 in request_deal, $15 in campaign_pipeline, $0 in discover_inventory/quote_flow). This caused the agent to present fabricated CPMs as if they came from the seller. Changes: - request_deal.py: return error string instead of $20 fallback - discover_inventory.py: show "Pricing unavailable" instead of $0 - quote_flow.py: return None instead of PricingResult with $0 - campaign_pipeline.py: require explicit CPM, default to None bead: ar-na3i Co-Authored-By: Claude Opus 4.6 (1M context) --- src/ad_buyer/booking/quote_flow.py | 17 +- src/ad_buyer/pipelines/campaign_pipeline.py | 16 +- src/ad_buyer/tools/dsp/discover_inventory.py | 14 +- src/ad_buyer/tools/dsp/request_deal.py | 16 +- tests/unit/test_booking/test_pricing.py | 4 +- tests/unit/test_cpm_fallback_removal.py | 345 +++++++++++++++++++ tests/unit/test_dsp_discovery_pricing.py | 7 +- 7 files changed, 393 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_cpm_fallback_removal.py diff --git a/src/ad_buyer/booking/quote_flow.py b/src/ad_buyer/booking/quote_flow.py index ad8825e..d763031 100644 --- a/src/ad_buyer/booking/quote_flow.py +++ b/src/ad_buyer/booking/quote_flow.py @@ -56,7 +56,7 @@ def get_pricing( volume: int | None = None, target_cpm: float | None = None, deal_type: str | None = None, - ) -> PricingResult: + ) -> PricingResult | None: """Calculate pricing for a product based on buyer context. Args: @@ -66,11 +66,12 @@ def get_pricing( deal_type: Deal type requested. Returns: - PricingResult with all pricing details. + PricingResult with all pricing details, or None if no + valid pricing is available from the seller. """ - base_price = product.get("basePrice", product.get("price", 0)) + base_price = product.get("basePrice", product.get("price")) if not isinstance(base_price, (int, float)): - base_price = 0 + return None tier = self._buyer_context.identity.get_access_tier() tier_discount = self._buyer_context.identity.get_discount_percentage() @@ -96,7 +97,7 @@ def build_deal_data( flight_start: str | None = None, flight_end: str | None = None, target_cpm: float | None = None, - ) -> dict[str, Any]: + ) -> dict[str, Any] | None: """Build deal data dict for deal creation. Calculates pricing and generates a Deal ID, returning @@ -113,7 +114,8 @@ def build_deal_data( Returns: Dict with deal details including deal_id, pricing, - and activation instructions. + and activation instructions. Returns None if the + product has no valid pricing available. """ # Calculate pricing pricing = self.get_pricing( @@ -123,6 +125,9 @@ def build_deal_data( deal_type=deal_type, ) + if pricing is None: + return None + # Generate deal ID identity = self._buyer_context.identity identity_seed = identity.agency_id or identity.seat_id or "public" diff --git a/src/ad_buyer/pipelines/campaign_pipeline.py b/src/ad_buyer/pipelines/campaign_pipeline.py index c49e1ff..32e0acd 100644 --- a/src/ad_buyer/pipelines/campaign_pipeline.py +++ b/src/ad_buyer/pipelines/campaign_pipeline.py @@ -623,17 +623,23 @@ def _reconstruct_brief(self, campaign: dict[str, Any]) -> CampaignBrief: }) @staticmethod - def _estimate_impressions(budget: float, assumed_cpm: float = 15.0) -> int: - """Estimate impression count from budget and assumed CPM. + def _estimate_impressions( + budget: float, assumed_cpm: float | None = None + ) -> int: + """Estimate impression count from budget and CPM. + + When no CPM is available (assumed_cpm is None), returns 0 + rather than fabricating impressions from a made-up price. Args: budget: Channel budget in currency units. - assumed_cpm: Assumed CPM for estimation (default $15). + assumed_cpm: CPM to use for estimation. Must be explicitly + provided — no default is assumed. Returns: - Estimated number of impressions. + Estimated number of impressions, or 0 if no CPM available. """ - if budget <= 0 or assumed_cpm <= 0: + if assumed_cpm is None or budget <= 0 or assumed_cpm <= 0: return 0 # impressions = (budget / CPM) * 1000 return int((budget / assumed_cpm) * 1000) diff --git a/src/ad_buyer/tools/dsp/discover_inventory.py b/src/ad_buyer/tools/dsp/discover_inventory.py index 63042d8..b133536 100644 --- a/src/ad_buyer/tools/dsp/discover_inventory.py +++ b/src/ad_buyer/tools/dsp/discover_inventory.py @@ -186,7 +186,7 @@ def _format_results( product_id = product.get("id", "Unknown") name = product.get("name", "Unknown Product") publisher = product.get("publisherId", product.get("publisher", "Unknown")) - base_price = product.get("basePrice", product.get("price", 0)) + base_price = product.get("basePrice", product.get("price")) channel = product.get("channel", product.get("deliveryType", "N/A")) impressions = product.get( "availableImpressions", product.get("available_impressions", "N/A") @@ -194,7 +194,8 @@ def _format_results( targeting = product.get("targeting", product.get("availableTargeting", [])) # Calculate tiered price using centralized PricingCalculator - if isinstance(base_price, (int, float)) and discount > 0: + # When no pricing is available, show as unavailable + if isinstance(base_price, (int, float)) and base_price > 0 and discount > 0: tier_obj = self._buyer_context.identity.get_access_tier() calculator = PricingCalculator() pricing_result = calculator.calculate( @@ -203,12 +204,11 @@ def _format_results( tier_discount=discount, ) price_display = f"${pricing_result.tiered_price:.2f} (was ${base_price:.2f})" + elif isinstance(base_price, (int, float)) and base_price > 0: + price_display = f"${base_price:.2f}" else: - price_display = ( - f"${base_price:.2f}" - if isinstance(base_price, (int, float)) - else str(base_price) - ) + # No valid pricing from seller — do not fabricate + price_display = "Pricing unavailable (rate on request)" output_lines.extend( [ diff --git a/src/ad_buyer/tools/dsp/request_deal.py b/src/ad_buyer/tools/dsp/request_deal.py index 6e79eed..be3dda4 100644 --- a/src/ad_buyer/tools/dsp/request_deal.py +++ b/src/ad_buyer/tools/dsp/request_deal.py @@ -170,6 +170,14 @@ async def _arun( target_cpm=target_cpm, ) + if deal_response is None: + product_name = product.get("name", product_id) + return ( + f"Error: No pricing available for product '{product_name}' " + f"(ID: {product_id}). Seller has not provided a CPM. " + "Negotiation required before deal creation." + ) + return self._format_deal_response(deal_response) except (OSError, ValueError, RuntimeError) as e: @@ -183,18 +191,20 @@ def _create_deal_response( flight_start: str | None, flight_end: str | None, target_cpm: float | None, - ) -> DealResponse: + ) -> DealResponse | None: """Create a deal response with calculated pricing. Uses the centralized PricingCalculator and deal ID generator from ad_buyer.booking to avoid duplicated logic. + + Returns None when the product has no valid pricing available. """ tier = self._buyer_context.identity.get_access_tier() discount = self._buyer_context.identity.get_discount_percentage() - base_price = product.get("basePrice", product.get("price", 20.0)) + base_price = product.get("basePrice", product.get("price")) if not isinstance(base_price, (int, float)): - base_price = 20.0 + return None calculator = PricingCalculator() pricing = calculator.calculate( diff --git a/tests/unit/test_booking/test_pricing.py b/tests/unit/test_booking/test_pricing.py index 7355cb9..f51fb09 100644 --- a/tests/unit/test_booking/test_pricing.py +++ b/tests/unit/test_booking/test_pricing.py @@ -211,8 +211,8 @@ def test_negotiation_not_available_when_product_not_negotiable(self): class TestPricingCalculatorEdgeCases: """Test edge cases and data integrity.""" - def test_non_numeric_base_price_defaults_to_fallback(self): - """Non-numeric base price falls back to 0.""" + def test_zero_base_price_produces_zero_output(self): + """Zero base price correctly produces zero pricing output.""" calc = PricingCalculator() result = calc.calculate( base_price=0, diff --git a/tests/unit/test_cpm_fallback_removal.py b/tests/unit/test_cpm_fallback_removal.py new file mode 100644 index 0000000..f6de7a8 --- /dev/null +++ b/tests/unit/test_cpm_fallback_removal.py @@ -0,0 +1,345 @@ +# Author: Green Mountain Systems AI Inc. +# Donated to IAB Tech Lab + +"""Tests for CPM hallucination fix — Layer 2a: remove hardcoded price fallbacks. + +Bead: ar-na3i (child of epic ar-rrgw) + +These tests verify that the buyer agent no longer fabricates pricing +when sellers have not provided it. Each test targets a specific +fallback that was previously hardcoded in the codebase. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ad_buyer.booking.quote_flow import QuoteFlowClient +from ad_buyer.models.buyer_identity import ( + AccessTier, + BuyerContext, + BuyerIdentity, +) +from ad_buyer.pipelines.campaign_pipeline import CampaignPipeline +from ad_buyer.tools.dsp import DiscoverInventoryTool, RequestDealTool + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def agency_identity(): + """Agency-tier identity for testing.""" + return BuyerIdentity( + seat_id="ttd-seat-100", + agency_id="omnicom-200", + agency_name="OMD", + ) + + +@pytest.fixture +def agency_context(agency_identity): + """Agency buyer context.""" + return BuyerContext(identity=agency_identity, is_authenticated=True) + + +@pytest.fixture +def mock_client(): + """Mock UnifiedClient.""" + client = MagicMock() + client.get_product = AsyncMock() + client.search_products = AsyncMock() + client.list_products = AsyncMock() + return client + + +# --------------------------------------------------------------------------- +# 1. request_deal.py — no $20 fallback +# --------------------------------------------------------------------------- + + +class TestRequestDealNoFallback: + """request_deal must return an error string when no pricing is available, + not silently use $20.00 as a fallback CPM.""" + + @pytest.mark.asyncio + async def test_no_base_price_returns_error(self, mock_client, agency_context): + """Product with no basePrice or price should return an error string.""" + product_no_price = { + "id": "prod-001", + "name": "Premium CTV", + "channel": "ctv", + # No basePrice, no price + } + mock_client.get_product.return_value = MagicMock( + success=True, data=product_no_price + ) + + tool = RequestDealTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun(product_id="prod-001") + + # Must return an error string, not a deal with $20 CPM + assert isinstance(result, str) + assert "error" in result.lower() or "pricing" in result.lower() + assert "$20" not in result + assert "DEAL CREATED" not in result + + @pytest.mark.asyncio + async def test_non_numeric_base_price_returns_error(self, mock_client, agency_context): + """Product with non-numeric basePrice should return an error string, + not silently fall back to $20.""" + product_bad_price = { + "id": "prod-002", + "name": "Premium Display", + "basePrice": "contact_sales", + } + mock_client.get_product.return_value = MagicMock( + success=True, data=product_bad_price + ) + + tool = RequestDealTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun(product_id="prod-002") + + assert isinstance(result, str) + assert "error" in result.lower() or "pricing" in result.lower() + assert "$20" not in result + assert "DEAL CREATED" not in result + + @pytest.mark.asyncio + async def test_null_base_price_returns_error(self, mock_client, agency_context): + """Product with basePrice=None should return an error string.""" + product_null_price = { + "id": "prod-003", + "name": "Premium Audio", + "basePrice": None, + } + mock_client.get_product.return_value = MagicMock( + success=True, data=product_null_price + ) + + tool = RequestDealTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun(product_id="prod-003") + + assert isinstance(result, str) + assert "error" in result.lower() or "pricing" in result.lower() + assert "DEAL CREATED" not in result + + @pytest.mark.asyncio + async def test_valid_price_still_works(self, mock_client, agency_context): + """Product with a valid basePrice should still create a deal normally.""" + product_with_price = { + "id": "prod-004", + "name": "Premium Display", + "basePrice": 25.0, + } + mock_client.get_product.return_value = MagicMock( + success=True, data=product_with_price + ) + + tool = RequestDealTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun(product_id="prod-004") + + assert "DEAL CREATED" in result + + +# --------------------------------------------------------------------------- +# 2. discover_inventory.py — no 0 fallback +# --------------------------------------------------------------------------- + + +class TestDiscoverInventoryNoFallback: + """discover_inventory must show None/unavailable price when no + basePrice exists, not silently default to 0.""" + + @pytest.mark.asyncio + async def test_no_base_price_shows_unavailable(self, mock_client, agency_context): + """Product with no basePrice should show pricing as unavailable.""" + product_no_price = { + "id": "prod-001", + "name": "Premium CTV", + "channel": "ctv", + "availableImpressions": 5_000_000, + # No basePrice, no price + } + mock_client.list_products.return_value = MagicMock( + success=True, data=[product_no_price] + ) + + tool = DiscoverInventoryTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun() + + # Must NOT show $0.00 as the price + assert "$0.00" not in result + # Should indicate pricing is unavailable + assert "unavailable" in result.lower() or "request" in result.lower() or "N/A" in result + + @pytest.mark.asyncio + async def test_null_base_price_shows_unavailable(self, mock_client, agency_context): + """Product with basePrice=None should show pricing as unavailable.""" + product_null_price = { + "id": "prod-002", + "name": "Premium Display", + "basePrice": None, + "channel": "display", + "availableImpressions": 3_000_000, + } + mock_client.list_products.return_value = MagicMock( + success=True, data=[product_null_price] + ) + + tool = DiscoverInventoryTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun() + + assert "$0.00" not in result + + @pytest.mark.asyncio + async def test_valid_price_still_displays(self, mock_client, agency_context): + """Product with valid basePrice should still show the price normally.""" + product_with_price = { + "id": "prod-003", + "name": "Premium Display", + "basePrice": 20.0, + "channel": "display", + "availableImpressions": 5_000_000, + } + mock_client.list_products.return_value = MagicMock( + success=True, data=[product_with_price] + ) + + tool = DiscoverInventoryTool(client=mock_client, buyer_context=agency_context) + result = await tool._arun() + + # Should show the actual price + assert "$" in result + + +# --------------------------------------------------------------------------- +# 3. quote_flow.py — no 0 fallback +# --------------------------------------------------------------------------- + + +class TestQuoteFlowNoFallback: + """quote_flow.get_pricing must return an error/unavailable indicator + when no basePrice exists, not silently default to 0.""" + + def test_no_base_price_returns_unavailable(self, agency_context): + """Product with no basePrice should return pricing_source=unavailable.""" + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + + product_no_price = { + "id": "prod-001", + "name": "Premium CTV", + } + + result = client.get_pricing(product_no_price) + + # Must NOT return a PricingResult with base_price=0 + # Should indicate pricing is unavailable + assert result is None or (hasattr(result, "pricing_source") and result.pricing_source == "unavailable") + + def test_null_base_price_returns_unavailable(self, agency_context): + """Product with basePrice=None should return unavailable.""" + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + + product_null_price = { + "id": "prod-002", + "name": "Premium Display", + "basePrice": None, + } + + result = client.get_pricing(product_null_price) + + assert result is None or (hasattr(result, "pricing_source") and result.pricing_source == "unavailable") + + def test_non_numeric_base_price_returns_unavailable(self, agency_context): + """Product with non-numeric basePrice should return unavailable.""" + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + + product_bad_price = { + "id": "prod-003", + "name": "Premium Audio", + "basePrice": "contact_sales", + } + + result = client.get_pricing(product_bad_price) + + assert result is None or (hasattr(result, "pricing_source") and result.pricing_source == "unavailable") + + def test_valid_price_returns_pricing_result(self, agency_context): + """Product with valid basePrice should return normal PricingResult.""" + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + + product_with_price = { + "id": "prod-004", + "name": "Premium Display", + "basePrice": 20.0, + } + + result = client.get_pricing(product_with_price) + + # Should return a valid PricingResult with the actual price + assert result is not None + assert result.base_price == 20.0 + + def test_build_deal_data_no_price_returns_error(self, agency_context): + """build_deal_data with no pricing should return error indicator.""" + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + + product_no_price = { + "id": "prod-001", + "name": "Premium CTV", + } + + result = client.build_deal_data(product_no_price) + + # Should indicate pricing is unavailable, not create a deal with $0 + assert result is None or result.get("pricing_source") == "unavailable" + + +# --------------------------------------------------------------------------- +# 4. campaign_pipeline.py — no assumed_cpm=15.0 +# --------------------------------------------------------------------------- + + +class TestCampaignPipelineNoAssumedCPM: + """campaign_pipeline must not fabricate impressions from assumed CPM. + When no CPM is available, channels should be flagged as 'pricing TBD'.""" + + def test_estimate_impressions_requires_cpm(self): + """_estimate_impressions without a CPM should not fabricate impressions.""" + # The method should no longer accept a default assumed_cpm + # Calling without a CPM should return 0 or raise + result = CampaignPipeline._estimate_impressions(budget=60_000.0) + + # Must NOT return impressions based on a fabricated $15 CPM + # Old behavior: (60000 / 15) * 1000 = 4,000,000 + assert result != 4_000_000 + # Should return 0 or None when no CPM provided + assert result == 0 or result is None + + def test_estimate_impressions_with_explicit_cpm(self): + """_estimate_impressions with an explicit CPM should still work.""" + result = CampaignPipeline._estimate_impressions( + budget=60_000.0, assumed_cpm=20.0 + ) + + # (60000 / 20) * 1000 = 3,000,000 + assert result == 3_000_000 diff --git a/tests/unit/test_dsp_discovery_pricing.py b/tests/unit/test_dsp_discovery_pricing.py index c123324..2ead4a7 100644 --- a/tests/unit/test_dsp_discovery_pricing.py +++ b/tests/unit/test_dsp_discovery_pricing.py @@ -791,14 +791,15 @@ async def test_deal_shows_impression_count(self, mock_client, agency_context): @pytest.mark.asyncio async def test_deal_handles_non_numeric_baseprice(self, mock_client, agency_context): - """Product with non-numeric basePrice should use default of $20.""" + """Product with non-numeric basePrice should return a pricing error.""" mock_client.get_product.return_value = MagicMock( success=True, data={"id": "bad", "name": "Bad Price", "basePrice": "TBD"} ) tool = RequestDealTool(client=mock_client, buyer_context=agency_context) result = await tool._arun(product_id="bad") - # Should not crash, should use $20 default - assert "DEAL-" in result + # Should not crash; should return error instead of fabricating $20 CPM + assert "Error" in result or "pricing" in result.lower() + assert "DEAL CREATED" not in result @pytest.mark.asyncio async def test_deal_exception_handling(self, mock_client, agency_context): From 784c09a94f298d832078fadc98da6165c2405f50 Mon Sep 17 00:00:00 2001 From: ATC964 <82515918+atc964@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:43:45 -0400 Subject: [PATCH 2/3] Add pricing discipline guardrails to all LLM agent prompts (Layer 3) Append CRITICAL pricing discipline instruction to all 12 agent backstories preventing CPM hallucination. Update channel_crews.py expected_output template to show cpm as "Y or null" with explicit instruction that cpm must be null when no seller pricing was provided. Add 37 parametrized tests verifying guardrails are present in every agent. bead: ar-8opr Co-Authored-By: Claude Opus 4.6 (1M context) --- .../agents/level1/portfolio_manager.py | 7 +- src/ad_buyer/agents/level2/branding_agent.py | 7 +- src/ad_buyer/agents/level2/ctv_agent.py | 7 +- .../agents/level2/deal_library_agent.py | 7 +- src/ad_buyer/agents/level2/dsp_agent.py | 7 +- src/ad_buyer/agents/level2/linear_tv_agent.py | 7 +- .../agents/level2/mobile_app_agent.py | 7 +- .../agents/level2/performance_agent.py | 7 +- .../agents/level3/audience_planner_agent.py | 7 +- src/ad_buyer/agents/level3/execution_agent.py | 7 +- src/ad_buyer/agents/level3/reporting_agent.py | 7 +- src/ad_buyer/agents/level3/research_agent.py | 7 +- src/ad_buyer/crews/channel_crews.py | 7 +- tests/unit/test_llm_pricing_guardrails.py | 150 ++++++++++++++++++ 14 files changed, 226 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_llm_pricing_guardrails.py diff --git a/src/ad_buyer/agents/level1/portfolio_manager.py b/src/ad_buyer/agents/level1/portfolio_manager.py index 368a80f..efd337f 100644 --- a/src/ad_buyer/agents/level1/portfolio_manager.py +++ b/src/ad_buyer/agents/level1/portfolio_manager.py @@ -47,7 +47,12 @@ def create_portfolio_manager( 2. Delegate to channel specialists (Branding, Mobile, CTV, Performance) 3. Consolidate recommendations and ensure coherent strategy 4. Monitor overall campaign performance -5. Make real-time optimization decisions""", +5. Make real-time optimization decisions + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.manager_llm_model, temperature=0.3, diff --git a/src/ad_buyer/agents/level2/branding_agent.py b/src/ad_buyer/agents/level2/branding_agent.py index 5f8ba2c..0926525 100644 --- a/src/ad_buyer/agents/level2/branding_agent.py +++ b/src/ad_buyer/agents/level2/branding_agent.py @@ -50,7 +50,12 @@ def create_branding_agent( You work closely with the Research Agent to discover inventory and the Execution Agent to book placements. You report to the Portfolio Manager -and coordinate with other channel specialists for cohesive campaigns.""", +and coordinate with other channel specialists for cohesive campaigns. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.5, diff --git a/src/ad_buyer/agents/level2/ctv_agent.py b/src/ad_buyer/agents/level2/ctv_agent.py index 65597a1..66f2961 100644 --- a/src/ad_buyer/agents/level2/ctv_agent.py +++ b/src/ad_buyer/agents/level2/ctv_agent.py @@ -55,7 +55,12 @@ def create_ctv_agent( You work with the Research Agent to discover premium CTV inventory and the Execution Agent to book placements. You coordinate with Branding -for cohesive video strategies across screens.""", +for cohesive video strategies across screens. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.5, diff --git a/src/ad_buyer/agents/level2/deal_library_agent.py b/src/ad_buyer/agents/level2/deal_library_agent.py index e437b60..00f01ab 100644 --- a/src/ad_buyer/agents/level2/deal_library_agent.py +++ b/src/ad_buyer/agents/level2/deal_library_agent.py @@ -86,7 +86,12 @@ def create_deal_library_agent( When a deal needs to be booked for a campaign, you hand off to the appropriate campaign flow. When you detect underperforming deals or better supply paths, you -propose changes that the campaign flow can execute.""", +propose changes that the campaign flow can execute. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.3, diff --git a/src/ad_buyer/agents/level2/dsp_agent.py b/src/ad_buyer/agents/level2/dsp_agent.py index 53f2f9f..d3286a5 100644 --- a/src/ad_buyer/agents/level2/dsp_agent.py +++ b/src/ad_buyer/agents/level2/dsp_agent.py @@ -77,7 +77,12 @@ def create_dsp_agent( You work closely with channel specialists (CTV, Branding, Performance, Mobile) to understand inventory requirements and with the Execution Agent for -any direct booking needs outside of DSP activation.""", +any direct booking needs outside of DSP activation. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.5, diff --git a/src/ad_buyer/agents/level2/linear_tv_agent.py b/src/ad_buyer/agents/level2/linear_tv_agent.py index ad5405e..cc09fe3 100644 --- a/src/ad_buyer/agents/level2/linear_tv_agent.py +++ b/src/ad_buyer/agents/level2/linear_tv_agent.py @@ -70,7 +70,12 @@ def create_linear_tv_agent( You work with the Research Agent to discover available linear TV inventory and the Execution Agent to book placements. You coordinate with the CTV Specialist for cross-screen TV strategies and with Branding for cohesive -video campaigns across linear and streaming.""", +video campaigns across linear and streaming. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.5, diff --git a/src/ad_buyer/agents/level2/mobile_app_agent.py b/src/ad_buyer/agents/level2/mobile_app_agent.py index 8dcca76..d617ec4 100644 --- a/src/ad_buyer/agents/level2/mobile_app_agent.py +++ b/src/ad_buyer/agents/level2/mobile_app_agent.py @@ -53,7 +53,12 @@ def create_mobile_app_agent( You work closely with the Research Agent to find quality mobile inventory and the Execution Agent to book campaigns. You coordinate with the -Performance Agent for retargeting strategies.""", +Performance Agent for retargeting strategies. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.5, diff --git a/src/ad_buyer/agents/level2/performance_agent.py b/src/ad_buyer/agents/level2/performance_agent.py index 9a1d228..6d2f15a 100644 --- a/src/ad_buyer/agents/level2/performance_agent.py +++ b/src/ad_buyer/agents/level2/performance_agent.py @@ -57,7 +57,12 @@ def create_performance_agent( and the Execution Agent to book campaigns. You collaborate with the Reporting Agent to analyze performance and identify optimization opportunities. You coordinate with other specialists for full-funnel -strategies.""", +strategies. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.5, diff --git a/src/ad_buyer/agents/level3/audience_planner_agent.py b/src/ad_buyer/agents/level3/audience_planner_agent.py index 8605c28..1a2b48e 100644 --- a/src/ad_buyer/agents/level3/audience_planner_agent.py +++ b/src/ad_buyer/agents/level3/audience_planner_agent.py @@ -87,7 +87,12 @@ def create_audience_planner_agent( - Portfolio Manager on campaign audience strategy - Channel Specialists on channel-specific audience availability - Research Agent on inventory discovery - - Sellers' Audience Validator agents via UCP exchange""", + - Sellers' Audience Validator agents via UCP exchange + + CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices + explicitly provided by sellers through quotes or media kits. If no pricing is + available from the seller, state clearly that pricing requires negotiation. Do + not fill in CPMs from market knowledge or training data.""", verbose=verbose, allow_delegation=False, # Makes final audience decisions memory=True, diff --git a/src/ad_buyer/agents/level3/execution_agent.py b/src/ad_buyer/agents/level3/execution_agent.py index 90b9b8b..49c424d 100644 --- a/src/ad_buyer/agents/level3/execution_agent.py +++ b/src/ad_buyer/agents/level3/execution_agent.py @@ -63,7 +63,12 @@ def create_execution_agent( You work for the channel specialists and execute bookings only after receiving approved recommendations. Always verify parameters before -executing any booking action.""", +executing any booking action. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.1, diff --git a/src/ad_buyer/agents/level3/reporting_agent.py b/src/ad_buyer/agents/level3/reporting_agent.py index a869ec4..7640054 100644 --- a/src/ad_buyer/agents/level3/reporting_agent.py +++ b/src/ad_buyer/agents/level3/reporting_agent.py @@ -63,7 +63,12 @@ def create_reporting_agent( - Are there any lines that need attention? You work for the channel specialists and Portfolio Manager to provide -insights that inform optimization decisions.""", +insights that inform optimization decisions. + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.2, diff --git a/src/ad_buyer/agents/level3/research_agent.py b/src/ad_buyer/agents/level3/research_agent.py index 0cc15d4..da5620b 100644 --- a/src/ad_buyer/agents/level3/research_agent.py +++ b/src/ad_buyer/agents/level3/research_agent.py @@ -57,7 +57,12 @@ def create_research_agent( - Targeting requirements - Flight dates - Quality metrics -- Publisher reputation""", +- Publisher reputation + +CRITICAL: NEVER estimate, assume, or fabricate CPM pricing. Only use prices +explicitly provided by sellers through quotes or media kits. If no pricing is +available from the seller, state clearly that pricing requires negotiation. Do +not fill in CPMs from market knowledge or training data.""", llm=LLM( model=settings.default_llm_model, temperature=0.2, diff --git a/src/ad_buyer/crews/channel_crews.py b/src/ad_buyer/crews/channel_crews.py index 0fdcd21..c8badc1 100644 --- a/src/ad_buyer/crews/channel_crews.py +++ b/src/ad_buyer/crews/channel_crews.py @@ -135,11 +135,12 @@ def create_branding_crew( "publisher": "...", "format": "...", "impressions": X, - "cpm": Y, - "cost": Z, + "cpm": Y or null, + "cost": Z or null, "rationale": "..." } -]""", +] +Note: cpm must be null if no seller pricing was provided. NEVER estimate CPM.""", agent=research_agent, ) diff --git a/tests/unit/test_llm_pricing_guardrails.py b/tests/unit/test_llm_pricing_guardrails.py new file mode 100644 index 0000000..ad97945 --- /dev/null +++ b/tests/unit/test_llm_pricing_guardrails.py @@ -0,0 +1,150 @@ +# Author: Green Mountain Systems AI Inc. +# Donated to IAB Tech Lab + +"""Tests for CPM hallucination fix -- Layer 3: LLM prompt guardrails. + +Bead: ar-8opr (child of epic ar-rrgw) + +These tests verify that all agent backstories contain pricing discipline +language preventing LLM hallucination of CPM values, and that the +channel_crews expected_output template allows null CPM. +""" + +import os + +# Set a dummy API key for tests (agents validate on creation) +os.environ.setdefault("ANTHROPIC_API_KEY", "test-key-for-unit-tests") + +import pytest + +from ad_buyer.agents.level1.portfolio_manager import create_portfolio_manager +from ad_buyer.agents.level2.branding_agent import create_branding_agent +from ad_buyer.agents.level2.ctv_agent import create_ctv_agent +from ad_buyer.agents.level2.deal_library_agent import create_deal_library_agent +from ad_buyer.agents.level2.dsp_agent import create_dsp_agent +from ad_buyer.agents.level2.linear_tv_agent import create_linear_tv_agent +from ad_buyer.agents.level2.mobile_app_agent import create_mobile_app_agent +from ad_buyer.agents.level2.performance_agent import create_performance_agent +from ad_buyer.agents.level3.audience_planner_agent import create_audience_planner_agent +from ad_buyer.agents.level3.execution_agent import create_execution_agent +from ad_buyer.agents.level3.reporting_agent import create_reporting_agent +from ad_buyer.agents.level3.research_agent import create_research_agent + + +# --------------------------------------------------------------------------- +# All agent factory functions, for parametrized testing +# --------------------------------------------------------------------------- + +ALL_AGENT_FACTORIES = [ + ("portfolio_manager", create_portfolio_manager), + ("branding_agent", create_branding_agent), + ("ctv_agent", create_ctv_agent), + ("deal_library_agent", create_deal_library_agent), + ("dsp_agent", create_dsp_agent), + ("linear_tv_agent", create_linear_tv_agent), + ("mobile_app_agent", create_mobile_app_agent), + ("performance_agent", create_performance_agent), + ("audience_planner_agent", create_audience_planner_agent), + ("execution_agent", create_execution_agent), + ("reporting_agent", create_reporting_agent), + ("research_agent", create_research_agent), +] + + +# --------------------------------------------------------------------------- +# 1. Agent backstory pricing discipline tests +# --------------------------------------------------------------------------- + + +class TestAgentPricingDiscipline: + """Verify all agent backstories contain pricing discipline guardrails. + + Every agent in the buyer system must include explicit instructions + to never fabricate CPM pricing. This prevents the LLM from filling + in pricing values from its training data when sellers have not + provided them. + """ + + @pytest.mark.parametrize( + "agent_name,factory", + ALL_AGENT_FACTORIES, + ids=[name for name, _ in ALL_AGENT_FACTORIES], + ) + def test_backstory_contains_never_fabricate(self, agent_name, factory): + """Each agent backstory must contain 'NEVER' and 'fabricate' keywords.""" + agent = factory(verbose=False) + backstory = agent.backstory.lower() + + assert "never" in backstory, ( + f"{agent_name} backstory missing 'NEVER' pricing discipline keyword" + ) + assert "fabricate" in backstory, ( + f"{agent_name} backstory missing 'fabricate' pricing discipline keyword" + ) + + @pytest.mark.parametrize( + "agent_name,factory", + ALL_AGENT_FACTORIES, + ids=[name for name, _ in ALL_AGENT_FACTORIES], + ) + def test_backstory_contains_pricing_negotiation_fallback(self, agent_name, factory): + """Each agent backstory must instruct to state pricing requires negotiation.""" + agent = factory(verbose=False) + backstory = agent.backstory.lower() + + assert "negotiation" in backstory, ( + f"{agent_name} backstory missing negotiation fallback instruction" + ) + + @pytest.mark.parametrize( + "agent_name,factory", + ALL_AGENT_FACTORIES, + ids=[name for name, _ in ALL_AGENT_FACTORIES], + ) + def test_backstory_prohibits_market_knowledge_pricing(self, agent_name, factory): + """Each agent backstory must prohibit using market knowledge for pricing.""" + agent = factory(verbose=False) + backstory = agent.backstory.lower() + + assert "training data" in backstory, ( + f"{agent_name} backstory missing prohibition on using training data for pricing" + ) + + +# --------------------------------------------------------------------------- +# 2. channel_crews expected_output template tests +# --------------------------------------------------------------------------- + + +class TestChannelCrewsExpectedOutput: + """Verify the channel_crews expected_output template allows null CPM.""" + + def test_branding_research_task_allows_null_cpm(self): + """The branding research task expected_output must allow null cpm.""" + from unittest.mock import MagicMock + + from ad_buyer.crews.channel_crews import create_branding_crew + + client = MagicMock() + brief = { + "budget": 10000, + "start_date": "2025-03-01", + "end_date": "2025-03-31", + "target_audience": {"age": "25-54"}, + "objectives": ["awareness"], + } + + crew = create_branding_crew(client, brief) + + # Find the research task (first task) + research_task = crew.tasks[0] + expected = research_task.expected_output.lower() + + # Must allow null cpm + assert "null" in expected, ( + "Branding research task expected_output must show cpm can be null" + ) + # Must contain the NEVER estimate instruction + assert "never" in expected, ( + "Branding research task expected_output must say NEVER estimate CPM" + ) From 6344d98c58a306019abb1dc7aecf6ac4ad57a3a7 Mon Sep 17 00:00:00 2001 From: ATC964 <82515918+atc964@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:46:32 -0400 Subject: [PATCH 3/3] Add pricing provenance tracking to prevent CPM fabrication (Layer 2b) Every pricing value now carries a source (seller_quoted, negotiated, or unavailable). The system refuses to produce a CPM when the seller has not provided pricing, and unpriced quotes flow through normalization and orchestration without crashing. bead: ar-r76d Co-Authored-By: Claude Opus 4.6 (1M context) --- src/ad_buyer/booking/pricing.py | 57 ++- src/ad_buyer/booking/quote_normalizer.py | 66 ++- src/ad_buyer/clients/deals_client.py | 8 +- src/ad_buyer/clients/unified_client.py | 17 +- src/ad_buyer/models/deals.py | 8 +- src/ad_buyer/orchestration/multi_seller.py | 14 +- src/ad_buyer/tools/dsp/get_pricing.py | 14 +- tests/unit/test_pricing_provenance.py | 500 +++++++++++++++++++++ 8 files changed, 647 insertions(+), 37 deletions(-) create mode 100644 tests/unit/test_pricing_provenance.py diff --git a/src/ad_buyer/booking/pricing.py b/src/ad_buyer/booking/pricing.py index be9fe7d..dca37e1 100644 --- a/src/ad_buyer/booking/pricing.py +++ b/src/ad_buyer/booking/pricing.py @@ -20,33 +20,57 @@ """ from dataclasses import dataclass +from enum import Enum from ..models.buyer_identity import AccessTier +class PricingSource(Enum): + """Provenance of a pricing value. + + Every PricingResult carries a pricing_source indicating where the + price came from. This prevents the system from silently using + fabricated CPMs. + + Values: + SELLER_QUOTED: Price was provided by the seller (base price exists). + NEGOTIATED: Price was agreed via negotiation (target_cpm accepted). + UNAVAILABLE: No pricing is available (seller has not provided a price). + """ + + SELLER_QUOTED = "seller_quoted" + NEGOTIATED = "negotiated" + UNAVAILABLE = "unavailable" + + @dataclass class PricingResult: """Result of a pricing calculation. Attributes: base_price: Original base price before any discounts. + None when pricing_source is UNAVAILABLE. tier: The buyer's access tier. tier_discount: Tier-based discount percentage applied. volume_discount: Volume-based discount percentage applied. tiered_price: Price after tier discount (before volume discount). + None when pricing_source is UNAVAILABLE. final_price: Price after all discounts (tier + volume + negotiation). + None when pricing_source is UNAVAILABLE. requested_volume: Impression volume used for volume discount calculation. deal_type: Deal type requested (if any). + pricing_source: Provenance of the pricing value. """ - base_price: float + base_price: float | None tier: AccessTier tier_discount: float volume_discount: float - tiered_price: float - final_price: float + tiered_price: float | None + final_price: float | None requested_volume: int | None = None deal_type: str | None = None + pricing_source: PricingSource = PricingSource.SELLER_QUOTED class PricingCalculator: @@ -81,7 +105,7 @@ class PricingCalculator: def calculate( self, - base_price: float, + base_price: float | None, tier: AccessTier, tier_discount: float, volume: int | None = None, @@ -93,7 +117,9 @@ def calculate( """Calculate the final price after tier and volume discounts. Args: - base_price: Base CPM price from the product. + base_price: Base CPM price from the product. When None, + the calculator refuses to compute and returns a result + with pricing_source=UNAVAILABLE. tier: Buyer's access tier (public/seat/agency/advertiser). tier_discount: Discount percentage for the tier (0-15). volume: Requested impression volume (may unlock volume discounts). @@ -103,8 +129,24 @@ def calculate( deal_type: Deal type requested (for informational purposes). Returns: - PricingResult with all pricing details. + PricingResult with all pricing details. When base_price is + None, all price fields are None and pricing_source is + UNAVAILABLE. """ + # Guard: refuse to compute when base_price is None + if base_price is None: + return PricingResult( + base_price=None, + tier=tier, + tier_discount=tier_discount, + volume_discount=0.0, + tiered_price=None, + final_price=None, + requested_volume=volume, + deal_type=deal_type, + pricing_source=PricingSource.UNAVAILABLE, + ) + # Step 1: Apply tier discount tiered_price = base_price * (1 - tier_discount / 100) @@ -118,6 +160,7 @@ def calculate( final_price = tiered_price # Step 4: Handle negotiation + pricing_source = PricingSource.SELLER_QUOTED if target_cpm is not None and can_negotiate and negotiation_enabled: floor_price = tiered_price * 0.90 if target_cpm >= floor_price: @@ -125,6 +168,7 @@ def calculate( else: # Counter at floor final_price = floor_price + pricing_source = PricingSource.NEGOTIATED return PricingResult( base_price=base_price, @@ -135,6 +179,7 @@ def calculate( final_price=final_price, requested_volume=volume, deal_type=deal_type, + pricing_source=pricing_source, ) def _get_volume_discount( diff --git a/src/ad_buyer/booking/quote_normalizer.py b/src/ad_buyer/booking/quote_normalizer.py index 27168d3..95286f6 100644 --- a/src/ad_buyer/booking/quote_normalizer.py +++ b/src/ad_buyer/booking/quote_normalizer.py @@ -58,9 +58,10 @@ class NormalizedQuote: quote_id: The original quote identifier. raw_cpm: The CPM as quoted by the seller (final_cpm from the quote response, after any tier/volume discounts the seller - already applied). + already applied). None when pricing is unavailable. effective_cpm: The true cost-per-mille after deal-type - adjustment and estimated fees. + adjustment and estimated fees. None when pricing is + unavailable. deal_type: Deal type string (PG, PD, PA). fee_estimate: Estimated intermediary + tech fees added to raw_cpm, in currency units per mille. @@ -68,17 +69,20 @@ class NormalizedQuote: score: Composite ranking score (0-100, higher is better). fill_rate_estimate: Optional fill-rate from seller availability data, if provided. + pricing_source: Provenance of the pricing value. One of + "seller_quoted", "negotiated", or "unavailable". """ seller_id: str quote_id: str - raw_cpm: float - effective_cpm: float + raw_cpm: float | None + effective_cpm: float | None deal_type: str fee_estimate: float minimum_spend: float score: float fill_rate_estimate: float | None = None + pricing_source: str = "seller_quoted" # --------------------------------------------------------------------------- @@ -164,11 +168,34 @@ def normalize_quote( Returns: NormalizedQuote with effective CPM and a preliminary score. + When the quote has no pricing (final_cpm is None), returns + an unpriced NormalizedQuote with pricing_source="unavailable". """ raw_cpm = quote.pricing.final_cpm seller_id = quote.seller_id or "unknown" quote_id = quote.quote_id + # Short-circuit: when final_cpm is None the seller has not + # provided pricing. Return an unpriced NormalizedQuote instead + # of crashing on arithmetic with None. + if raw_cpm is None: + fill_rate: float | None = None + if quote.availability and quote.availability.estimated_fill_rate is not None: + fill_rate = quote.availability.estimated_fill_rate + + return NormalizedQuote( + seller_id=seller_id, + quote_id=quote_id, + raw_cpm=None, + effective_cpm=None, + deal_type=deal_type, + fee_estimate=0.0, + minimum_spend=minimum_spend, + score=0.0, + fill_rate_estimate=fill_rate, + pricing_source="unavailable", + ) + # Step 1: Deal-type adjustment adjusted_cpm = self._apply_deal_type_adjustment(raw_cpm, deal_type) @@ -179,7 +206,7 @@ def normalize_quote( effective_cpm = adjusted_cpm + fee_estimate # Step 4: Extract fill-rate if available - fill_rate: float | None = None + fill_rate = None if quote.availability and quote.availability.estimated_fill_rate is not None: fill_rate = quote.availability.estimated_fill_rate @@ -197,6 +224,7 @@ def normalize_quote( minimum_spend=minimum_spend, score=score, fill_rate_estimate=fill_rate, + pricing_source="seller_quoted", ) def compare_quotes( @@ -206,6 +234,10 @@ def compare_quotes( ) -> list[NormalizedQuote]: """Normalize and rank multiple quotes. + Unpriced quotes (pricing_source="unavailable") are separated from + the ranked set and appended at the end so they do not interfere + with the relative scoring of priced quotes. + Args: quotes: List of (QuoteResponse, deal_type) tuples. minimum_spends: Optional mapping of quote_id to minimum @@ -213,7 +245,7 @@ def compare_quotes( Returns: List of NormalizedQuote sorted by score descending - (best quote first). + (best quote first), with unpriced quotes appended at the end. """ if not quotes: return [] @@ -221,20 +253,26 @@ def compare_quotes( minimum_spends = minimum_spends or {} # Normalize all quotes - normalized: list[NormalizedQuote] = [] + priced: list[NormalizedQuote] = [] + unpriced: list[NormalizedQuote] = [] for quote, deal_type in quotes: min_spend = minimum_spends.get(quote.quote_id, 0.0) nq = self.normalize_quote(quote, deal_type, minimum_spend=min_spend) - normalized.append(nq) + if nq.pricing_source == "unavailable": + unpriced.append(nq) + else: + priced.append(nq) # Re-score relative to the set (best effective CPM gets highest - # CPM sub-score) - if len(normalized) > 1: - self._rescore_relative(normalized) + # CPM sub-score) — only for priced quotes + if len(priced) > 1: + self._rescore_relative(priced) + + # Sort priced by score descending (best first) + priced.sort(key=lambda nq: nq.score, reverse=True) - # Sort by score descending (best first) - normalized.sort(key=lambda nq: nq.score, reverse=True) - return normalized + # Append unpriced quotes at the end + return priced + unpriced # ------------------------------------------------------------------ # Internal helpers diff --git a/src/ad_buyer/clients/deals_client.py b/src/ad_buyer/clients/deals_client.py index 69c80b1..8928124 100644 --- a/src/ad_buyer/clients/deals_client.py +++ b/src/ad_buyer/clients/deals_client.py @@ -406,8 +406,8 @@ def _persist_quote(self, quote: QuoteResponse, request: QuoteRequest) -> None: product_name=quote.product.name, deal_type=request.deal_type, status="quoted", - price=quote.pricing.final_cpm, - original_price=quote.pricing.base_cpm, + price=quote.pricing.final_cpm if quote.pricing.final_cpm is not None else 0.0, + original_price=quote.pricing.base_cpm if quote.pricing.base_cpm is not None else 0.0, impressions=quote.terms.impressions, flight_start=quote.terms.flight_start, flight_end=quote.terms.flight_end, @@ -435,8 +435,8 @@ def _persist_deal(self, deal: DealResponse) -> None: product_name=deal.product.name, deal_type=deal.deal_type, status="booked", - price=deal.pricing.final_cpm, - original_price=deal.pricing.base_cpm, + price=deal.pricing.final_cpm if deal.pricing.final_cpm is not None else 0.0, + original_price=deal.pricing.base_cpm if deal.pricing.base_cpm is not None else 0.0, impressions=deal.terms.impressions, flight_start=deal.terms.flight_start, flight_end=deal.terms.flight_end, diff --git a/src/ad_buyer/clients/unified_client.py b/src/ad_buyer/clients/unified_client.py index ede2c3e..1d1e612 100644 --- a/src/ad_buyer/clients/unified_client.py +++ b/src/ad_buyer/clients/unified_client.py @@ -508,7 +508,7 @@ async def get_pricing( # Enhance result with tiered pricing calculation if result.data and isinstance(result.data, dict): - base_price = result.data.get("basePrice", result.data.get("price", 0)) + base_price = result.data.get("basePrice", result.data.get("price")) if isinstance(base_price, (int, float)) and self.buyer_identity: from ..models.buyer_identity import AccessTier @@ -526,12 +526,25 @@ async def get_pricing( result.data["pricing"] = { "base_price": pricing.base_price, - "tiered_price": round(pricing.final_price, 2), + "tiered_price": round(pricing.final_price, 2) if pricing.final_price is not None else None, "tier": tier_obj.value if self.buyer_identity else "public", "tier_discount": discount if self.buyer_identity else 0, "volume_discount": pricing.volume_discount, "requested_volume": volume, "deal_type": deal_type, + "pricing_source": pricing.pricing_source.value, + } + elif not isinstance(base_price, (int, float)): + # No valid pricing available — mark as unavailable + result.data["pricing"] = { + "base_price": None, + "tiered_price": None, + "tier": self.buyer_identity.get_access_tier().value if self.buyer_identity else "public", + "tier_discount": self.buyer_identity.get_discount_percentage() if self.buyer_identity else 0, + "volume_discount": 0.0, + "requested_volume": volume, + "deal_type": deal_type, + "pricing_source": "unavailable", } return result diff --git a/src/ad_buyer/models/deals.py b/src/ad_buyer/models/deals.py index 45df9c0..45bc088 100644 --- a/src/ad_buyer/models/deals.py +++ b/src/ad_buyer/models/deals.py @@ -49,12 +49,16 @@ class PricingInfo(BaseModel): """Pricing breakdown returned by the seller. Extended with CPP fields for linear TV (pricing_model "cpp" or "hybrid"). + + base_cpm and final_cpm are Optional to support ``pricing_type=on_request`` + (Layer 2b — pricing provenance tracking, bead ar-r76d). When the seller + has not provided pricing, these fields are None. """ - base_cpm: float + base_cpm: float | None = None tier_discount_pct: float = 0.0 volume_discount_pct: float = 0.0 - final_cpm: float + final_cpm: float | None = None currency: str = "USD" pricing_model: str = "cpm" # "cpm", "cpp", "unit_rate", "hybrid" rationale: str = "" diff --git a/src/ad_buyer/orchestration/multi_seller.py b/src/ad_buyer/orchestration/multi_seller.py index 03f9497..f9b8060 100644 --- a/src/ad_buyer/orchestration/multi_seller.py +++ b/src/ad_buyer/orchestration/multi_seller.py @@ -353,11 +353,12 @@ async def _request_one(seller: AgentCard) -> SellerQuoteResult: timeout=self._quote_timeout, ) + cpm_display = f"{quote.pricing.final_cpm:.2f}" if quote.pricing.final_cpm is not None else "unavailable" logger.info( - "Received quote %s from seller %s (CPM: %.2f)", + "Received quote %s from seller %s (CPM: %s)", quote.quote_id, seller.agent_id, - quote.pricing.final_cpm, + cpm_display, ) return SellerQuoteResult( @@ -460,11 +461,11 @@ async def evaluate_and_rank( # Normalize and rank ranked = self._normalizer.compare_quotes(quote_tuples) - # Apply max CPM filter + # Apply max CPM filter (skip unpriced quotes — they have effective_cpm=None) if max_cpm is not None: ranked = [ nq for nq in ranked - if nq.effective_cpm <= max_cpm + if nq.effective_cpm is not None and nq.effective_cpm <= max_cpm ] logger.info( @@ -557,11 +558,12 @@ async def select_and_book( }, ) + deal_cpm_display = f"{deal.pricing.final_cpm:.2f}" if deal.pricing.final_cpm is not None else "unavailable" logger.info( - "Booked deal %s from seller %s (CPM: %.2f)", + "Booked deal %s from seller %s (CPM: %s)", deal.deal_id, nq.seller_id, - deal.pricing.final_cpm, + deal_cpm_display, ) except Exception as exc: # noqa: BLE001 - per-deal isolation; continue booking remaining deals diff --git a/src/ad_buyer/tools/dsp/get_pricing.py b/src/ad_buyer/tools/dsp/get_pricing.py index e04e46a..bcf84bd 100644 --- a/src/ad_buyer/tools/dsp/get_pricing.py +++ b/src/ad_buyer/tools/dsp/get_pricing.py @@ -151,13 +151,21 @@ def _format_pricing( # Extract product info product_id = product.get("id", "Unknown") name = product.get("name", "Unknown Product") - base_price = product.get("basePrice", product.get("price", 0)) + base_price = product.get("basePrice", product.get("price")) publisher = product.get("publisherId", product.get("publisher", "Unknown")) rate_type = product.get("rateType", "CPM") - # Calculate pricing using centralized calculator + # Guard: no pricing available from the seller if not isinstance(base_price, (int, float)): - base_price = 0 + return ( + f"Pricing for: {name}\n" + f"Product ID: {product_id}\n" + f"Publisher: {publisher}\n" + f"{'=' * 50}\n\n" + "Pricing: UNAVAILABLE\n" + "No pricing has been provided by the seller for this product.\n" + "Contact the seller to negotiate pricing before proceeding." + ) calculator = PricingCalculator() pricing = calculator.calculate( diff --git a/tests/unit/test_pricing_provenance.py b/tests/unit/test_pricing_provenance.py new file mode 100644 index 0000000..525c9c7 --- /dev/null +++ b/tests/unit/test_pricing_provenance.py @@ -0,0 +1,500 @@ +# Author: Green Mountain Systems AI Inc. +# Donated to IAB Tech Lab + +"""Tests for CPM hallucination fix — Layer 2b: pricing provenance tracking. + +Bead: ar-r76d (child of epic ar-rrgw) + +These tests verify that every pricing value in the buyer agent carries +a provenance source (`pricing_source`), and that the system refuses to +produce a CPM when the seller has not provided one. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ad_buyer.booking.pricing import PricingCalculator, PricingResult, PricingSource +from ad_buyer.booking.quote_normalizer import NormalizedQuote, QuoteNormalizer +from ad_buyer.models.buyer_identity import AccessTier, BuyerContext, BuyerIdentity +from ad_buyer.models.deals import PricingInfo, ProductInfo, QuoteResponse, TermsInfo + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def agency_identity(): + """Agency-tier identity for testing.""" + return BuyerIdentity( + seat_id="ttd-seat-100", + agency_id="omnicom-200", + agency_name="OMD", + ) + + +@pytest.fixture +def agency_context(agency_identity): + """Agency buyer context.""" + return BuyerContext(identity=agency_identity, is_authenticated=True) + + +# --------------------------------------------------------------------------- +# 1. PricingSource enum exists on PricingResult +# --------------------------------------------------------------------------- + + +class TestPricingSourceEnum: + """PricingResult must carry a pricing_source field with provenance.""" + + def test_pricing_source_seller_quoted(self): + """PricingResult from a valid price has pricing_source=seller_quoted.""" + calc = PricingCalculator() + result = calc.calculate( + base_price=20.0, + tier=AccessTier.AGENCY, + tier_discount=10.0, + ) + assert result.pricing_source == PricingSource.SELLER_QUOTED + + def test_pricing_source_negotiated(self): + """PricingResult from successful negotiation has pricing_source=negotiated.""" + calc = PricingCalculator() + result = calc.calculate( + base_price=20.0, + tier=AccessTier.AGENCY, + tier_discount=10.0, + target_cpm=17.0, + can_negotiate=True, + negotiation_enabled=True, + ) + assert result.pricing_source == PricingSource.NEGOTIATED + + def test_pricing_source_unavailable_when_base_price_none(self): + """PricingCalculator with base_price=None returns pricing_source=unavailable.""" + calc = PricingCalculator() + result = calc.calculate( + base_price=None, + tier=AccessTier.AGENCY, + tier_discount=10.0, + ) + assert result.pricing_source == PricingSource.UNAVAILABLE + assert result.base_price is None + assert result.final_price is None + + def test_pricing_source_enum_values(self): + """PricingSource enum has exactly the expected values.""" + assert PricingSource.SELLER_QUOTED.value == "seller_quoted" + assert PricingSource.NEGOTIATED.value == "negotiated" + assert PricingSource.UNAVAILABLE.value == "unavailable" + + +# --------------------------------------------------------------------------- +# 2. PricingCalculator refuses to compute when base_price is None +# --------------------------------------------------------------------------- + + +class TestPricingCalculatorNullGuard: + """PricingCalculator must refuse to compute when base_price is None.""" + + def test_none_base_price_returns_unavailable_result(self): + """calculate() with base_price=None returns an unavailable PricingResult.""" + calc = PricingCalculator() + result = calc.calculate( + base_price=None, + tier=AccessTier.PUBLIC, + tier_discount=0.0, + ) + assert result.pricing_source == PricingSource.UNAVAILABLE + assert result.base_price is None + assert result.tiered_price is None + assert result.final_price is None + assert result.tier_discount == 0.0 + assert result.volume_discount == 0.0 + + def test_none_base_price_with_volume_returns_unavailable(self): + """calculate() with base_price=None and volume still returns unavailable.""" + calc = PricingCalculator() + result = calc.calculate( + base_price=None, + tier=AccessTier.AGENCY, + tier_discount=10.0, + volume=5_000_000, + ) + assert result.pricing_source == PricingSource.UNAVAILABLE + assert result.final_price is None + + def test_valid_base_price_still_works(self): + """calculate() with a valid base_price still produces correct pricing.""" + calc = PricingCalculator() + result = calc.calculate( + base_price=20.0, + tier=AccessTier.AGENCY, + tier_discount=10.0, + ) + assert result.pricing_source == PricingSource.SELLER_QUOTED + assert result.base_price == 20.0 + assert result.tiered_price == 18.0 + assert result.final_price == 18.0 + + +# --------------------------------------------------------------------------- +# 3. PricingInfo model — base_cpm and final_cpm are Optional[float] +# --------------------------------------------------------------------------- + + +class TestPricingInfoOptional: + """PricingInfo.base_cpm and final_cpm must be Optional[float].""" + + def test_pricing_info_accepts_none_base_cpm(self): + """PricingInfo can be created with base_cpm=None.""" + info = PricingInfo(base_cpm=None, final_cpm=None) + assert info.base_cpm is None + assert info.final_cpm is None + + def test_pricing_info_with_values_still_works(self): + """PricingInfo with float values still works.""" + info = PricingInfo(base_cpm=20.0, final_cpm=18.0) + assert info.base_cpm == 20.0 + assert info.final_cpm == 18.0 + + def test_pricing_info_mixed_none_and_value(self): + """PricingInfo with base_cpm=None but final_cpm set is valid.""" + info = PricingInfo(base_cpm=None, final_cpm=10.0) + assert info.base_cpm is None + assert info.final_cpm == 10.0 + + +# --------------------------------------------------------------------------- +# 4. QuoteNormalizer — short-circuit guard for None pricing +# --------------------------------------------------------------------------- + + +def _make_quote( + *, + quote_id: str = "q-001", + seller_id: str = "seller-a", + base_cpm: float | None = 10.0, + final_cpm: float | None = 10.0, + fill_rate: float | None = None, +) -> QuoteResponse: + """Helper to build a QuoteResponse for testing.""" + from ad_buyer.models.deals import AvailabilityInfo + + availability = None + if fill_rate is not None: + availability = AvailabilityInfo( + inventory_available=True, + estimated_fill_rate=fill_rate, + ) + + return QuoteResponse( + quote_id=quote_id, + status="available", + product=ProductInfo( + product_id=f"prod-{seller_id}", + name=f"Package from {seller_id}", + ), + pricing=PricingInfo( + base_cpm=base_cpm, + final_cpm=final_cpm, + ), + terms=TermsInfo( + impressions=500_000, + flight_start="2026-04-01", + flight_end="2026-04-30", + ), + availability=availability, + seller_id=seller_id, + buyer_tier="agency", + ) + + +class TestQuoteNormalizerNullPricing: + """QuoteNormalizer must short-circuit when quote.pricing.final_cpm is None. + + The arithmetic in normalize_quote() (lines 173-179) would crash on None + if not guarded. This tests the short-circuit guard. + """ + + def test_null_final_cpm_does_not_crash(self): + """normalize_quote with final_cpm=None should NOT raise TypeError.""" + normalizer = QuoteNormalizer() + quote = _make_quote(final_cpm=None, base_cpm=None) + # Must not raise — should return an NormalizedQuote with + # pricing_source indicating unavailable + result = normalizer.normalize_quote(quote, deal_type="PD") + assert result is not None + + def test_null_final_cpm_returns_unavailable_marker(self): + """normalize_quote with final_cpm=None returns a NormalizedQuote + flagged as unpriced.""" + normalizer = QuoteNormalizer() + quote = _make_quote(final_cpm=None, base_cpm=None) + result = normalizer.normalize_quote(quote, deal_type="PD") + # The NormalizedQuote should have pricing_source=unavailable + assert result.pricing_source == "unavailable" + + def test_null_final_cpm_raw_cpm_is_none(self): + """normalize_quote with final_cpm=None sets raw_cpm to None.""" + normalizer = QuoteNormalizer() + quote = _make_quote(final_cpm=None, base_cpm=None) + result = normalizer.normalize_quote(quote, deal_type="PD") + assert result.raw_cpm is None + + def test_priced_quote_still_normalizes(self): + """A normally priced quote still normalizes correctly.""" + normalizer = QuoteNormalizer() + quote = _make_quote(final_cpm=12.0, base_cpm=15.0) + result = normalizer.normalize_quote(quote, deal_type="PD") + assert result.raw_cpm == 12.0 + assert result.effective_cpm == 12.0 + assert result.pricing_source == "seller_quoted" + + def test_compare_quotes_filters_unpriced(self): + """compare_quotes with a mix of priced and unpriced quotes handles + both gracefully — unpriced quotes are excluded from ranking.""" + normalizer = QuoteNormalizer() + quotes = [ + (_make_quote(quote_id="q-priced", seller_id="seller-a", + final_cpm=10.0, base_cpm=12.0), "PD"), + (_make_quote(quote_id="q-unpriced", seller_id="seller-b", + final_cpm=None, base_cpm=None), "PD"), + ] + ranked = normalizer.compare_quotes(quotes) + # Unpriced quotes should be separated from the ranked list + priced = [q for q in ranked if q.pricing_source != "unavailable"] + unpriced = [q for q in ranked if q.pricing_source == "unavailable"] + assert len(priced) == 1 + assert priced[0].quote_id == "q-priced" + assert len(unpriced) == 1 + assert unpriced[0].quote_id == "q-unpriced" + + +# --------------------------------------------------------------------------- +# 5. multi_seller.py — handle unpriced quotes gracefully +# --------------------------------------------------------------------------- + + +class TestMultiSellerUnpricedQuotes: + """Multi-seller orchestration must handle a mix of priced and unpriced quotes.""" + + @pytest.mark.asyncio + async def test_evaluate_and_rank_filters_unpriced(self): + """evaluate_and_rank should handle unpriced quotes without crashing.""" + from ad_buyer.orchestration.multi_seller import ( + MultiSellerOrchestrator, + SellerQuoteResult, + ) + + mock_registry = AsyncMock() + mock_factory = MagicMock() + normalizer = QuoteNormalizer() + + orchestrator = MultiSellerOrchestrator( + registry_client=mock_registry, + deals_client_factory=mock_factory, + quote_normalizer=normalizer, + quote_timeout=5.0, + ) + + # One priced, one unpriced + quote_results = [ + SellerQuoteResult( + seller_id="seller-a", + seller_url="http://seller-a.example.com", + quote=_make_quote( + quote_id="q-priced", + seller_id="seller-a", + final_cpm=10.0, + base_cpm=12.0, + ), + deal_type="PD", + error=None, + ), + SellerQuoteResult( + seller_id="seller-b", + seller_url="http://seller-b.example.com", + quote=_make_quote( + quote_id="q-unpriced", + seller_id="seller-b", + final_cpm=None, + base_cpm=None, + ), + deal_type="PD", + error=None, + ), + ] + + # Must not crash + ranked = await orchestrator.evaluate_and_rank(quote_results) + # Should have at least the priced quote + priced = [q for q in ranked if q.pricing_source != "unavailable"] + assert len(priced) >= 1 + assert priced[0].quote_id == "q-priced" + + +# --------------------------------------------------------------------------- +# 6. get_pricing tool — guard null pricing +# --------------------------------------------------------------------------- + + +class TestGetPricingToolNullGuard: + """get_pricing tool must guard against null pricing from products.""" + + @pytest.mark.asyncio + async def test_no_base_price_shows_unavailable(self): + """Product with no basePrice should show pricing as unavailable.""" + from ad_buyer.tools.dsp.get_pricing import GetPricingTool + + mock_client = MagicMock() + mock_client.get_product = AsyncMock( + return_value=MagicMock( + success=True, + data={ + "id": "prod-001", + "name": "Premium CTV", + "channel": "ctv", + # No basePrice + }, + ) + ) + + agency_identity = BuyerIdentity( + seat_id="ttd-seat-100", + agency_id="omnicom-200", + agency_name="OMD", + ) + buyer_context = BuyerContext(identity=agency_identity, is_authenticated=True) + + tool = GetPricingTool(client=mock_client, buyer_context=buyer_context) + result = await tool._arun(product_id="prod-001") + + # Should NOT show $0.00 pricing + assert "$0.00" not in result + # Should indicate pricing is unavailable + assert "unavailable" in result.lower() or "no pricing" in result.lower() + + @pytest.mark.asyncio + async def test_none_base_price_shows_unavailable(self): + """Product with basePrice=None should show pricing as unavailable.""" + from ad_buyer.tools.dsp.get_pricing import GetPricingTool + + mock_client = MagicMock() + mock_client.get_product = AsyncMock( + return_value=MagicMock( + success=True, + data={ + "id": "prod-002", + "name": "Premium Display", + "basePrice": None, + }, + ) + ) + + agency_identity = BuyerIdentity( + seat_id="ttd-seat-100", + agency_id="omnicom-200", + agency_name="OMD", + ) + buyer_context = BuyerContext(identity=agency_identity, is_authenticated=True) + + tool = GetPricingTool(client=mock_client, buyer_context=buyer_context) + result = await tool._arun(product_id="prod-002") + + assert "$0.00" not in result + assert "unavailable" in result.lower() or "no pricing" in result.lower() + + +# --------------------------------------------------------------------------- +# 7. QuoteFlowClient — pricing_source propagation +# --------------------------------------------------------------------------- + + +class TestQuoteFlowPricingSource: + """QuoteFlowClient must propagate pricing_source from PricingCalculator.""" + + def test_get_pricing_with_price_has_seller_quoted(self, agency_context): + """get_pricing with valid price returns PricingResult with seller_quoted.""" + from ad_buyer.booking.quote_flow import QuoteFlowClient + + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + product = {"id": "prod-001", "name": "Test", "basePrice": 20.0} + result = client.get_pricing(product) + assert result is not None + assert result.pricing_source == PricingSource.SELLER_QUOTED + + def test_get_pricing_without_price_returns_unavailable(self, agency_context): + """get_pricing with no price returns None (unchanged from 2a).""" + from ad_buyer.booking.quote_flow import QuoteFlowClient + + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + product = {"id": "prod-001", "name": "Test"} + result = client.get_pricing(product) + # Layer 2a already returns None; the pricing_source is tracked + # at PricingCalculator level + assert result is None + + def test_build_deal_data_no_price_returns_none(self, agency_context): + """build_deal_data with no pricing returns None (no CPM populated).""" + from ad_buyer.booking.quote_flow import QuoteFlowClient + + client = QuoteFlowClient( + buyer_context=agency_context, + seller_base_url="http://localhost:5000", + ) + product = {"id": "prod-001", "name": "Test"} + result = client.build_deal_data(product) + assert result is None + + +# --------------------------------------------------------------------------- +# 8. unified_client.py — guard null pricing in get_pricing +# --------------------------------------------------------------------------- + + +class TestUnifiedClientPricingGuard: + """unified_client.get_pricing must guard against null base price.""" + + @pytest.mark.asyncio + async def test_get_pricing_no_base_price_no_crash(self): + """get_pricing on a product with no basePrice should not crash + and should not populate fabricated pricing.""" + from ad_buyer.clients.unified_client import UnifiedClient, Protocol + + client = UnifiedClient( + base_url="http://localhost:5000", + buyer_identity=BuyerIdentity( + seat_id="ttd-seat-100", + agency_id="omnicom-200", + agency_name="OMD", + ), + ) + + # Mock the get_product call to return a product with no price + mock_result = MagicMock() + mock_result.success = True + mock_result.data = { + "id": "prod-001", + "name": "Premium CTV", + # No basePrice + } + client.get_product = AsyncMock(return_value=mock_result) + + result = await client.get_pricing("prod-001") + + # Should succeed but not have fabricated pricing + assert result.success + pricing_data = result.data.get("pricing", {}) + # Should NOT have a non-zero base_price from a fabricated fallback + if pricing_data: + assert pricing_data.get("pricing_source") == "unavailable"