Cost Optimization Strategy¶

resource "azurerm_eventhub_namespace" "optimized" {
  name                = "crm-events-namespace"
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  sku                 = "Standard"
  capacity            = 1  # Start with 1 TU

  # Enable auto-inflate to handle traffic spikes
  auto_inflate_enabled     = true
  maximum_throughput_units = 5  # Scale up to 5 TU when needed

  tags = {
    Environment = "Production"
    CostCenter  = "Analytics"
  }
}

// Analyze Event Hub usage patterns
EventHubMetrics
| where TimeGenerated >= ago(30d)
| where MetricName == "IncomingMessages"
| summarize 
    HourlyMessages = avg(Total),
    PeakMessages = max(Total),
    MinMessages = min(Total)
    by bin(TimeGenerated, 1h)
| extend Hour = hourofday(TimeGenerated)
| summarize 
    AvgHourly = avg(HourlyMessages),
    PeakHourly = max(HourlyMessages)
    by Hour
| order by Hour

# Cost comparison for 7 namespaces
# Standard: 7 × $25.95 = $181.65/month
# Basic: 7 × $11.16 = $78.12/month
# Savings: $103.53/month (57%)

{
  "name": "ConsolidatedProcessing",
  "properties": {
    "activities": [
      {
        "name": "ProcessAllDomains",
        "type": "ForEach",
        "typeProperties": {
          "items": {
            "value": ["CRM", "Matching", "Profile", "Assessment"],
            "type": "Expression"
          },
          "batchCount": 4,
          "activities": [
            {
              "name": "ProcessDomain",
              "type": "SqlServerStoredProcedure",
              "typeProperties": {
                "storedProcedureName": "ProcessDomainEvents",
                "storedProcedureParameters": {
                  "DomainName": "@item()"
                }
              }
            }
          ]
        }
      }
    ]
  }
}

{
  "rules": [
    {
      "name": "aggressive-archival",
      "enabled": true,
      "type": "Lifecycle",
      "definition": {
        "filters": {
          "blobTypes": ["blockBlob"],
          "prefixMatch": ["raw-events/"]
        },
        "actions": {
          "baseBlob": {
            "tierToCool": {
              "daysAfterModificationGreaterThan": 7
            },
            "tierToArchive": {
              "daysAfterModificationGreaterThan": 30
            },
            "delete": {
              "daysAfterModificationGreaterThan": 2555
            }
          }
        }
      }
    },
    {
      "name": "bronze-optimization",
      "enabled": true,
      "type": "Lifecycle",
      "definition": {
        "filters": {
          "prefixMatch": ["bronze/"]
        },
        "actions": {
          "baseBlob": {
            "tierToCool": {
              "daysAfterModificationGreaterThan": 14
            },
            "tierToArchive": {
              "daysAfterModificationGreaterThan": 60
            }
          }
        }
      }
    }
  ]
}

// Analyze storage cost distribution
StorageAccount
| where TimeGenerated >= ago(30d)
| where MetricName == "UsedCapacity"
| extend StorageTier = case(
    BlobType == "BlockBlob" and AccessTier == "Hot", "Hot",
    BlobType == "BlockBlob" and AccessTier == "Cool", "Cool",
    BlobType == "BlockBlob" and AccessTier == "Archive", "Archive",
    "Unknown"
)
| summarize 
    AvgSize_GB = avg(Average) / 1024 / 1024 / 1024,
    MaxSize_GB = max(Average) / 1024 / 1024 / 1024
    by StorageTier, bin(TimeGenerated, 1d)
| extend EstimatedCost = case(
    StorageTier == "Hot", AvgSize_GB * 0.0208,
    StorageTier == "Cool", AvgSize_GB * 0.01,
    StorageTier == "Archive", AvgSize_GB * 0.00099,
    0
)

-- Optimized query with partition elimination
SELECT 
    aggregate_type,
    COUNT(*) as event_count,
    SUM(CAST(JSON_VALUE(payload, '$.value') AS FLOAT)) as total_value
FROM silver.business_events
WHERE year = 2024
  AND month = 1  -- Partition pruning reduces scan by 92%
  AND day >= 15
  AND aggregate_type IN ('Contact', 'Match')  -- Predicate pushdown
GROUP BY aggregate_type;

-- Before optimization: Scans 12 months = 100GB
-- After optimization: Scans 16 days = 5GB
-- Cost savings: 95% reduction in scan costs

-- Only select needed columns to reduce I/O
SELECT 
    event_id,
    timestamp,
    aggregate_type,
    JSON_VALUE(payload, '$.status') as status  -- Extract only needed JSON fields
FROM silver.business_events
WHERE year = 2024 AND month = 1
-- Don't select entire payload column (saves 60% scan volume)

-- Enable query result cache (automatic in Synapse)
-- Cached results valid for 24 hours
SELECT 
    aggregate_type,
    COUNT(*) as daily_count
FROM silver.business_events
WHERE CAST(timestamp AS DATE) = CAST(GETDATE() AS DATE)
GROUP BY aggregate_type;

-- Create materialized view for daily aggregations
CREATE MATERIALIZED VIEW gold.daily_summary_mv
AS
SELECT 
    CAST(timestamp AS DATE) as date,
    aggregate_type,
    COUNT(*) as event_count,
    COUNT(DISTINCT aggregate_id) as unique_entities,
    COUNT(DISTINCT user_id) as unique_users
FROM silver.business_events
GROUP BY CAST(timestamp AS DATE), aggregate_type;

-- Auto-refresh materialized view
ALTER MATERIALIZED VIEW gold.daily_summary_mv SET (AUTO_REFRESH = ON);

// Optimize function memory allocation
[FunctionName("ProcessEvents")]
[MemorySize(512)]  // Reduced from default 1024MB
[Timeout("00:05:00")]  // 5-minute timeout
public static async Task<IActionResult> ProcessEvents(
    [HttpTrigger(AuthorizationLevel.Function, "post")] HttpRequest req,
    ILogger log)
{
    // Use connection pooling
    using var connection = connectionPool.Get();

    // Batch processing for efficiency
    var events = await ReadEventsBatch(req, batchSize: 100);

    await ProcessEventsBatch(events);

    return new OkObjectResult($"Processed {events.Count} events");
}

// Alert on unexpected cost increases
AzureBilling
| where TimeGenerated >= ago(7d)
| where ServiceName in ("Event Hubs", "Data Factory", "Storage", "Synapse")
| summarize DailyCost = sum(Cost) by bin(TimeGenerated, 1d), ServiceName
| extend CostChange = DailyCost - prev(DailyCost, 1)
| where CostChange > 10  // Alert if daily cost increases by $10
| project TimeGenerated, ServiceName, DailyCost, CostChange

{
  "workspaceCapping": {
    "dailyQuotaGb": 1,
    "quotaNextResetTime": "2024-01-01T00:00:00Z"
  },
  "retentionInDays": 30,
  "sku": {
    "name": "PerGB2018"
  }
}

#!/bin/bash
# Auto-shutdown script for development environments

# Stop Event Hub (scale to 0 TU during off-hours)
az eventhubs namespace update \
    --name dev-events-namespace \
    --resource-group rg-dev \
    --capacity 0

# Pause Data Factory triggers
az datafactory trigger stop \
    --factory-name dev-data-factory \
    --resource-group rg-dev \
    --name dev-processing-trigger

# Schedule restart at 8 AM
at 08:00 <<EOF
az eventhubs namespace update --name dev-events-namespace --resource-group rg-dev --capacity 1
az datafactory trigger start --factory-name dev-data-factory --resource-group rg-dev --name dev-processing-trigger
EOF

# Current annual cost: $375 × 12 = $4,500
# With reserved instances: $375 × 0.75 × 12 = $3,375
# Annual savings: $1,125 (25% reduction)

// Cost tracking measures
DailyCost = 
CALCULATE(
    SUM(CostTable[Cost]),
    CostTable[Date] = TODAY()
)

MonthlyBudgetUtilization = 
VAR MonthlyBudget = 400
VAR MonthToDateCost = 
    CALCULATE(
        SUM(CostTable[Cost]),
        DATESINPERIOD(CostTable[Date], TODAY(), -1, MONTH)
    )
RETURN
    DIVIDE(MonthToDateCost, MonthlyBudget, 0)

CostEfficiencyScore = 
VAR EventsProcessed = SUM(MetricsTable[EventCount])
VAR TotalCost = [MonthToDateCost]
RETURN
    DIVIDE(EventsProcessed, TotalCost, 0)  // Events per dollar

// Alert if utilization > 80%
BudgetAlert = 
IF(
    [MonthlyBudgetUtilization] > 0.8,
    "⚠️ Budget Alert: " & FORMAT([MonthlyBudgetUtilization], "0%") & " used",
    "✅ Within Budget: " & FORMAT([MonthlyBudgetUtilization], "0%") & " used"
)