AI in Production

Taking AI from Prototype to Production

Building an AI demo is easy. Deploying a reliable, cost-effective AI application that serves thousands of users is hard. This lesson covers the engineering practices that separate production AI systems from prototypes.

Latency Optimization

// Stream AI responses in a Next.js API route
import { OpenAI } from 'openai';

const openai = new OpenAI();

export async function POST(req) {
  const { prompt } = await req.json();

  const stream = await openai.chat.completions.create({
    model: 'gpt-4o-mini',
    messages: [{ role: 'user', content: prompt }],
    stream: true,
  });

  // Convert OpenAI stream to ReadableStream
  const encoder = new TextEncoder();
  const readable = new ReadableStream({
    async start(controller) {
      for await (const chunk of stream) {
        const text = chunk.choices[0]?.delta?.content || '';
        controller.enqueue(encoder.encode(text));
      }
      controller.close();
    },
  });

  return new Response(readable, {
    headers: { 'Content-Type': 'text/plain; charset=utf-8' },
  });
}

// Simple LLM response cache
const cache = new Map();

async function cachedLLMCall(prompt, options = {}) {
  const cacheKey = JSON.stringify({ prompt, model: options.model });

  if (cache.has(cacheKey)) {
    console.log('Cache hit!');
    return cache.get(cacheKey);
  }

  const response = await openai.chat.completions.create({
    model: options.model || 'gpt-4o-mini',
    messages: [{ role: 'user', content: prompt }],
  });

  const result = response.choices[0].message.content;
  cache.set(cacheKey, result);

  // TTL: expire after 1 hour
  setTimeout(() => cache.delete(cacheKey), 3600000);

  return result;
}

// Smart model routing
function selectModel(query) {
  const complexity = estimateComplexity(query);

  if (complexity === 'simple') return 'gpt-4o-mini';    // Fast, cheap
  if (complexity === 'medium') return 'gpt-4o';          // Balanced
  return 'gpt-4';                                         // Most capable
}

function estimateComplexity(query) {
  const wordCount = query.split(' ').length;
  const hasCode = /```|function |const |class /.test(query);
  const needsReasoning = /why|how|explain|compare|analyze/.test(query.toLowerCase());

  if (wordCount < 20 && !hasCode && !needsReasoning) return 'simple';
  if (hasCode || needsReasoning) return 'complex';
  return 'medium';
}

Cost Management

Monitoring & Observability

// Production AI monitoring
class AIMonitor {
  constructor() {
    this.metrics = [];
  }

  async trackCall(name, fn) {
    const start = Date.now();
    let success = true;
    let tokensUsed = 0;
    let error = null;

    try {
      const result = await fn();
      tokensUsed = result.usage?.total_tokens || 0;
      return result;
    } catch (err) {
      success = false;
      error = err.message;
      throw err;
    } finally {
      const metric = {
        name,
        duration: Date.now() - start,
        success,
        tokensUsed,
        error,
        timestamp: new Date().toISOString(),
      };

      this.metrics.push(metric);
      await this.sendToAnalytics(metric);

      // Alert on anomalies
      if (metric.duration > 30000) {
        console.warn(`⚠️ Slow AI call: ${name} took ${metric.duration}ms`);
      }
      if (!success) {
        console.error(`❌ AI call failed: ${name} - ${error}`);
      }
    }
  }

  getAverageLatency(name) {
    const relevant = this.metrics.filter(m => m.name === name && m.success);
    return relevant.reduce((sum, m) => sum + m.duration, 0) / relevant.length;
  }

  getSuccessRate(name) {
    const relevant = this.metrics.filter(m => m.name === name);
    const successful = relevant.filter(m => m.success).length;
    return (successful / relevant.length * 100).toFixed(1) + '%';
  }
}

Handling Failures Gracefully

// Retry with exponential backoff + fallback
async function resilientAICall(prompt, options = {}) {
  const models = ['gpt-4o', 'gpt-4o-mini']; // Primary + fallback
  const maxRetries = 3;

  for (const model of models) {
    for (let attempt = 1; attempt <= maxRetries; attempt++) {
      try {
        return await openai.chat.completions.create({
          model,
          messages: [{ role: 'user', content: prompt }],
          timeout: 30000, // 30s timeout
          ...options,
        });
      } catch (err) {
        const isRateLimit = err.status === 429;
        const isServerError = err.status >= 500;

        if (isRateLimit || isServerError) {
          const delay = Math.pow(2, attempt) * 1000; // 2s, 4s, 8s
          console.warn(`Retry ${attempt}/${maxRetries} for ${model} in ${delay}ms`);
          await new Promise(r => setTimeout(r, delay));
          continue;
        }
        throw err; // Non-retryable error
      }
    }
    console.warn(`All retries failed for ${model}, trying fallback...`);
  }

  throw new Error('All AI models and retries exhausted');
}

Security Considerations