AI-Native DevOps: Building Self-Healing Infrastructure with Prometheus and LLMs

import express from 'express';
import { ChatOpenAI } from "@langchain/openai";
import { initializeAgentExecutorWithOptions } from "langchain/agents";
import { Tool } from "langchain/tools";
import { exec } from 'child_process';

const app = express();
app.use(express.json());

// Define a Custom Tool for Kubectl
class KubectlLogsTool extends Tool {
  name = "kubectl_logs";
  description = "Get logs for a pod. Input should be pod name.";
  async _call(podName: string) {
    return new Promise((resolve) => {
        exec(`kubectl logs ${podName} --tail=50`, (error, stdout) => {
            resolve(stdout || "No logs found or error.");
        });
    });
  }
}

app.post('/webhook/alertmanager', async (req, res) => {
  const alert = req.body.alerts[0];
  const podName = alert.labels.pod;

  const model = new ChatOpenAI({ temperature: 0 });
  const tools = [new KubectlLogsTool()];

  const executor = await initializeAgentExecutorWithOptions(tools, model, {
    agentType: "openai-functions",
  });

  const prompt = `
    Alert received for pod ${podName}.
    The error is: ${alert.annotations.description}.
    1. Get the logs.
    2. Analyze the root cause.
    3. Suggest a fix.
  `;

  const result = await executor.run(prompt);

  // Send result to Slack (omitted)
  console.log("AI Investigation Result:", result);

  res.sendStatus(200);
});

@RestController
public class DiagnosticController {

    @Autowired
    private DataSource dataSource;

    @GetMapping("/internal/diagnose")
    public Map<String, Object> getDiagnostics() {
        Map<String, Object> status = new HashMap<>();

        // 1. DB Connection Pool Status
        if (dataSource instanceof HikariDataSource) {
            HikariPoolMXBean pool = ((HikariDataSource) dataSource).getHikariPoolMXBean();
            status.put("db_active_connections", pool.getActiveConnections());
            status.put("db_threads_awaiting", pool.getThreadsAwaitingConnection());
        }

        // 2. Memory Usage
        status.put("heap_used", Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory());

        // 3. Last 5 Exceptions (from a memory buffer)
        status.put("recent_errors", ErrorBuffer.getLatest());

        return status;
    }
}