The LLM gateway

# gateway.toml: secrets are never in this file, only the names of the
# environment variables that hold them.
[[routes]]
name = "support"
models = [
  { provider = "openai", model = "gpt-5-mini", api_key_env = "OPENAI_API_KEY" },
  { provider = "ollama", model = "llama3.2" },   # used when OpenAI fails
]

[[keys]]
name = "team-a"
key_env = "GATEWAY_KEY_TEAM_A"
routes = ["support"]
requests_per_minute = 60                          # optional
tokens_per_minute = 100000                        # optional
budget = { tokens = 5000000, period = "month" }   # optional: "day", "month" or "total"

criogenai-gateway --config gateway.toml --listen 0.0.0.0:4000 --log-usage

from openai import OpenAI

client = OpenAI(base_url="http://localhost:4000/v1", api_key="<team-a key>")
reply = client.chat.completions.create(
    model="support",
    messages=[{"role": "user", "content": "Hello"}],
)

from anthropic import Anthropic

client = Anthropic(base_url="http://localhost:4000", api_key="<team-a key>")
reply = client.messages.create(
    model="support", max_tokens=512,
    messages=[{"role": "user", "content": "Hello"}],
)

use std::sync::Arc;

use criogenai_gateway::Gateway;
use criogenai_providers::{Endpoint, ProviderModel};

let openai = ProviderModel::new(
    Endpoint::openai(std::env::var("OPENAI_API_KEY")?),
    "gpt-5-mini",
)?;
let local = ProviderModel::new(Endpoint::ollama(), "llama3.2")?;
let team_key = std::env::var("GATEWAY_KEY_TEAM_A")?;

let gateway = Gateway::builder()
    .route_with_fallbacks("support", [Arc::new(openai) as _, Arc::new(local) as _])
    .key("team-a", team_key, ["support"])
    .build()?;

// An axum Router: serve it, or nest it in your own server.
let listener = tokio::net::TcpListener::bind("127.0.0.1:4000").await?;
axum::serve(listener, gateway.router()).await?;