Build it

Put Nginx in front of three API copies with least connections, passive health checks, sane timeouts and correct headers, add a real readiness endpoint, then see what active checks look like in HAProxy. To practise the two most common picking rules in code, try the JavaScript exercise in the Practice section; it runs in your browser.

1. Define the pool

   max_fails=3 fail_timeout=10s means: if 3 attempts fail within 10 seconds, skip this server for the next 10 seconds, then try it again.

   nginxCopy

   # /etc/nginx/conf.d/api.conf
   upstream api_pool {
       least_conn;                     # fewest in-progress requests wins
   
       server 10.0.1.11:3000 max_fails=3 fail_timeout=10s;
       server 10.0.1.12:3000 max_fails=3 fail_timeout=10s;
       server 10.0.1.13:3000 max_fails=3 fail_timeout=10s weight=2;  # bigger machine
   
       keepalive 32;                   # idle connections kept open to the pool
   }
   
   upstream report_pool {
       least_conn;
       server 10.0.1.21:3000 max_fails=3 fail_timeout=10s;
       server 10.0.1.22:3000 max_fails=3 fail_timeout=10s;
   }

2. Terminate TLS, route by path, set headers and timeouts

   proxy_next_upstream lists which failures make Nginx try the next server. Connection errors and timeouts always count as failures for max_fails; 502 and 503 count only if you list them. Since Nginx 1.9.13, a POST that already reached a server is not retried elsewhere unless you add non_idempotent, which protects you from recording the same delivery twice (APIs done right).

   nginxCopy

   server {
       listen 443 ssl;
       server_name api.example.com;
       ssl_certificate     /etc/letsencrypt/live/api.example.com/fullchain.pem;
       ssl_certificate_key /etc/letsencrypt/live/api.example.com/privkey.pem;
   
       proxy_http_version 1.1;
       proxy_set_header Connection "";                    # allow upstream keepalive
       proxy_set_header Host $host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
       proxy_set_header X-Request-Id $request_id;         # trace one request in logs
   
       proxy_connect_timeout 2s;      # a healthy server on a LAN accepts in ms
       proxy_send_timeout    30s;
       proxy_read_timeout    30s;     # max wait between two reads, not total time
   
       proxy_next_upstream error timeout http_502 http_503;
       proxy_next_upstream_tries 2;   # at most one retry on another server
       proxy_next_upstream_timeout 5s;
   
       location /reports/ {
           proxy_pass http://report_pool;
           proxy_read_timeout 120s;   # exports are slow; better: make them jobs
       }
   
       location / {
           proxy_pass http://api_pool;
       }
   }

   TerminalCopy

   sudo nginx -t && sudo systemctl reload nginx   # test config, then reload with no downtime

3. Tell NestJS it is behind a proxy

   Without trust proxy, req.ip is the LB's address for every user, which breaks logging and per-IP rate limits. Raising keepAliveTimeout fixes the random 502s from the callout above.

   TypeScriptCopy

   // main.ts
   import { NestFactory } from '@nestjs/core';
   import { NestExpressApplication } from '@nestjs/platform-express';
   import { AppModule } from './app.module';
   
   async function bootstrap() {
     const app = await NestFactory.create<NestExpressApplication>(AppModule);
     app.set('trust proxy', 1);   // trust ONE hop: our LB sets X-Forwarded-For
     app.enableShutdownHooks();
   
     const server = app.getHttpServer();
     server.keepAliveTimeout = 65_000;  // longer than the LB idle timeout (60 s)
     server.headersTimeout = 66_000;    // must be larger than keepAliveTimeout
   
     await app.listen(3000);
   }
   bootstrap();

4. Add liveness and readiness with @nestjs/terminus

   Install with npm i @nestjs/terminus and add TerminusModule to the module's imports. PrismaHealthIndicator is built into Terminus 10 and later. When a check fails, @HealthCheck() makes the endpoint return 503, which is exactly what the LB needs to see.

   TypeScriptCopy

   // health.controller.ts
   import { Controller, Get } from '@nestjs/common';
   import { HealthCheck, HealthCheckService, PrismaHealthIndicator } from '@nestjs/terminus';
   import { PrismaService } from '../prisma/prisma.service';
   
   @Controller('health')
   export class HealthController {
     constructor(
       private readonly health: HealthCheckService,
       private readonly prismaHealth: PrismaHealthIndicator,
       private readonly prisma: PrismaService,
     ) {}
   
     // Liveness: is the process responding? No dependencies on purpose.
     @Get('live')
     live() {
       return { status: 'ok' };
     }
   
     // Readiness: can this copy serve traffic? The LB checks this one.
     @Get('ready')
     @HealthCheck()
     ready() {
       return this.health.check([
         () => this.prismaHealth.pingCheck('database', this.prisma, { timeout: 1000 }),
       ]);
     }
   }

   OutputCopy

   $ curl -s localhost:3000/health/ready
   {"status":"ok","info":{"database":{"status":"up"}},"error":{},"details":{"database":{"status":"up"}}}
   
   # with Postgres stopped: HTTP 503
   {"status":"error","info":{},"error":{"database":{"status":"down"}},"details":{"database":{"status":"down"}}}

5. See active health checks in HAProxy

   If you need active checks without NGINX Plus, HAProxy has them built in. inter 2s fall 2 rise 1 means: check every 2 seconds, remove after 2 failures in a row, add back after 1 pass. That is the same rule the simulator uses. The http-check send syntax needs HAProxy 2.2 or later.

   TerminalCopy

   sudo tee /etc/haproxy/haproxy.cfg > /dev/null <<'EOF'
   defaults
       mode http
       timeout connect 2s
       timeout client  30s
       timeout server  30s
       option forwardfor                  # adds X-Forwarded-For
   
   frontend web
       bind :80
       default_backend api
   
   backend api
       balance leastconn
       option httpchk
       http-check send meth GET uri /health/ready
       http-check expect status 200
       default-server inter 2s fall 2 rise 1
       server api1 10.0.1.11:3000 check
       server api2 10.0.1.12:3000 check
       server api3 10.0.1.13:3000 check
   EOF
   haproxy -c -f /etc/haproxy/haproxy.cfg   # prints "Configuration file is valid"
   sudo systemctl reload haproxy

6. Break a server on purpose

   Stop the API on one machine and send traffic. Nginx logs the failed attempts, skips that server for fail_timeout, and users of the other two notice nothing.

   TerminalCopy

   # on 10.0.1.13
   sudo systemctl stop api
   
   # from anywhere: 20 requests, print status codes
   for i in $(seq 1 20); do curl -s -o /dev/null -w "%{http_code} " https://api.example.com/health/ready; done; echo
   
   # on the LB: what Nginx saw
   sudo tail -n 3 /var/log/nginx/error.log

   OutputCopy

   200 200 200 200 200 200 200 200 200 200 200 200 200 200 200 200 200 200 200 200
   ... connect() failed (111: Connection refused) while connecting to upstream,
       upstream: "http://10.0.1.13:3000/health/ready" ...

   All 200s. "Connection refused" happens before any request bytes are sent, so Nginx can safely try the next server, even for a POST. A server that crashes in the middle of a request is different: that request was already sent, so Nginx does not resend a POST, and that one write fails. That is fine as long as the app retries with an idempotency key (APIs done right).