xingliuhua
文章 标签 分类
xingliuhua

目录

Nginx-13 实战案例（Go 服务 + Docker）

xingliuhua 收录于 笔记
   约 7181 字   预计阅读 15 分钟 

前置阅读：Nginx-12 性能调优实战

前面 12 篇讲的都是「怎么配」，这一篇把它们串起来，做几个能跑起来的完整案例。

1. 案例一：Go 多实例负载均衡

1.1 准备 Go 应用

// main.go
package main

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var nodeName = os.Getenv("SERVER_NODE_NAME")

func main() {
	if nodeName == "" {
		nodeName = "unknown"
	}

	mux := http.NewServeMux()

	// 业务接口：返回自己是哪个节点，用来观察负载均衡效果
	mux.HandleFunc("/api/whoami", func(w http.ResponseWriter, r *http.Request) {
		rid := r.Header.Get("X-Request-ID")
		slog.Info("request", "node", nodeName, "uri", r.RequestURI, "request_id", rid)

		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, `{"node":%q,"request_id":%q,"real_ip":%q,"xff":%q,"host":%q,"proto":%q}`,
			nodeName, rid,
			r.Header.Get("X-Real-IP"),
			r.Header.Get("X-Forwarded-For"),
			r.Host,
			r.Header.Get("X-Forwarded-Proto"),
		)
	})

	// 模拟慢接口，用来观察 least_conn 和超时行为
	mux.HandleFunc("/api/slow", func(w http.ResponseWriter, r *http.Request) {
		d := r.URL.Query().Get("d")
		dur, err := time.ParseDuration(d)
		if err != nil {
			dur = 2 * time.Second
		}
		time.Sleep(dur)
		fmt.Fprintf(w, `{"node":%q,"slept":%q}`, nodeName, dur)
	})

	// 健康检查：K8s readinessProbe 和 Nginx 探活都用它
	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("ok"))
	})

	srv := &http.Server{
		Addr:              ":8800",
		Handler:           mux,
		ReadHeaderTimeout: 10 * time.Second,
		ReadTimeout:       30 * time.Second,
		WriteTimeout:      60 * time.Second,
		// ★ 必须长于 Nginx upstream 的 keepalive_timeout，
		//   否则后端先关连接会导致 Nginx 偶发 502
		IdleTimeout: 90 * time.Second,
	}

	// 优雅关闭：收到 SIGTERM 后先停止接受新请求，处理完存量再退出
	go func() {
		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
			slog.Error("listen failed", "err", err)
			os.Exit(1)
		}
	}()
	slog.Info("server started", "node", nodeName, "addr", srv.Addr)

	quit := make(chan os.Signal, 1)
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
	<-quit
	slog.Info("shutting down")

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	if err := srv.Shutdown(ctx); err != nil {
		slog.Error("shutdown failed", "err", err)
	}
	slog.Info("stopped")
}

关键点：

  • IdleTimeout 必须长于 Nginx 的 keepalive_timeout（这里 90s > 30s）。反过来的话后端会先关闭空闲连接，Nginx 恰好在这一瞬间发请求就会 502。
  • 必须实现优雅关闭。滚动发布时容器收到 SIGTERM，如果直接退出，正在处理的请求全部失败。
  • /health 要足够轻量（不查数据库），否则健康检查本身就成了负担。

1.2 Dockerfile

# 多阶段构建，最终镜像只有十几 MB
FROM golang:1.23-alpine AS builder
WORKDIR /src
COPY go.mod ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /app/server .

FROM alpine:3.20
RUN apk add --no-cache ca-certificates tzdata curl \
    && addgroup -S app && adduser -S -G app app
WORKDIR /app
COPY --from=builder /app/server .
USER app
EXPOSE 8800
# 容器内健康检查（docker-compose 的 depends_on: condition 依赖它）
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
    CMD curl -fsS http://127.0.0.1:8800/health || exit 1
ENTRYPOINT ["./server"]

1.3 Nginx 配置

# nginx/nginx.conf
user  nginx;
worker_processes auto;
worker_rlimit_nofile 65535;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections 10240;
    use epoll;
    multi_accept on;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    server_tokens off;

    log_format json escape=json
    '{'
      '"time":"$time_iso8601",'
      '"id":"$request_id",'
      '"addr":"$remote_addr",'
      '"xff":"$http_x_forwarded_for",'
      '"host":"$host",'
      '"method":"$request_method",'
      '"uri":"$request_uri",'
      '"status":$status,'
      '"bytes":$body_bytes_sent,'
      '"rt":$request_time,'
      '"urt":"$upstream_response_time",'
      '"uct":"$upstream_connect_time",'
      '"uaddr":"$upstream_addr",'
      '"ustatus":"$upstream_status",'
      '"cache":"$upstream_cache_status"'
    '}';

    access_log /var/log/nginx/access.json.log json buffer=32k flush=5s;

    sendfile     on;
    tcp_nopush   on;
    tcp_nodelay  on;

    keepalive_timeout  65s;
    keepalive_requests 1000;
    reset_timedout_connection on;

    client_max_body_size 50m;
    client_header_timeout 15s;
    client_body_timeout   15s;

    gzip on;
    gzip_vary on;
    gzip_min_length 1k;
    gzip_comp_level 5;
    gzip_types text/plain text/css application/json
               application/javascript image/svg+xml;

    limit_req_zone  $binary_remote_addr zone=perip:10m rate=50r/s;
    limit_conn_zone $binary_remote_addr zone=conn:10m;
    limit_req_status  429;
    limit_conn_status 429;

    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }

    # ---------- 后端集群 ----------
    upstream app_backend {
        least_conn;                 # 有慢接口，用最少连接更均衡

        # Docker 内部 DNS 会解析服务名
        server app1:8800 max_fails=3 fail_timeout=15s;
        server app2:8800 max_fails=3 fail_timeout=15s;
        server app3:8800 max_fails=3 fail_timeout=15s;

        keepalive 32;
        keepalive_requests 1000;
        keepalive_timeout 30s;      # 短于 Go 的 IdleTimeout 90s
    }

    include /etc/nginx/conf.d/*.conf;
}

# nginx/conf.d/app.conf
server {
    listen 80 default_server;
    server_name localhost;

    limit_conn conn 50;
    limit_req  zone=perip burst=100 nodelay;

    # 健康检查（Nginx 自己的，给上层 LB 用）
    location = /nginx-health {
        access_log off;
        return 200 "ok\n";
    }

    # 状态页
    location = /nginx-status {
        stub_status;
        access_log off;
        allow 127.0.0.1;
        allow 172.16.0.0/12;    # docker 网段
        deny all;
    }

    # API 反向代理
    location ^~ /api/ {
        proxy_pass http://app_backend;

        proxy_http_version 1.1;
        proxy_set_header Connection        "";
        proxy_set_header Host              $host;
        proxy_set_header X-Real-IP         $remote_addr;
        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Request-ID      $request_id;

        proxy_connect_timeout 3s;
        proxy_send_timeout    30s;
        proxy_read_timeout    30s;

        proxy_next_upstream error timeout http_502 http_503 http_504;
        proxy_next_upstream_tries 2;
        proxy_next_upstream_timeout 10s;

        add_header X-Request-ID  $request_id always;
        add_header X-Upstream    $upstream_addr always;

        proxy_intercept_errors on;
        error_page 502 503 504 = @api_unavailable;
    }

    location @api_unavailable {
        default_type application/json;
        return 503 '{"code":503,"message":"service unavailable","request_id":"$request_id"}';
    }
}

1.4 docker-compose.yml

services:
  app1:
    build: .
    environment:
      SERVER_NODE_NAME: "app1"
    networks: [appnet]
    restart: unless-stopped

  app2:
    build: .
    environment:
      SERVER_NODE_NAME: "app2"
    networks: [appnet]
    restart: unless-stopped

  app3:
    build: .
    environment:
      SERVER_NODE_NAME: "app3"
    networks: [appnet]
    restart: unless-stopped

  nginx:
    image: nginx:1.27-alpine
    ports:
      - "8080:80"
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./nginx/conf.d:/etc/nginx/conf.d:ro
      - ./logs/nginx:/var/log/nginx
    depends_on:
      app1: {condition: service_healthy}
      app2: {condition: service_healthy}
      app3: {condition: service_healthy}
    networks: [appnet]
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://127.0.0.1/nginx-health"]
      interval: 10s
      timeout: 3s
      retries: 3

networks:
  appnet:
    driver: bridge

depends_on 用 condition: service_healthy 而不是简单的 depends_on: [app1]——后者只等容器启动，不等它就绪。Nginx 启动时如果后端还没 ready，会解析不到或连不上。

1.5 验证

docker compose up -d --build
docker compose ps      # 确认所有服务都是 healthy

# ① 观察负载均衡
for i in $(seq 12); do
  curl -s http://127.0.0.1:8080/api/whoami | grep -o '"node":"[^"]*"'
done
# "node":"app1"
# "node":"app2"
# "node":"app3"
# ... 轮流出现

# ② 验证请求头正确传递
curl -s http://127.0.0.1:8080/api/whoami | python3 -m json.tool
# {
#     "node": "app2",
#     "request_id": "a1b2c3...",
#     "real_ip": "172.18.0.1",
#     "xff": "172.18.0.1",
#     "host": "127.0.0.1",
#     "proto": "http"
# }

# ③ 验证故障转移
docker compose stop app2
for i in $(seq 12); do
  curl -s http://127.0.0.1:8080/api/whoami | grep -o '"node":"[^"]*"'
done
# 只有 app1 和 app3，请求不会失败

docker compose start app2
sleep 20     # 等 fail_timeout 过去
# app2 自动回到轮询

# ④ 验证 upstream keepalive 生效
docker compose exec nginx sh -c 'apk add -q iproute2 2>/dev/null; ss -tan | grep 8800 | awk "{print \$1}" | sort | uniq -c'
#  32 ESTAB          ← 长连接池
#   0 TIME-WAIT      ← 没有大量 TIME_WAIT 说明 keepalive 生效

# ⑤ 验证限流
for i in $(seq 200); do
  curl -s -o /dev/null -w "%{http_code} " http://127.0.0.1:8080/api/whoami &
done | tr ' ' '\n' | sort | uniq -c
#  151 200
#   49 429          ← burst=100 + rate=50r/s 之外的被限流

# ⑥ 验证全链路追踪
RID=$(curl -sI http://127.0.0.1:8080/api/whoami | grep -i x-request-id | tr -d '\r' | awk '{print $2}')
echo "request_id: $RID"
grep "$RID" logs/nginx/access.json.log | jq .
docker compose logs app1 app2 app3 2>&1 | grep "$RID"
# Nginx 日志和后端日志用同一个 ID 串起来了

# ⑦ 验证超时行为
curl -s -w "\ntime: %{time_total}s\n" "http://127.0.0.1:8080/api/slow?d=40s"
# 40s > proxy_read_timeout 30s，返回 504（被 error_page 转成 503 JSON）

2. 案例二：灰度发布

在案例一的基础上加一套灰度环境。

# nginx/nginx.conf 的 http 块里
upstream app_stable {
    least_conn;
    server app1:8800 max_fails=3 fail_timeout=15s;
    server app2:8800 max_fails=3 fail_timeout=15s;
    keepalive 32;
    keepalive_timeout 30s;
}

upstream app_canary {
    server canary1:8800 max_fails=3 fail_timeout=15s;
    keepalive 16;
    keepalive_timeout 30s;
}

# ---------- 灰度分流规则（优先级从高到低）----------

# ① 请求头强制指定（测试用，优先级最高）
map $http_x_canary $pool_by_header {
    default  "";
    "1"      "canary";
    "true"   "canary";
    "0"      "stable";
    "false"  "stable";
}

# ② Cookie 指定（用户主动加入灰度）
map $cookie_canary $pool_by_cookie {
    default  "";
    "1"      "canary";
    "0"      "stable";
}

# ③ 按用户 ID 尾号灰度（稳定分流，同一用户永远在同一侧）
map $cookie_uid $uid_tail {
    default   "";
    "~(\d)$"  $1;
}
map $uid_tail $pool_by_uid {
    default  "";
    "7"      "canary";     # 尾号 7 的用户，约 10%
}

# ④ 按 IP+UA 哈希做百分比灰度（无登录态时用）
split_clients "${remote_addr}${http_user_agent}" $pool_by_hash {
    5%   "canary";
    *    "stable";
}

# 合并：前面的优先
map "$pool_by_header|$pool_by_cookie|$pool_by_uid|$pool_by_hash" $target_pool {
    default            "app_stable";
    "~^canary"         "app_canary";     # header 命中
    "~^\|canary"       "app_canary";     # cookie 命中
    "~^\|\|canary"     "app_canary";     # uid 命中
    "~^\|\|\|canary"   "app_canary";     # hash 命中
    "~^stable"         "app_stable";
    "~^\|stable"       "app_stable";
}

# nginx/conf.d/app.conf
location ^~ /api/ {
    proxy_pass http://$target_pool;

    include /etc/nginx/snippets/proxy-headers.conf;

    # 告诉后端和调试者走的是哪一侧
    proxy_set_header X-Pool $target_pool;
    add_header X-Pool      $target_pool always;
    add_header X-Upstream  $upstream_addr always;
}

验证：

# 强制走灰度
curl -sI -H "X-Canary: 1" http://127.0.0.1:8080/api/whoami | grep -i x-pool
# x-pool: app_canary

# 强制走稳定版
curl -sI -H "X-Canary: 0" http://127.0.0.1:8080/api/whoami | grep -i x-pool
# x-pool: app_stable

# Cookie 方式
curl -sI -b "canary=1" http://127.0.0.1:8080/api/whoami | grep -i x-pool
# x-pool: app_canary

# 按 uid 尾号
curl -sI -b "uid=10007" http://127.0.0.1:8080/api/whoami | grep -i x-pool
# x-pool: app_canary
curl -sI -b "uid=10003" http://127.0.0.1:8080/api/whoami | grep -i x-pool
# x-pool: app_stable

# 统计百分比灰度的实际比例
for i in $(seq 500); do
  curl -sI -A "ua-$RANDOM" http://127.0.0.1:8080/api/whoami | grep -i "^x-pool" | tr -d '\r'
done | sort | uniq -c
#  475 x-pool: app_stable
#   25 x-pool: app_canary      ← 约 5%

灰度期间的关键监控：分别对比两个 pool 的 5xx 率和 P99 延迟。

# 分 pool 统计错误率（需要在 log_format 里加 $upstream_addr）
jq -r 'select(.status >= 500) | .uaddr' logs/nginx/access.json.log | sort | uniq -c

# 分 pool 统计 P99
jq -r 'select(.uaddr | startswith("172.18.0.5")) | .rt' logs/nginx/access.json.log \
  | sort -n | awk '{a[NR]=$1} END{printf "P99: %.3f\n", a[int(NR*0.99)]}'

灰度回滚只需一行：把 map 里灰度的比例改成 0，nginx -s reload。秒级生效，不用重新部署。

3. 案例三：WebSocket

// ws.go（用 gorilla/websocket）
package main

import (
	"log/slog"
	"net/http"
	"time"

	"github.com/gorilla/websocket"
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		// 生产环境要做真实的 Origin 校验
		return true
	},
}

func wsHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		slog.Error("upgrade failed", "err", err)
		return
	}
	defer conn.Close()

	// 心跳：定期发 ping，保持 Nginx 和中间设备不认为连接空闲
	conn.SetPongHandler(func(string) error {
		return conn.SetReadDeadline(time.Now().Add(90 * time.Second))
	})
	go func() {
		t := time.NewTicker(30 * time.Second)
		defer t.Stop()
		for range t.C {
			if err := conn.WriteControl(websocket.PingMessage, nil,
				time.Now().Add(10*time.Second)); err != nil {
				return
			}
		}
	}()

	conn.SetReadDeadline(time.Now().Add(90 * time.Second))
	for {
		mt, msg, err := conn.ReadMessage()
		if err != nil {
			slog.Info("read closed", "err", err)
			return
		}
		conn.SetReadDeadline(time.Now().Add(90 * time.Second))
		if err := conn.WriteMessage(mt, append([]byte(nodeName+" echo: "), msg...)); err != nil {
			return
		}
	}
}

# 关键：WebSocket 需要 HTTP/1.1 + Upgrade 头 + 长超时
http {
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }

    upstream ws_backend {
        # ★ WebSocket 是长连接且有状态，必须做会话保持
        #   否则重连后可能落到别的节点（如果业务有内存状态）
        hash $remote_addr consistent;
        server app1:8800;
        server app2:8800;
        server app3:8800;
        # WebSocket 不需要 upstream keepalive（连接本身就是长期的）
    }
}

server {
    location ^~ /ws/ {
        proxy_pass http://ws_backend;

        # ★ 这三行是 WebSocket 的核心
        proxy_http_version 1.1;
        proxy_set_header Upgrade    $http_upgrade;
        proxy_set_header Connection $connection_upgrade;

        proxy_set_header Host            $host;
        proxy_set_header X-Real-IP       $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        # ★ 超时必须放宽，否则连接会被 Nginx 掐断
        proxy_connect_timeout 5s;
        proxy_send_timeout    3600s;
        proxy_read_timeout    3600s;

        # ★ 必须关闭缓冲，否则消息会被攒着不发
        proxy_buffering off;

        # 不记 access_log（一个 WS 连接只有一条建连记录，价值不大）
        access_log off;
    }
}

WebSocket 的四个必知要点：

  1. proxy_http_version 1.1：WebSocket 的 Upgrade 机制要求 HTTP/1.1。
  2. Connection: $connection_upgrade：不能写死 upgrade，因为普通请求的 Connection 应该是 close。用 map 动态判断。
  3. proxy_read_timeout 要长：默认 60s，客户端 60 秒不发消息连接就断了。设 3600s 并配合应用层心跳。
  4. proxy_buffering off：不关的话 Nginx 会攒够缓冲区才发，实时性全无。

reload 对 WebSocket 的影响：老 worker 上的 WS 连接不会主动断开，会一直挂着直到客户端断开或 worker_shutdown_timeout 到期。所以：

worker_shutdown_timeout 30s;    # 避免老 worker 永久残留

客户端必须实现自动重连，因为 reload、发版、网络抖动都会断连。

测试：

# 用 websocat
websocat ws://127.0.0.1:8080/ws/echo
# 输入 hello，回显 "app2 echo: hello"

# 或用 wscat
npm i -g wscat
wscat -c ws://127.0.0.1:8080/ws/echo

4. 案例四：大文件上传与下载

4.1 上传

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	// 流式接收，不把整个文件读进内存
	reader, err := r.MultipartReader()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	for {
		part, err := reader.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		if part.FileName() == "" {
			continue
		}

		dst, err := os.Create(filepath.Join("/data/uploads", filepath.Base(part.FileName())))
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		n, err := io.Copy(dst, part)   // 流式拷贝，内存占用恒定
		dst.Close()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		slog.Info("uploaded", "name", part.FileName(), "bytes", n)
	}
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(`{"ok":true}`))
}

location = /api/upload {
    proxy_pass http://app_backend;
    include /etc/nginx/snippets/proxy-headers.conf;

    client_max_body_size 5g;

    # ★ 关闭请求体缓冲：边收边转，不先落盘
    #   代价是失去重试能力（请求体已流给上游，无法重发到另一台）
    proxy_request_buffering off;

    # 上传慢，超时要放宽
    proxy_send_timeout    600s;
    proxy_read_timeout    600s;
    client_body_timeout   600s;

    # 上传时不重试（请求体已经流走了，重试也没意义）
    proxy_next_upstream off;
}

proxy_request_buffering on（默认）vs off：

on（默认） off
行为 Nginx 收完整个请求体再转给上游 边收边转
内存/磁盘 超过 client_body_buffer_size 写临时文件 不落盘
首字节延迟 高（要等整个上传完成） 低
后端占用 后端只被占用很短时间 后端全程被占用
重试能力 有 无
适合 小文件、需要重试保障 大文件、流式处理

大文件（>100MB）用 off，小文件用默认的 on。

4.2 下载（X-Accel-Redirect）

func downloadHandler(w http.ResponseWriter, r *http.Request) {
	fileID := r.URL.Query().Get("id")

	// 只做鉴权和查路径，几毫秒的事
	if !checkPermission(r, fileID) {
		http.Error(w, `{"error":"forbidden"}`, http.StatusForbidden)
		return
	}
	relPath, fileName, err := lookupFile(fileID)
	if err != nil {
		http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
		return
	}

	// ★ 把传输工作交给 Nginx，本函数立即返回
	w.Header().Set("X-Accel-Redirect", "/internal-files/"+relPath)
	w.Header().Set("Content-Disposition",
		`attachment; filename*=UTF-8''`+url.PathEscape(fileName))
	w.Header().Set("Content-Type", "application/octet-stream")
	// 注意：不要写任何响应体
}

# 客户端访问这里
location = /api/download {
    proxy_pass http://app_backend;
    include /etc/nginx/snippets/proxy-headers.conf;
}

# Nginx 内部真正发文件的地方
location ^~ /internal-files/ {
    internal;                    # ★ 客户端直接访问返回 404
    alias /data/files/;

    sendfile on;
    sendfile_max_chunk 2m;
    aio threads;
    directio 8m;
    output_buffers 2 512k;

    limit_rate 5m;               # 单连接 5MB/s
    limit_rate_after 20m;        # 前 20MB 不限速
    limit_conn conn 2;           # 单 IP 最多 2 个并发下载

    gzip off;
    access_log off;
}

# main 块
thread_pool default threads=32 max_queue=65536;

收益：一个 1GB 文件的下载，Go 的 handler 只执行了几毫秒（鉴权 + 查路径）就返回了，goroutine 立即释放。传输由 Nginx 的 sendfile 完成，还免费获得了 Range 断点续传支持。

不用这个技巧的话，Go 要用 io.Copy(w, file) 陪着客户端传几分钟，1000 个并发下载就是 1000 个 goroutine 长期占用。

5. 案例五：蓝绿部署（零停机发布）

思路：两套完整环境（blue / green），Nginx 通过一个可热切换的配置指向当前生效的那套。

# nginx/upstreams/blue.conf
upstream app_active {
    least_conn;
    server blue1:8800 max_fails=3 fail_timeout=15s;
    server blue2:8800 max_fails=3 fail_timeout=15s;
    keepalive 32;
    keepalive_timeout 30s;
}
upstream app_standby {
    least_conn;
    server green1:8800 max_fails=3 fail_timeout=15s;
    server green2:8800 max_fails=3 fail_timeout=15s;
    keepalive 32;
    keepalive_timeout 30s;
}

# nginx/nginx.conf
http {
    # 用软链指向当前生效的配置
    include /etc/nginx/upstreams/active.conf;
}

# nginx/conf.d/app.conf
location ^~ /api/ {
    proxy_pass http://app_active;
    include /etc/nginx/snippets/proxy-headers.conf;
}

# 预发布验证入口：只有带特定头才能访问 standby
location ^~ /standby/api/ {
    internal;   # 或用 IP 白名单
    rewrite ^/standby/(.*)$ /$1 break;
    proxy_pass http://app_standby;
    include /etc/nginx/snippets/proxy-headers.conf;
}

切换脚本：

#!/bin/bash
# switch.sh —— 蓝绿切换
set -euo pipefail

NGINX_UPSTREAM_DIR=/etc/nginx/upstreams
CURRENT=$(readlink "$NGINX_UPSTREAM_DIR/active.conf" | xargs basename)

if [ "$CURRENT" = "blue.conf" ]; then
    TARGET="green.conf"; TARGET_NAME="green"; NEW_HOSTS="green1 green2"
else
    TARGET="blue.conf";  TARGET_NAME="blue";  NEW_HOSTS="blue1 blue2"
fi

echo "当前: $CURRENT → 切换到: $TARGET"

# ① 确认目标环境的所有实例都健康
for h in $NEW_HOSTS; do
    if ! curl -fsS --max-time 3 "http://$h:8800/health" > /dev/null; then
        echo "ERROR: $h 健康检查失败，中止切换"
        exit 1
    fi
    echo "  $h ✓"
done

# ② 切换软链
ln -sfn "$NGINX_UPSTREAM_DIR/$TARGET" "$NGINX_UPSTREAM_DIR/active.conf"

# ③ 校验配置后 reload
if ! nginx -t; then
    echo "ERROR: 配置校验失败，回滚软链"
    ln -sfn "$NGINX_UPSTREAM_DIR/$CURRENT" "$NGINX_UPSTREAM_DIR/active.conf"
    exit 1
fi
nginx -s reload

echo "已切换到 $TARGET_NAME"

# ④ 观察 60 秒，5xx 超标自动回滚
echo "观察 60 秒..."
sleep 5
START=$(date +%s)
while [ $(( $(date +%s) - START )) -lt 60 ]; do
    TOTAL=$(tail -2000 /var/log/nginx/access.json.log | wc -l)
    ERR=$(tail -2000 /var/log/nginx/access.json.log | jq -r 'select(.status>=500)' 2>/dev/null | wc -l)
    if [ "$TOTAL" -gt 100 ] && [ "$ERR" -gt $((TOTAL / 100)) ]; then
        echo "ERROR: 5xx 比例 $ERR/$TOTAL 超过 1%，自动回滚"
        ln -sfn "$NGINX_UPSTREAM_DIR/$CURRENT" "$NGINX_UPSTREAM_DIR/active.conf"
        nginx -t && nginx -s reload
        exit 1
    fi
    sleep 5
done

echo "切换成功，$CURRENT 环境可以下线更新了"

蓝绿 vs 滚动：

蓝绿 滚动
资源 需要 2 倍机器 只需少量余量
回滚 秒级（切回软链） 需要重新部署旧版本
新旧共存 不共存（瞬间全切） 共存（要求版本兼容）
数据库迁移 麻烦（两套代码要兼容同一个库） 同样麻烦

滚动发布更省资源，但要求新旧版本能同时在线（尤其是 API 契约和数据库 schema）。蓝绿的回滚更干脆。

6. 案例六：K8s Ingress-nginx

K8s 环境下不需要自己维护 upstream 列表——Service 的 Endpoint 由 kubelet 的探针自动维护。

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: app-ingress
  annotations:
    # ---------- 基础 ----------
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"

    # ---------- 超时 ----------
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "3"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "30"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "30"

    # ---------- 请求体 ----------
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"

    # ---------- 限流 ----------
    nginx.ingress.kubernetes.io/limit-rps: "50"
    nginx.ingress.kubernetes.io/limit-burst-multiplier: "3"
    nginx.ingress.kubernetes.io/limit-connections: "30"

    # ---------- 灰度（canary）----------
    # 在另一个 Ingress 上设置这些即可分流
    # nginx.ingress.kubernetes.io/canary: "true"
    # nginx.ingress.kubernetes.io/canary-weight: "10"
    # nginx.ingress.kubernetes.io/canary-by-header: "X-Canary"
    # nginx.ingress.kubernetes.io/canary-by-cookie: "canary"

    # ---------- CORS ----------
    nginx.ingress.kubernetes.io/enable-cors: "true"
    nginx.ingress.kubernetes.io/cors-allow-origin: "https://example.com"
    nginx.ingress.kubernetes.io/cors-allow-credentials: "true"

    # ---------- 自定义配置片段 ----------
    nginx.ingress.kubernetes.io/configuration-snippet: |
      more_set_headers "X-Request-ID: $req_id";
      proxy_set_header X-Request-ID $req_id;      

    # ---------- 会话保持（有状态服务）----------
    # nginx.ingress.kubernetes.io/affinity: "cookie"
    # nginx.ingress.kubernetes.io/session-cookie-name: "route"
spec:
  ingressClassName: nginx
  tls:
  - hosts: [api.example.com]
    secretName: api-tls
  rules:
  - host: api.example.com
    http:
      paths:
      - path: /api
        pathType: Prefix
        backend:
          service:
            name: app-svc
            port: {number: 80}

全局配置用 ConfigMap：

apiVersion: v1
kind: ConfigMap
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
data:
  # 上游长连接
  upstream-keepalive-connections: "64"
  upstream-keepalive-requests: "1000"
  upstream-keepalive-timeout: "30"

  # 客户端连接
  keep-alive: "65"
  keep-alive-requests: "1000"

  # worker
  worker-processes: "auto"
  max-worker-connections: "16384"

  # 日志（JSON）
  log-format-escape-json: "true"
  log-format-upstream: >-
    {"time":"$time_iso8601","id":"$req_id","addr":"$remote_addr",
    "host":"$host","method":"$request_method","uri":"$request_uri",
    "status":$status,"rt":$request_time,"urt":"$upstream_response_time",
    "uaddr":"$upstream_addr","namespace":"$namespace",
    "service":"$service_name","ingress":"$ingress_name"}    

  # 真实 IP（云 LB 后面）
  use-forwarded-headers: "true"
  compute-full-forwarded-for: "true"
  proxy-real-ip-cidr: "10.0.0.0/8,172.16.0.0/12"

  # TLS
  ssl-protocols: "TLSv1.2 TLSv1.3"
  ssl-session-cache: "true"
  ssl-session-cache-size: "50m"
  ssl-session-timeout: "1d"

  # 压缩
  use-gzip: "true"
  gzip-level: "5"

  # 安全
  server-tokens: "false"
  enable-modsecurity: "false"

  # 优雅关闭
  worker-shutdown-timeout: "30s"

配套的 Deployment 优雅关闭配置：

spec:
  template:
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: app
        lifecycle:
          preStop:
            exec:
              # ★ 关键：先睡一会，让 Endpoint 从 Service 里摘掉、
              #   Ingress 的 upstream 列表更新完，再开始关闭
              command: ["sh", "-c", "sleep 10"]
        readinessProbe:
          httpGet: {path: /health, port: 8800}
          periodSeconds: 3
          failureThreshold: 2
        livenessProbe:
          httpGet: {path: /health, port: 8800}
          periodSeconds: 10
          failureThreshold: 3

preStop 里的 sleep 10 非常关键。K8s 删 Pod 的流程是「发 SIGTERM」和「从 Endpoint 移除」并行的，不是串行。如果应用收到 SIGTERM 立刻停止接受新连接，但 Ingress 的 upstream 列表还没更新，这段时间内的请求就会 502。sleep 10 让 Endpoint 先更新完，是消灭发布期间 502 的标准手段。

7. 案例七：完整的前后端分离站点

综合前面所有内容，一个生产级的完整配置：

# nginx/conf.d/example.com.conf

# HTTP → HTTPS
server {
    listen 80;
    listen [::]:80;
    server_name example.com www.example.com;

    # ACME 验证要在跳转之前
    location ^~ /.well-known/acme-challenge/ {
        root /var/www/acme;
        default_type "text/plain";
    }

    location / {
        return 301 https://example.com$request_uri;
    }
}

# www → 非 www
server {
    listen 443 ssl;
    http2 on;
    server_name www.example.com;
    include snippets/ssl-common.conf;
    ssl_certificate     /etc/nginx/ssl/example.com/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/example.com/privkey.pem;
    return 301 https://example.com$request_uri;
}

# 主站
server {
    listen 443 ssl reuseport;
    listen [::]:443 ssl;
    http2 on;
    server_name example.com;

    root /var/www/dist;
    index index.html;

    ssl_certificate     /etc/nginx/ssl/example.com/ecdsa/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/example.com/ecdsa/privkey.pem;
    ssl_certificate     /etc/nginx/ssl/example.com/rsa/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/example.com/rsa/privkey.pem;

    include snippets/ssl-common.conf;
    include snippets/security-headers.conf;

    limit_conn conn 50;
    limit_req  zone=perip burst=100 nodelay;

    # ---------- 健康检查 ----------
    location = /nginx-health {
        access_log off;
        return 200 "ok\n";
    }

    # ---------- API ----------
    location ^~ /api/ {
        include snippets/proxy-headers.conf;
        proxy_pass http://app_backend;

        proxy_cache api_cache;
        proxy_cache_valid 200 2m;
        proxy_cache_lock on;
        proxy_cache_use_stale error timeout updating http_5xx;
        proxy_cache_background_update on;
        proxy_cache_bypass $cookie_sessionid;
        proxy_no_cache     $cookie_sessionid;

        add_header X-Cache-Status $upstream_cache_status always;
        add_header X-Request-ID   $request_id always;

        proxy_intercept_errors on;
        error_page 502 503 504 = @api_error;
    }

    # 写接口不缓存、不重试
    location ~ ^/api/(login|register|order|pay) {
        include snippets/proxy-headers.conf;
        proxy_pass http://app_backend;
        proxy_cache off;
        proxy_next_upstream error;      # 只在连不上时重试
        limit_req zone=strict burst=5 nodelay;
    }

    # ---------- WebSocket ----------
    location ^~ /ws/ {
        proxy_pass http://ws_backend;
        proxy_http_version 1.1;
        proxy_set_header Upgrade    $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_set_header Host       $host;
        proxy_set_header X-Real-IP  $remote_addr;
        proxy_read_timeout 3600s;
        proxy_send_timeout 3600s;
        proxy_buffering off;
        access_log off;
    }

    # ---------- 文件下载 ----------
    location = /api/download {
        include snippets/proxy-headers.conf;
        proxy_pass http://app_backend;
    }
    location ^~ /internal-files/ {
        internal;
        alias /data/files/;
        sendfile on;
        sendfile_max_chunk 2m;
        aio threads;
        directio 8m;
        limit_rate 5m;
        limit_rate_after 20m;
        gzip off;
        access_log off;
    }

    # ---------- 静态资源 ----------
    # 带 hash：永久缓存
    location ~* \.[0-9a-f]{8,}\.(js|css|woff2?)$ {
        expires 1y;
        add_header Cache-Control "public, max-age=31536000, immutable";
        access_log off;
        try_files $uri =404;
    }

    # 图片字体：长缓存
    location ~* \.(png|jpe?g|gif|svg|ico|webp|avif|woff2?|ttf)$ {
        expires 30d;
        add_header Cache-Control "public";
        access_log off;
        try_files $uri =404;
    }

    # ---------- 安全屏蔽 ----------
    location ~ /\.(?!well-known)   { return 404; access_log off; }
    location ~* \.(bak|sql|env|conf|ini|log|map)$ { return 404; }
    location ~* /(phpmyadmin|wp-admin|actuator|druid)/ { return 444; }

    # ---------- SPA 兜底 ----------
    location / {
        try_files $uri $uri/ /index.html;

        location = /index.html {
            add_header Cache-Control "no-cache, no-store, must-revalidate";
            expires -1;
        }
    }

    location @api_error {
        default_type application/json;
        add_header X-Request-ID $request_id always;
        return 503 '{"code":503,"message":"service unavailable","request_id":"$request_id"}';
    }
}

8. 上线前的验收脚本

#!/bin/bash
# verify.sh —— 上线前逐项验收
set -uo pipefail
HOST=${1:-example.com}
PASS=0; FAIL=0

check() {
    local desc="$1"; shift
    if "$@" > /dev/null 2>&1; then
        echo "  ✓ $desc"; PASS=$((PASS+1))
    else
        echo "  ✗ $desc"; FAIL=$((FAIL+1))
    fi
}

echo "=== 配置 ==="
check "nginx -t 通过" nginx -t

echo "=== 连通性 ==="
check "HTTP 跳转 HTTPS" bash -c "curl -sI http://$HOST | grep -q '301'"
check "HTTPS 可访问"    bash -c "curl -sI https://$HOST | grep -q '200'"
check "HTTP/2 生效"     bash -c "curl -sI --http2 https://$HOST | head -1 | grep -q 'HTTP/2'"

echo "=== TLS ==="
check "证书链完整" bash -c "echo | openssl s_client -connect $HOST:443 -servername $HOST 2>&1 | grep -q 'Verify return code: 0'"
check "会话复用生效" bash -c "openssl s_client -connect $HOST:443 -servername $HOST -reconnect </dev/null 2>&1 | grep -q '^Reused'"
check "OCSP Stapling" bash -c "echo | openssl s_client -connect $HOST:443 -servername $HOST -status 2>&1 | grep -q 'Cert Status: good'"
check "TLS 1.0 已禁用" bash -c "! echo | openssl s_client -connect $HOST:443 -tls1 2>&1 | grep -q 'Cipher is'"
check "证书 15 天内不过期" bash -c "
    e=\$(echo | openssl s_client -connect $HOST:443 -servername $HOST 2>/dev/null | openssl x509 -noout -enddate | cut -d= -f2)
    [ \$(( (\$(date -d \"\$e\" +%s) - \$(date +%s)) / 86400 )) -gt 15 ]"

echo "=== 安全头 ==="
for h in strict-transport-security x-content-type-options x-frame-options; do
    check "$h" bash -c "curl -sI https://$HOST | grep -qi '$h'"
done
check "版本号已隐藏" bash -c "! curl -sI https://$HOST | grep -qiE 'server: nginx/[0-9]'"

echo "=== 功能 ==="
check "健康检查"   bash -c "curl -fsS https://$HOST/nginx-health | grep -q ok"
check "API 可用"   bash -c "curl -fsS https://$HOST/api/whoami | grep -q node"
check "gzip 生效"  bash -c "curl -sI -H 'Accept-Encoding: gzip' https://$HOST/ | grep -qi 'content-encoding'"
check "request_id 返回" bash -c "curl -sI https://$HOST/api/whoami | grep -qi x-request-id"
check "限流生效" bash -c "
    for i in \$(seq 300); do curl -so /dev/null -w '%{http_code}\n' https://$HOST/api/whoami & done | grep -q 429"

echo "=== 安全屏蔽 ==="
check ".git 已屏蔽"    bash -c "curl -so /dev/null -w '%{http_code}' https://$HOST/.git/config | grep -qE '404|403'"
check ".env 已屏蔽"    bash -c "curl -so /dev/null -w '%{http_code}' https://$HOST/.env | grep -qE '404|403'"
check "sourcemap 屏蔽" bash -c "curl -so /dev/null -w '%{http_code}' https://$HOST/app.js.map | grep -q 404"

echo
echo "通过 $PASS 项，失败 $FAIL 项"
[ "$FAIL" -eq 0 ]

9. 面试题

Q：Docker 里 Nginx 反代其他容器，proxy_pass 写服务名有什么坑？

Docker 的内嵌 DNS 会解析服务名，但Nginx 对写死的域名只在启动/reload 时解析一次并缓存到进程结束。容器重建后 IP 变了，Nginx 还在用旧 IP，导致持续 502。解决方案：(1) 用 resolver 127.0.0.11 valid=10s; + 变量式 proxy_pass http://$backend$request_uri; 强制运行时解析；(2) 或者用 upstream 里的 resolve 参数（1.27.3+ 开源版支持）；(3) 或者交给 K8s Service。

Q：WebSocket 通过 Nginx 代理需要哪些配置？

四项：(1) proxy_http_version 1.1（Upgrade 机制要求 HTTP/1.1）；(2) proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade;（用 map 动态判断，不能写死 upgrade，否则普通请求会出错）；(3) proxy_read_timeout/proxy_send_timeout 放大到 3600s（默认 60s 会掐断空闲连接）；(4) proxy_buffering off（否则消息被攒着不发，实时性全无）。此外要配 worker_shutdown_timeout 避免 reload 时老 worker 被 WS 长连接拖住不退。

Q：大文件上传怎么优化？proxy_request_buffering off 的代价是什么？

默认 on 时 Nginx 先收完整个请求体（超出 client_body_buffer_size 就写临时文件）再转给上游，大文件会造成高延迟和大量磁盘 I/O。设 off 后边收边转，不落盘、延迟低。代价是失去重试能力——请求体已经流给上游了，无法重发到另一台，所以要配 proxy_next_upstream off。另外后端全程被占用（而不是像 on 那样只被占用很短时间）。大文件用 off，小文件保持默认。

Q：K8s 里滚动更新时为什么会出现 502？怎么解决？

K8s 删 Pod 时「发送 SIGTERM」和「从 Endpoint 移除」是并行的，不是串行。应用收到 SIGTERM 立刻停止接受新连接时，Ingress-nginx 的 upstream 列表可能还没更新，这个窗口期的请求就会 502。标准解法是在 preStop 里加 sleep 10——让 Endpoint 先更新完、Ingress 感知到变化，再开始关闭应用。同时应用必须实现优雅关闭（处理完存量请求再退出），并设置足够的 terminationGracePeriodSeconds。

Q：蓝绿部署和滚动发布怎么选？

蓝绿需要 2 倍机器，但回滚是秒级的（切回软链/切回 upstream），且新旧版本不共存（不用担心版本兼容）。滚动只需少量余量，但发布期间新旧版本同时在线，要求 API 契约和数据库 schema 向前兼容，回滚需要重新部署。资源充足且要求快速回滚 → 蓝绿；资源紧张且版本兼容做得好 → 滚动。

Q：X-Accel-Redirect 怎么用在鉴权下载场景？

后端 handler 只做鉴权和查文件路径（几毫秒），然后返回 X-Accel-Redirect: /internal-files/xxx 头且不写响应体。Nginx 收到这个头后丢弃上游响应，对该 URI 做内部重定向，由标了 internal 的 location 用 sendfile 发送文件。后端进程/goroutine 立即释放，不用陪着客户端传几分钟，还免费获得 Range 断点续传。

Q：Nginx 的 upstream keepalive 超时为什么要短于后端的 IdleTimeout？

如果 Nginx 侧超时更长，后端会先关闭空闲连接，而 Nginx 可能刚好在这个瞬间往这条连接上发请求，触发 upstream prematurely closed connection 导致偶发 502。让 Nginx 主动淘汰连接（Nginx 30s，后端 90s）就能避免。规律是：连接池的空闲超时应由持有池的一方先关闭。

Q：灰度发布用 Nginx 怎么实现？各种分流方式的取舍？

用 map + 变量式 proxy_pass http://$target_pool。分流方式按优先级组合：(1) 请求头 X-Canary——测试和验证用，最灵活；(2) Cookie——用户主动加入灰度；(3) 用户 ID 尾号（map 提取）——稳定分流，同一用户永远在同一侧，体验一致，这是生产首选；(4) split_clients 按 IP+UA 哈希——无登录态时的百分比分流，同样稳定。不要用 upstream 的 weight 做灰度——同一用户的连续请求会在新旧版本间跳，体验不一致。回滚时改 map 比例并 reload，秒级生效。

上一篇：Nginx-12 性能调优实战 ｜ 下一篇：Nginx-14 高频面试题汇总

更新于 2026-08-03
 nginx
返回 | 主页