Nginx作为WEB服务器被广泛使用。其自身支持热更新,在修改配置文件后,使用nginx -s reload命令可以不停服务重新加载配置。然而对于Dockerize的Nginx来说,如果每次都进到容器里执行对应命令去实现配置重载,这个过程是很痛苦的。本文介绍了一种kubernetes集群下nginx的热更新方案。

环境说明

  • Kubernetes集群中部署Nginx Pod。该Pod包含两个Container,一个是nginx container,实现nginx自身的功能;另一个是nginx-reloader container,负责实时监测目标configmap的变化,当发现configmap更新以后,会主动向nginx的master进程发送HUP信号,实现配置的热加载。
  • 配置文件是通过ConfigMap的形式挂载到Nginx Pod上,两个Container共享该ConfigMap。
  • 依赖K8s集群的shareProcessNamespace特性(版本需在1.12之后),两个Container需要在Pod中共享进程名字空间。

实现过程

镜像准备

  • 主容器

使用官方的nginx镜像即可,可以稍作修改(如封装常用的命令)并推送到自己的私有镜像仓库。

  • Sidecar容器
    nginx-reloader镜像的Dockerfile如下:
FROM golang:1.12.0 as build

RUN go get github.com/fsnotify/fsnotify
RUN go get github.com/shirou/gopsutil/process
RUN mkdir -p /go/src/app
ADD main.go /go/src/app/
WORKDIR /go/src/app
RUN CGO_ENABLED=0 GOOS=linux go build -a -o nginx-reloader .

# main image
FROM nginx:1.14.2-alpine

COPY --from=build /go/src/app/nginx-reloader /

CMD ["/nginx-reloader"]

基于该Dockerfile构建nginx-reloader镜像:

docker build -t xxx/nginx-reloader .

其功能是用一个叫main.go的go脚本实现的:

package main

import (
    "log"
    "os"
    "path/filepath"
    "syscall"

    "github.com/fsnotify/fsnotify"
    proc "github.com/shirou/gopsutil/process"
)

const (
    nginxProcessName = "nginx"
    defaultNginxConfPath = "/etc/nginx"
    watchPathEnvVarName = "WATCH_NGINX_CONF_PATH"
)

var stderrLogger = log.New(os.Stderr, "error: ", log.Lshortfile)
var stdoutLogger = log.New(os.Stdout, "", log.Lshortfile)

func getMasterNginxPid() (int, error) {
    processes, processesErr := proc.Processes()
    if processesErr != nil {
        return 0, processesErr
    }

    nginxProcesses := map[int32]int32{}

    for _, process := range processes {
        processName, processNameErr := process.Name()
        if processNameErr != nil {
            return 0, processNameErr
        }

        if processName == nginxProcessName {
            ppid, ppidErr := process.Ppid()

            if ppidErr != nil {
                return 0, ppidErr
            }

            nginxProcesses[process.Pid] = ppid
        }
    }

    var masterNginxPid int32

    for pid, ppid := range nginxProcesses {
        if ppid == 0 {
            masterNginxPid = pid

            break
        }
    }

    stdoutLogger.Println("found master nginx pid:", masterNginxPid)

    return int(masterNginxPid), nil
}

func signalNginxReload(pid int) error {
    stdoutLogger.Printf("signaling master nginx process (pid: %d) -> SIGHUP\n", pid)
    nginxProcess, nginxProcessErr := os.FindProcess(pid)

    if nginxProcessErr != nil {
        return nginxProcessErr
    }

    return nginxProcess.Signal(syscall.SIGHUP)
}

func main() {
    watcher, watcherErr := fsnotify.NewWatcher()
    if watcherErr != nil {
        stderrLogger.Fatal(watcherErr)
    }
    defer watcher.Close()

    done := make(chan bool)
    go func() {
        for {
            select {
            case event, ok := <-watcher.Events:
                if !ok {
                    return
                }

                if event.Op&fsnotify.Create == fsnotify.Create {
                    if filepath.Base(event.Name) == "..data" {
                        stdoutLogger.Println("config map updated")

                        nginxPid, nginxPidErr := getMasterNginxPid()
                        if nginxPidErr != nil {
                            stderrLogger.Printf("getting master nginx pid failed: %s", nginxPidErr.Error())

                            continue
                        }

                        if err := signalNginxReload(nginxPid); err != nil {
                            stderrLogger.Printf("signaling master nginx process failed: %s", err)
                        }
                    }
                }
            case err, ok := <-watcher.Errors:
                if !ok {
                    return
                }
                stderrLogger.Printf("received watcher.Error: %s", err)
            }
        }
    }()

    pathToWatch, ok := os.LookupEnv(watchPathEnvVarName)
    if !ok {
        pathToWatch = defaultNginxConfPath
    }

    stdoutLogger.Printf("adding path: `%s` to watch\n", pathToWatch)

    if err := watcher.Add(pathToWatch); err != nil {
        stderrLogger.Fatal(err)
    }
    <-done
}

部署Nginx Pod

  • 创建nginx configmap
// nginx-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx
data:
  nginx-config: |
    user  nginx;
    worker_processes  1;

    error_log  /var/log/nginx/error.log warn;
    pid        /var/run/nginx.pid;

    events {
        worker_connections  1024;
    }

    http {
      server {
        server_name localhost;
        listen 80 default_server;

        location = /healthz {
          add_header Content-Type text/plain;
          return 200 'ok';
        }

        location / {
            root   /usr/share/nginx/html;
            index  index.html index.htm;
        }

        error_page   500 502 503 504  /50x.html;

        location = /50x.html {
            root   /usr/share/nginx/html;
        }
      }

      default_type  application/octet-stream;

      log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                        '$status $body_bytes_sent "$http_referer" '
                        '"$http_user_agent" "$http_x_forwarded_for"';

      access_log  /var/log/nginx/access.log  main;

      sendfile        on;
      #tcp_nopush     on;

      keepalive_timeout  65;

      #gzip  on;

      include /etc/nginx/conf.d/*.conf;
    }
kubectl apply -f nginx-config.yaml

需打开共享进程命名空间特性:shareProcessNamespace: true

//nginx-all-reloader.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      name: nginx
      labels:
        app: nginx
    spec:
      shareProcessNamespace: true
      volumes:
        - name: nginx
          configMap:
            name: nginx-config
      containers:
        - name: nginx
          image: nginx:latest
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          volumeMounts:
            - name: nginx-config
              mountPath: /etc/nginx/conf.d
              readOnly: true           
        - name: nginx-reloader
          image: nginx-reloader
          env:
            - name: WATCH_NGINX_CONF_PATH
              value: /etc/nginx/conf.d
          volumeMounts:
            - name: nginx-config
              mountPath: /etc/nginx/conf.d
              readOnly: true
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  selector:
    app: nginx
  ports:
    - name: http
      port: 80
    - name: https
      port: 443
​​‌‌​​​‌‌​‌​​‌‌‍​‌​‌‌‌​​‌‌‌‌​‌​‍​‌​​‌​​​‌​​​‌‌​‍​‌​‌‌​​​‌‌​​​​​‍​​‌​‌‌‌‌‌‌‌‌​​​‍​‌‌​​‌‌‌​‌‌​​‌‌‌‍​‌‌​​​‌‌‌​​​‌​‌‍​​‌‌‌‌‌‌‌‌​​‌‌‍‌​‌‌​‌​​‍‌‌​​​‌‌‌‍‌​​​‌‌​​‍​​​‌‌​​​‌​‌​​​​‍​‌​​‌‌‌​‌‌‌‌‌​​‍​‌‌​​​‌‌‌‌‌​‌​​‍‌​‌‌​​​‌‍‌​​‌‌​​​‍‌​​‌​‌‌​‍‌​​‌​​​‌‍‌​​​​‌‌‌‍​‌​​‌​​​‌​​​‌‌​‍​‌​‌​​‌‌​​‌​‌‌‌‍​​​‌​​‌​‌‌‌‌​‌‌‍​‌‌‌‌‌‌​​​​‌​‌​‌‍​‌​‌‌​‌​‌​‌​‌‌‌‍​​​‌‌‌‌​​​‌​​‌​‍​​‌‌​​‌​​​​‌​‌‌‍​​‌‌​‌​​‌​​‌‌‌‌‍​​‌‌​‌​​‌​​​‌‌​‍​​‌​‌‌‌‌​‌‌​‌‌‌‍​​‌‌‌‌‌‌‌‌​​‌​‍​​​​​​​​‌‌‌‌​​‌‌‍​​​‌​‌​‌‌​​‌‌‌​‍‌​​‌‌‌‌​‍‌​​‌‌​‌‌‍‌​​‌​​‌​‍‌​​‌​‌‌​‍‌​​‌​​​‌‍​‌‌​​​‌​‌‌‌​​​‌‍‌‌​​‌‌​‌‍‌‌​​‌‌‌‌‍‌‌​​‌‌‌​‍‌‌​​​‌‌​‍‌‌​‌​​‌​‍‌‌​​‌‌‌‌‍‌‌​​‌​​‌‍‌‌​‌​​‌​‍‌‌​​‌‌​‌‍‌‌​​​‌‌‌‍​‌​‌‌​‌‌‌‌​​‌​​‍​‌‌​​​​‌​‌​​​‌‌‍​​​​​​​​‌‌‌‌​​‌‌‍​‌​‌‌​​​‌‌​​​​​‍​​‌‌​‌​​‌‌‌‌​​​‍​‌​‌​​​‌‌​​‌‌‌‌‍​‌​‌​​​‌​‌‌‌‌‌‌‍​​​​​​​​‌‌‌​​‌​‌‍‌​​‌​‌‌‌‍‌​​​‌​‌‌‍‌​​​‌​‌‌‍‌​​​‌‌‌‌‍‌‌​​​‌​‌‍‌​‌​​​‌‌‍‌​‌​​​‌‌‍‌​​​‌​​​‍‌​​​‌​​​‍‌​​​‌​​​‍‌‌​‌​​​‌‍‌​​‌​‌‌​‍‌​​‌​‌​​‍‌​​‌​‌‌​‍‌​​​‌​​​‍‌​​‌​‌‌​‍‌‌​‌​​​‌‍‌​​‌​​‌​‍‌​​‌‌​‌​‍‌​‌​​​‌‌‍‌​​‌‌‌‌​‍‌​​​‌‌​‌‍‌​​‌‌‌​​‍‌​​‌​‌‌‌‍‌​​‌​‌‌​‍‌​​​‌​​‌‍‌​​‌‌​‌​‍‌​​​‌‌​​‍‌​‌​​​‌‌‍‌‌​​‌​​‌‍‌‌​​‌‌​​‍‌‌​​‌‌​​‍‌‌​‌​​​‌‍‌​​‌​‌‌‌‍‌​​​‌​‌‌‍‌​​‌​​‌​‍‌​​‌​​‌‌

部署应用:

kubectl apply -f nginx-all-reloader.yaml
  • 进入对应容器查看进程,可以发现,在两个容器下看到的结果是一致的,说明了这两个容器已经成功共享了进程命名空间。

nginx:

nginx-reloader:

实现效果

手动修改configmap后,reloader监测到configmap变化,会主动向nginx主进程发起HUP信号,实现配置热更新。

相关日志:

文章目录