Nginx cache for `/validate` response and HashiCorp Vault/Consul templates for both Vouch and Nginx config about vouch-proxy HOT 8 CLOSED

user  nginx;
worker_processes  3;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    proxy_http_version 1.1;

    proxy_cache_path /cache/validate levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;

    server {
        listen       80;
        server_name  k8s-dashboard.example.com;

        # send all requests to the `/validate` endpoint for authorization
        auth_request /validate;

        location = /validate {
          internal;

          proxy_cache_valid 200 30s;
          proxy_cache auth_cache;
          proxy_cache_methods GET;
          proxy_cache_key $cookie_vouchcookie;

          proxy_buffer_size 128k;
          proxy_buffers 4 256k;
          proxy_busy_buffers_size 256k;
          
          proxy_set_header Host k8s-dashboard.example.com;
          proxy_pass http://vouch-proxy;

          proxy_pass_request_body off;
          proxy_set_header Content-Length "";

          auth_request_set $auth_resp_x_vouch_user $upstream_http_x_vouch_user;
          auth_request_set $auth_resp_x_vouch_idtoken $upstream_http_x_vouch_idtoken;

          auth_request_set $auth_resp_jwt $upstream_http_x_vouch_jwt;
          auth_request_set $auth_resp_err $upstream_http_x_vouch_err;
          auth_request_set $auth_resp_failcount $upstream_http_x_vouch_failcount;
        }

        error_page 401 = @error401;

        location @error401 {
            # redirect to Vouch Proxy for login
            return 302 https://vouch.example.com/login?url=$scheme://$http_host$request_uri&vouch-failcount=$auth_resp_failcount&X-Vouch-Token=$auth_resp_jwt&error=$auth_resp_err;
        }
        
        location / {
            proxy_pass https://kubernetes-dashboard.kube-system.svc.cluster.local;
            proxy_ssl_verify off;
            auth_request_set $auth_resp_x_vouch_idtoken $upstream_http_x_vouch_idtoken;
            proxy_set_header Authorization "Bearer $auth_resp_x_vouch_idtoken";
        }
    }

    server {
        listen 80;
        server_name vouch.example.com;
        location / {
          proxy_set_header Host vouch.example.com;
          proxy_pass http://vouch-proxy;
        }
    }

    server {
        listen       8088;
        server_name  _;
        
        location /healthz {
            stub_status;
            access_log off;
            allow all;
        }
    }
}

apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: vouch-gateway
  namespace: vouch
  labels:
    app: common
spec:
  selector:
    istio: ingressgateway
  servers:
  - port:
      number: 443
      name: https
      protocol: HTTPS
    tls:
      mode: SIMPLE
      serverCertificate: /etc/istio/ingressgateway-certs/tls.crt
      privateKey: /etc/istio/ingressgateway-certs/tls.key
    hosts:
    - "*.example.com"
---
apiVersion: authentication.istio.io/v1alpha1
kind: Policy
metadata:
  name: default
  namespace: vouch
spec:
  peers:
  - mtls:
      mode: PERMISSIVE
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: vouch-proxy-configuration
  namespace: vouch
  labels:
    app: vouch-proxy
data:
  vault-agent-config.hcl: |
    exit_after_auth = true
    pid_file = "/home/vault/pidfile"

    auto_auth {
        method "kubernetes" {
            mount_path = "auth/kubernetes"
            config = {
                role = "vouch"
            }
        }

        sink "file" {
            config = {
                path = "/home/vault/.vault-token"
            }
        }
    }
  consul-template-config.hcl: |
    log_level = "debug"

    vault {
      renew_token = false
      retry {
        backoff = "1s"
      }
    }

    template {
      destination = "/config/config.yml"
      contents = <<EOH
      vouch:
        logLevel: warning
        listen: 0.0.0.0
        port: 80
        AllowAllUsers: true
        cookie: 
          name: VouchCookie
          domain: example.com
          secure: true
          httpOnly: true
        headers:
          jwt: X-Vouch-Token
          querystring: access_token
          redirect: X-Vouch-Requested-URI
          idpidtoken: x-vouch-idtoken
        session:
          name: VouchSession
        jwt:
      {{- with secret "secrets/data/dev/secrets/vouchProxy" }}
          secret: {{ .Data.data.jwtSecret }}
      {{ end }}
          maxAge: 59
      oauth:
        provider: adfs
      {{- with secret "secrets/data/dev/secrets/adfs" }}
        client_id: {{ .Data.data.oidcClientId }}
        client_secret: {{ .Data.data.oidcSharedKey }}
      {{ end }}
        auth_url: https://adfs.example.com/adfs/oauth2/authorize/
        token_url: https://adfs.example.com/adfs/oauth2/token/
        scopes:
          - openid
          - email
          - profile
        callback_url: https://vouch.example.com/auth
      EOH
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configuration
  namespace: vouch
  labels:
    app: nginx
data:
  nginx.conf: |
    user  nginx;
    worker_processes  3;

    error_log  /var/log/nginx/error.log warn;
    pid        /var/run/nginx.pid;


    events {
        worker_connections  1024;
    }

    http {
        include       /etc/nginx/mime.types;
        default_type  application/octet-stream;

        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                          '$status $body_bytes_sent "$http_referer" '
                          '"$http_user_agent" "$http_x_forwarded_for"';

        access_log  /var/log/nginx/access.log  main;
        sendfile        on;
        keepalive_timeout  65;
        proxy_http_version 1.1;

        proxy_cache_path /cache/validate levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;

        server {
            listen       80;
            server_name  kubernetes-dashboard.example.com;

            auth_request /validate;

            location = /validate {
              internal;

              proxy_cache_valid 200 30s;
              proxy_cache auth_cache;
              proxy_cache_methods GET;
              proxy_cache_key $cookie_vouchcookie;

              proxy_buffer_size 128k;
              proxy_buffers 4 256k;
              proxy_busy_buffers_size 256k;
              
              proxy_set_header Host kubernetes-dashboard.example.com;
              proxy_pass http://vouch-proxy;

              proxy_pass_request_body off;
              proxy_set_header Content-Length "";

              auth_request_set $auth_resp_x_vouch_user $upstream_http_x_vouch_user;
              auth_request_set $auth_resp_x_vouch_idtoken $upstream_http_x_vouch_idtoken;

              auth_request_set $auth_resp_jwt $upstream_http_x_vouch_jwt;
              auth_request_set $auth_resp_err $upstream_http_x_vouch_err;
              auth_request_set $auth_resp_failcount $upstream_http_x_vouch_failcount;
            }

            error_page 401 = @error401;

            location @error401 {
                return 302 https://vouch.example.com/login?url=$scheme://$http_host$request_uri&vouch-failcount=$auth_resp_failcount&X-Vouch-Token=$auth_resp_jwt&error=$auth_resp_err;
            }
            
            location / {
                proxy_pass https://kubernetes-dashboard.kube-system.svc.cluster.local;
                proxy_ssl_verify off;
                auth_request_set $auth_resp_x_vouch_idtoken $upstream_http_x_vouch_idtoken;
                proxy_set_header Authorization "Bearer $auth_resp_x_vouch_idtoken";
            }
        }

        server {
            listen 80;
            server_name vouch.example.com;
            location / {
              proxy_set_header Host vouch.example.com;
              proxy_pass http://vouch-proxy;
            }
        }

        server {
            listen       8088;
            server_name  _;
            
            location /healthz {
                stub_status;
                access_log off;
                allow all;
            }
        }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vouch-proxy
  namespace: vouch
  labels:
    app: vouch-proxy
spec:
  selector:
    matchLabels:
      app: vouch-proxy
  replicas: 1
  template:
    metadata:
      labels:
        app: vouch-proxy
    spec:
      serviceAccountName: sa-vouch
      initContainers:
      - name: vaultagent-auth
        image: vault
        volumeMounts:
          - name: vaultagent-token
            mountPath: /home/vault
          - name: vouch-proxy-configuration
            mountPath: /etc/vault/vault-agent-config.hcl
            subPath: vault-agent-config.hcl
        env:
          - name: VAULT_ADDR
            value: https://vault.example.com
        args:
          [
            "agent",
            "-config=/etc/vault/vault-agent-config.hcl"
          ]
      - name: consultemplate-confgen
        image: hashicorp/consul-template
        volumeMounts:
          - name: vaultagent-token
            mountPath: /home/vault
          - name: vouch-proxy-configuration
            mountPath: /etc/consul-template/consul-template-config.hcl
            subPath: consul-template-config.hcl
          - name: vouch-proxy-volume
            mountPath: /config
        env:
          - name: HOME
            value: /home/vault
          - name: VAULT_ADDR
            value: https://vault.example.com
          - name: HOME
            value: /home/vault
        args:
          [
            "-config=/etc/consul-template/consul-template-config.hcl",
            "-once"
          ]
      containers:
      - image: voucher/vouch-proxy:latest
        name: vouch-proxy
        ports:
        - containerPort: 80
        volumeMounts:
        - name: vouch-proxy-volume
          mountPath: /config
        - name: vouch-proxy-data
          mountPath: /data
      volumes:
      - name: vaultagent-token
        emptyDir:
          medium: Memory
      - name: vouch-proxy-configuration
        configMap:
          name: vouch-proxy-configuration
      - name: vouch-proxy-volume
        emptyDir:
          medium: Memory
      - name: vouch-proxy-data
        emptyDir:
          medium: Memory
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  namespace: vouch
  labels:
    app: nginx
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 1
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx:stable-alpine
        name: nginx
        ports:
        - containerPort: 80
        imagePullPolicy: Always
        resources: {}
        volumeMounts:
        - name: nginx-configuration
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf
        - name: nginx-cache
          mountPath: /cache
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8088
          initialDelaySeconds: 3
          periodSeconds: 3
        readinessProbe:
          httpGet:
            path: /healthz
            port: 8088
          initialDelaySeconds: 5
          periodSeconds: 5
      volumes:
      - name: nginx-configuration
        configMap:
          name: nginx-configuration
      - name: nginx-cache
        emptyDir:
          medium: Memory
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: vouch
  labels:
    app: nginx
spec:
  type: ClusterIP
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
  selector:
    app: nginx
---
apiVersion: v1
kind: Service
metadata:
  name: vouch-proxy
  namespace: vouch
  labels:
    app: vouch-proxy
spec:
  type: ClusterIP
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
  selector:
    app: vouch-proxy
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: nginx-vouch-proxy
  namespace: vouch
  labels:
    app: nginx
spec:
  podSelector:
    matchLabels:
      app: vouch-proxy
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: nginx
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: nginx-ingress
  namespace: vouch
  labels:
    app: nginx
spec:
  podSelector:
    matchLabels:
      app: nginx
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          name: istio-system
    - podSelector:
        matchLabels:
          istio: ingressgateway
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: nginx-vouch-proxy
  namespace: vouch
  labels:
    app: nginx
spec:
  hosts:
  - vouch.example.com
  gateways:
  - vouch-gateway
  http:
  - match:
    - uri:
        prefix: /
    route:
    - destination:
        port:
          number: 80
        host: nginx
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: nginx-dash
  namespace: vouch
  labels:
    app: nginx
spec:
  hosts:
  - kubernetes-dashboard.example.com
  gateways:
  - vouch-gateway
  http:
  - match:
    - uri:
        prefix: /
    route:
    - destination:
        port:
          number: 80
        host: nginx
---