Google Kubernetes Engine (GKE)へのcert-managerのデプロイと、Let's Encryptを使用したIngressへのSSL証明書の生成

gcloud init

Do you want to configure a default Compute Region and Zone? (Y/n)?  Y

* Commands that require authentication will use firstname.lastname@example.com by default
* Commands will reference project `your-project` by default
* Compute Engine commands will use region `europe-west1` by default
* Compute Engine commands will use zone `europe-west1-b` by default

export PROJECT=your-project  # Your Google Cloud project ID.
export REGION=europe-west1   # Your Google Cloud region.

export CLUSTER=test-cluster-1

gcloud container clusters create $CLUSTER --preemptible --num-nodes=1

gcloud components install gke-gcloud-auth-plugin
export USE_GKE_GCLOUD_AUTH_PLUGIN=True
gcloud container clusters get-credentials $CLUSTER

kubectl get nodes -o wide

kubectl create deployment web --image=gcr.io/google-samples/hello-app:1.0

kubectl expose deployment web --port=8080

gcloud compute addresses create web-ip --global

gcloud compute addresses list

gcloud compute addresses describe web-ip --format='value(address)' --global

export IP_ADDRESS=198.51.100.1  # Replace with your IP address

export DOMAIN_NAME=heyapp.net

# ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: web-ingress
  annotations:
    # This tells Google Cloud to create an External Load Balancer to realize this Ingress
    kubernetes.io/ingress.class: gce
    # This enables HTTP connections from Internet clients
    kubernetes.io/ingress.allow-http: "true"
    # This tells Google Cloud to associate the External Load Balancer with the static IP which we created earlier
    kubernetes.io/ingress.global-static-ip-name: web-ip
spec:
  defaultBackend:
    service:
      name: web
      port:
        number: 8080

kubectl apply -f ingress.yaml

kubectl describe ingress web-ingress

curl http://$DOMAIN_NAME

Hello, world!
Version: 1.0.0
Hostname: web-79d88c97d6-t8hj2

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml

kubectl -n cert-manager get all

kubectl explain Certificate
kubectl explain CertificateRequest
kubectl explain Issuer

# issuer-lets-encrypt-staging.yaml
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: letsencrypt-staging
spec:
  acme:
    server: https://acme-staging-v02.api.letsencrypt.org/directory
    email: <email-address> # ❗ Replace this with your email address
    privateKeySecretRef:
      name: letsencrypt-staging
    solvers:
    - http01:
        ingress:
          name: web-ingress

kubectl apply -f issuer-lets-encrypt-staging.yaml

kubectl describe issuers.cert-manager.io letsencrypt-staging

Status:
  Acme:
    Last Registered Email:  firstname.lastname@example.com
    Uri:                    https://acme-staging-v02.api.letsencrypt.org/acme/acct/60706744
  Conditions:
    Last Transition Time:  2022-07-13T16:13:25Z
    Message:               The ACME account was registered with the ACME server
    Observed Generation:   1
    Reason:                ACMEAccountRegistered
    Status:                True
    Type:                  Ready

# secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: web-ssl
type: kubernetes.io/tls
stringData:
  tls.key: ""
  tls.crt: ""

kubectl apply -f secret.yaml

--- a/ingress.yaml
+++ b/ingress.yaml
@@ -7,7 +7,12 @@ metadata:
     kubernetes.io/ingress.class: gce
     kubernetes.io/ingress.allow-http: "true"
     kubernetes.io/ingress.global-static-ip-name: web-ip
+    cert-manager.io/issuer: letsencrypt-staging
 spec:
+  tls:
+    - secretName: web-ssl
+      hosts:
+        - $DOMAIN_NAME
   defaultBackend:
     service:
       name: web

kubectl apply -f ingress.yaml

curl -v --insecure https://$DOMAIN_NAME

* Server certificate:
*  subject: CN=www.example.com
*  start date: Jul 14 08:52:29 2022 GMT
*  expire date: Oct 12 08:52:28 2022 GMT
*  issuer: C=US; O=(STAGING) Let's Encrypt; CN=(STAGING) Artificial Apricot R3
*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.

# issuer-lets-encrypt-production.yaml
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: letsencrypt-production
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: <email-address> # ❗ Replace this with your email address
    privateKeySecretRef:
      name: letsencrypt-production
    solvers:
    - http01:
        ingress:
          name: web-ingress

kubectl apply -f issuer-lets-encrypt-production.yaml

kubectl annotate ingress web-ingress cert-manager.io/issuer=letsencrypt-production --overwrite

curl -v https://$DOMAIN_NAME

...
* Server certificate:
*  subject: CN=www.example.com
*  start date: Jul 14 09:44:29 2022 GMT
*  expire date: Oct 12 09:44:28 2022 GMT
*  subjectAltName: host "www.example.com" matched cert's "www.example.com"
*  issuer: C=US; O=Let's Encrypt; CN=R3
*  SSL certificate verify ok.
...
Hello, world!
Version: 1.0.0
Hostname: web-79d88c97d6-t8hj2

# Delete the cluster and all the Google Cloud resources related to the Ingress that it contains
gcloud container clusters delete $CLUSTER

# Delete the domain name
gcloud dns record-sets delete $DOMAIN_NAME --zone $ZONE --type A

# Delete the static IP address
gcloud compute addresses delete web-ip --global

$ kubectl describe ingress web-ingress
Name:             web-ingress
Labels:           <none>
Namespace:        default
Address:          34.120.159.51
Ingress Class:    <none>
Default backend:  web:8080 (10.52.0.13:8080)
TLS:
  web-ssl terminates www.example.com
Rules:
  Host        Path  Backends
  ----        ----  --------
  *           *     web:8080 (10.52.0.13:8080)
Annotations:  cert-manager.io/issuer: letsencrypt-staging
              ingress.kubernetes.io/backends: {"k8s1-01784147-default-web-8080-1647ccd2":"HEALTHY"}
              ingress.kubernetes.io/forwarding-rule: k8s2-fr-1lt9dzcy-default-web-ingress-yteotwe4
              ingress.kubernetes.io/https-forwarding-rule: k8s2-fs-1lt9dzcy-default-web-ingress-yteotwe4
              ingress.kubernetes.io/https-target-proxy: k8s2-ts-1lt9dzcy-default-web-ingress-yteotwe4
              ingress.kubernetes.io/ssl-cert: k8s2-cr-1lt9dzcy-4gjeakdb9n7k6ls7-a257650b5fefd174
              ingress.kubernetes.io/target-proxy: k8s2-tp-1lt9dzcy-default-web-ingress-yteotwe4
              ingress.kubernetes.io/url-map: k8s2-um-1lt9dzcy-default-web-ingress-yteotwe4
              kubernetes.io/ingress.allow-http: true
              kubernetes.io/ingress.class: gce
              kubernetes.io/ingress.global-static-ip-name: web-ip
Events:
  Type     Reason             Age                 From                       Message
  ----     ------             ----                ----                       -------
  Normal   CreateCertificate  28m                 cert-manager-ingress-shim  Successfully created Certificate "web-ssl"
  Normal   Sync               28m                 loadbalancer-controller    UrlMap "k8s2-um-1lt9dzcy-default-web-ingress-yteotwe4" updated
  Warning  Sync               24m (x16 over 28m)  loadbalancer-controller    Error syncing to GCP: error running load balancer syncing routine: loadbalancer 1lt9dzcy-default-web-ingress-yteotwe4 does not exist: googleapi: Error 404: The resource 'projects/your-project/global/sslCertificates/k8s2-cr-1lt9dzcy-4gjeakdb9n7k6ls7-e3b0c44298fc1c14' was not found, notFound
  Normal   Sync               34s (x16 over 65m)  loadbalancer-controller    Scheduled for sync

$ cmctl status certificate web-ssl
Name: web-ssl
Namespace: default
Created at: 2022-07-14T17:30:06+01:00
Conditions:
  Ready: False, Reason: MissingData, Message: Issuing certificate as Secret does not contain a private key
  Issuing: True, Reason: MissingData, Message: Issuing certificate as Secret does not contain a private key
DNS Names:
- www.example.com
Events:
  Type    Reason     Age    From                                       Message
  ----    ------     ----   ----                                       -------
  Normal  Issuing    4m37s  cert-manager-certificates-trigger          Issuing certificate as Secret does not contain a private key
  Normal  Generated  4m37s  cert-manager-certificates-key-manager      Stored new private key in temporary Secret resource "web-ssl-8gsqc"
  Normal  Requested  4m37s  cert-manager-certificates-request-manager  Created new CertificateRequest resource "web-ssl-dblrj"
Issuer:
  Name: letsencrypt-staging
  Kind: Issuer
  Conditions:
    Ready: True, Reason: ACMEAccountRegistered, Message: The ACME account was registered with the ACME server
  Events:  <none>
error: 'tls.crt' of Secret "web-ssl" is not set
Not Before: <none>
Not After: <none>
Renewal Time: <none>
CertificateRequest:
  Name: web-ssl-dblrj
  Namespace: default
  Conditions:
    Approved: True, Reason: cert-manager.io, Message: Certificate request has been approved by cert-manager.io
  Ready: False, Reason: Pending, Message: Waiting on certificate issuance from order default/web-ssl-dblrj-327645514: "pending"
  Events:
    Type    Reason           Age    From                                          Message
    ----    ------           ----   ----                                          -------
    Normal  cert-manager.io  4m37s  cert-manager-certificaterequests-approver     Certificate request has been approved by cert-manager.io
    Normal  OrderCreated     4m37s  cert-manager-certificaterequests-issuer-acme  Created Order resource default/web-ssl-dblrj-327645514
    Normal  OrderPending     4m37s  cert-manager-certificaterequests-issuer-acme  Waiting on certificate issuance from order default/web-ssl-dblrj-327645514: ""
Order:
  Name: web-ssl-dblrj-327645514
  State: pending, Reason:
  Authorizations:
    URL: https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/3008789144, Identifier: www.example.com, Initial State: pending, Wildcard: false
Challenges:
- Name: web-ssl-dblrj-327645514-2671694319, Type: HTTP-01, Token: TKspp86xMjQzTvMVXWkezEA2sE2GSWjnld5Lt4X13ro, Key: TKspp86xMjQzTvMVXWkezEA2sE2GSWjnld5Lt4X13ro.f4bppCOm-jXasFGMKjpBE5aQlhiQBeTPIs0Lx822xao, State: pending, Reason: Waiting for HTTP-01 challenge propagation: did not get expected response when querying endpoint, expected "TKspp86xMjQzTvMVXWkezEA2sE2GSWjnld5Lt4X13ro.f4bppCOm-jXasFGMKjpBE5aQlhiQBeTPIs0Lx822xao" but got: Hello, world!
Version: 1... (truncated), Processing: true, Presented: true

$ cmctl status certificate web-ssl
Name: web-ssl
Namespace: default
Created at: 2022-07-14T17:30:06+01:00
Conditions:
  Ready: True, Reason: Ready, Message: Certificate is up to date and has not expired
DNS Names:
- www.example.com
Events:
  Type    Reason     Age   From                                       Message
  ----    ------     ----  ----                                       -------
  Normal  Issuing    31m   cert-manager-certificates-trigger          Issuing certificate as Secret does not contain a private key
  Normal  Generated  31m   cert-manager-certificates-key-manager      Stored new private key in temporary Secret resource "web-ssl-8gsqc"
  Normal  Requested  31m   cert-manager-certificates-request-manager  Created new CertificateRequest resource "web-ssl-dblrj"
  Normal  Issuing    26m   cert-manager-certificates-issuing          The certificate has been successfully issued
Issuer:
  Name: letsencrypt-staging
  Kind: Issuer
  Conditions:
    Ready: True, Reason: ACMEAccountRegistered, Message: The ACME account was registered with the ACME server
  Events:  <none>
Secret:
  Name: web-ssl
  Issuer Country: US
  Issuer Organisation: (STAGING) Let's Encrypt
  Issuer Common Name: (STAGING) Artificial Apricot R3
  Key Usage: Digital Signature, Key Encipherment
  Extended Key Usages: Server Authentication, Client Authentication
  Public Key Algorithm: RSA
  Signature Algorithm: SHA256-RSA
  Subject Key ID: a51e3621f5c1138947810f27dce425b33c88cb16
  Authority Key ID: de727a48df31c3a650df9f8523df57374b5d2e65
  Serial Number: fa8bb0b603ca2cdbfdfb2872d05ee52cda10
  Events:  <none>
Not Before: 2022-07-14T16:34:52+01:00
Not After: 2022-10-12T16:34:51+01:00
Renewal Time: 2022-09-12T16:34:51+01:00

$ gcloud compute ssl-certificates list
NAME                                                TYPE          CREATION_TIMESTAMP             EXPIRE_TIME                    MANAGED_STATUS
k8s2-cr-1lt9dzcy-4gjeakdb9n7k6ls7-a257650b5fefd174  SELF_MANAGED  2022-07-14T09:37:06.920-07:00  2022-10-12T08:34:51.000-07:00

$ gcloud compute ssl-certificates describe k8s2-cr-1lt9dzcy-4gjeakdb9n7k6ls7-a257650b5fefd174 --format='value(certificate)' \
  | openssl x509 -in - -noout -text
...
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number:
            04:9f:47:f1:cb:25:37:9b:86:a3:ef:bf:2e:77:3b:45:fc:1a
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: C = US, O = Let's Encrypt, CN = R3
        Validity
            Not Before: Jul 14 17:11:15 2022 GMT
            Not After : Oct 12 17:11:14 2022 GMT
        Subject: CN = www.example.com

$ gcloud compute forwarding-rules describe k8s2-fs-1lt9dzcy-default-web-ingress-yteotwe4 --global
IPAddress: 34.120.159.51
IPProtocol: TCP
creationTimestamp: '2022-07-14T09:37:12.362-07:00'
description: '{"kubernetes.io/ingress-name": "default/web-ingress"}'
fingerprint: oBTg7dRaIqI=
id: '2303318464959215831'
kind: compute#forwardingRule
labelFingerprint: 42WmSpB8rSM=
loadBalancingScheme: EXTERNAL
name: k8s2-fs-1lt9dzcy-default-web-ingress-yteotwe4
networkTier: PREMIUM
portRange: 443-443
selfLink: https://www.googleapis.com/compute/v1/projects/your-project/global/forwardingRules/k8s2-fs-1lt9dzcy-default-web-ingress-yteotwe4
target: https://www.googleapis.com/compute/v1/projects/your-project/global/targetHttpsProxies/k8s2-ts-1lt9dzcy-default-web-ingress-yteotwe4