In this setup, we will integrate HashiCorp Vault within our Kubernetes cluster, utilizing a Google Cloud Storage (GCS) bucket as a storage backend for secure secret management. Vault will retrieve secrets stored in GCS and expose them as environment variables within our Kubernetes deployment YAML files, enabling secure and centralized secret handling for applications in the cluster.
- Configuring GCS Bucket for Vault Storage with Terraform
a. main.tf
# Enable required APIs
resource "google_project_service" "kms_api" {
project = "PROJECT-ID"
service = "cloudkms.googleapis.com"
disable_on_destroy = false
}
# GCS Bucket for Vault
resource "google_storage_bucket" "vault_bucket" {
name = "BUCKET-NAME"
location = "asia-south1"
force_destroy = true
storage_class = "STANDARD"
versioning {
enabled = true
}
}
# Service Account for Vault
resource "google_service_account" "vault_sa" {
project = "PROJECT-ID"
account_id = "vault-sa"
display_name = "Vault Service Account"
}
# Assign Storage Admin IAM Role to the Service Account for the GCS bucket
resource "google_storage_bucket_iam_member" "vault_sa_storage_admin" {
bucket = google_storage_bucket.vault_bucket.name
role = "roles/storage.admin"
member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# Assign Object Admin IAM Role to the Service Account for the GCS bucket
resource "google_storage_bucket_iam_member" "vault_sa_storage_object_admin" {
bucket = google_storage_bucket.vault_bucket.name
role = "roles/storage.objectAdmin"
member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# Generate a key for the Service Account
resource "google_service_account_key" "vault_sa_key" {
service_account_id = google_service_account.vault_sa.name
private_key_type = "TYPE_GOOGLE_CREDENTIALS_FILE"
key_algorithm = "KEY_ALG_RSA_2048"
}
# Output the private key (sensitive)
output "vault_sa_private_key" {
value = google_service_account_key.vault_sa_key.private_key
sensitive = true
}
# KMS Key Ring for Vault Encryption
resource "google_kms_key_ring" "my_key_ring" {
project = "PROJECT-ID"
name = "DUMMY-KEY-RING-NAME"
location = "asia-south1"
depends_on = [google_project_service.kms_api]
}
# KMS Crypto Key for Vault Encryption/Decryption
resource "google_kms_crypto_key" "my_crypto_key" {
name = "DUMMY-KEY-CRYPTO-NAME"
key_ring = google_kms_key_ring.my_key_ring.id
lifecycle {
prevent_destroy = true
}
rotation_period = "100000s"
purpose = "ENCRYPT_DECRYPT"
}
# IAM Role: Allow Vault Service Account to view KMS Key Ring
resource "google_project_iam_member" "kms_key_ring_viewer" {
project = "PROJECT-ID"
role = "roles/cloudkms.viewer"
member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# IAM Role: Allow Vault Service Account to administer KMS Key Ring
resource "google_project_iam_member" "kms_key_ring_admin" {
project = "PROJECT-ID"
role = "roles/cloudkms.admin"
member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# IAM Role: Allow Vault Service Account to encrypt and decrypt with KMS Crypto Key
resource "google_project_iam_member" "kms_crypto_key_user" {
project = "PROJECT-ID"
role = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
member = "serviceAccount:${google_service_account.vault_sa.email}"
}
# KMS crypto key IAM binding for encryption/decryption
resource "google_kms_crypto_key_iam_member" "crypto_key_iam" {
crypto_key_id = google_kms_crypto_key.my_crypto_key.id
role = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
member = "serviceAccount:${google_service_account.vault_sa.email}"
}
GCS for Vault storage using Terraform. It sets up a GCS bucket, a service account with necessary permissions, and a KMS key for Vault encryption.
b. provider.tf
provider "google" {
project = "PROJECT-ID"
region = "asia-south1"
}
c. Apply above HCL config’s
terraform init
terraform plan
terraform apply
terraform output vault_sa_private_key ## Keep this
output key safely
## (this will be used in your vault-credentials as credentials.json)
- Vault Setup Helm a. GCP Credentials Secret
# vault_credentials.yaml
apiVersion: v1
kind: Secret
metadata:
name: vault-credentials
namespace: vault
type: Opaque
data:
credentials.json: <This base64 encoded one from your terraform config>
#### NOTE: if you want OIDC enabled go for below one ###
client_id: <Id>
client_secret: <secret>
b. Vault Helm Values — (Which you will be overriding with your custom configs)
### vault_values.yaml
global:
enabled: true
tlsDisable: false
namespace: vault
server:
extraEnvironmentVars:
GOOGLE_REGION: <Put yours - 😂😂>
GOOGLE_PROJECT: <Your GCP Project - ofc i am not putting mine - 😂😂 >
GOOGLE_APPLICATION_CREDENTIALS: /vault/userconfig/vault-credentials/credentials.json
extraSecretMounts:
- name: cognito-client-details
secretName: vault-credentials
mountPath: /etc/secrets/vault-credentials
extraVolumes:
- type: "secret"
name: "vault-credentials"
ha:
enabled: true
replicas: 3
config: |
ui = true
listener "tcp" {
tls_disable = "false"
address = "[::]:8200"
cluster_address = "[::]:8201"
}
storage "gcs" {
bucket = "<Bucket Name>"
ha_enabled = true
}
seal "gcpckms" {
project = "<your gcp project name>"
region = "<Ofcourse the region>"
key_ring = "<the key ring name defined in your terraform setup>"
crypto_key = "<the crypto key name defined in your terraform setup>"
}
## Some colorful steps above haha!!
## If you want OIDC enable go for below steps, Please make sure you have ingress added
auth_config "oidc" {
type = "oidc"
description = "DUMMY-DESCRIPTION"
config = {
oidc_discovery_url = "<Your Discovery URL>"
oidc_client_id = "$__file{/etc/secrets/vault-credentials/client_id}"
oidc_client_secret = "$__file{/etc/secrets/vault-credentials/client_secret}"
default_role = "default"
bound_audiences = ["$__file{/etc/secrets/vault-credentials/client_id}"]
allowed_redirect_uris = [
"https://vault.<Whatever your domain name>/ui/vault/auth/oidc/oidc/callback",
"<http://localhost:8250/oidc/callback>"
]
claim_mappings = {
"cognito:groups" = "groups"
"email" = "email"
}
groups_claim = "cognito:groups"
scopes = ["email", "openid"]
}
}
service_registration "kubernetes" {}
service:
enabled: true
type: ClusterIP
port: 8200
targetPort: 8200
ui:
enabled: true
serviceType: LoadBalancer
ports:
- name: https
port: 443
targetPort: 8200
protocol: TCP
injector:
enabled: true
certs:
secretName: null
# caBundle is a base64-encoded PEM-encoded certificate bundle for the CA
# that signed the TLS certificate that the webhook serves. This must be set
# if secretName is non-null unless an external service like cert-manager is
# keeping the caBundle updated.
caBundle: "<Put the Ca Bundle in case you are having difference>"
c. Vault Unseal Process
d. MakeFile
SHELL=/bin/bash
SELF = $(MAKE)
TOPLEVEL = $(shell git rev-parse --show-toplevel)
include $(TOPLEVEL)/Makefile.helpers
export KUBECONFIG = $(HOME)/.kube/config
REPO_PATH ?= $(shell git rev-parse --show-toplevel)
APPLICATION_CONFIG_PATH = $(REPO_PATH)/MY-PATH-OF-APPLICATION
## Apply config and initialize Vault
apply_config:
kubectl --kubeconfig $(KUBECONFIG) create namespace NAMESPACE
kubectl --kubeconfig $(KUBECONFIG) apply -f $(APPLICATION_CONFIG_PATH)/vault/vault_gcp_credentials.yaml
helm install vault hashicorp/vault -f vault_values.yaml
sleep 30
$(SELF) vault_init
$(SELF) vault_auth_k8s
## Initialize Vault and unseal it
vault_init:
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- vault operator init > vault_init_output.txt ## Please save this file content somewhere also dont push to github Hehe !!
export UNSEAL_KEY_1=$$(grep 'Unseal Key 1:' vault_init_output.txt | awk '{print $$NF}') && \\
export UNSEAL_KEY_2=$$(grep 'Unseal Key 2:' vault_init_output.txt | awk '{print $$NF}') && \\
export UNSEAL_KEY_3=$$(grep 'Unseal Key 3:' vault_init_output.txt | awk '{print $$NF}') && \\
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- vault operator unseal $$UNSEAL_KEY_1 && \\
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-1 -- vault operator unseal $$UNSEAL_KEY_2 && \\
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-2 -- vault operator unseal $$UNSEAL_KEY_3
## Configure Vault for Kubernetes authentication and set policies/roles
vault_auth_k8s:
@read -p "Enter the Vault Token: " VAULT_TOKEN && \\
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault login $$VAULT_TOKEN"
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault auth enable kubernetes"
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault write auth/kubernetes/config \\
kubernetes_host=\\"https://\\$${KUBERNETES_PORT_443_TCP_ADDR}:443\\" \\
token_reviewer_jwt=\\"\\$$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\\" \\
kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \\
issuer=\\"<https://kubernetes.default.svc.cluster.local>\\""
kubectl --kubeconfig $(KUBECONFIG) exec -n vault vault-values-0 -- vault secrets enable -path=streaming_service kv-v2
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "echo 'path \\"stream_service/*\\" { capabilities = [\\"read\\"] }' | vault policy write vault-policy-streaming -"
kubectl --kubeconfig $(KUBECONFIG) exec -it -n vault vault-values-0 -- /bin/sh -c "vault write auth/kubernetes/role/vault-role-stream \\
bound_service_account_names=service-account-name \\
bound_service_account_namespaces=Namespace-Name \\
policies=vault-policy-stream \\
ttl=1h"
To Apply above makefile
make apply_config
d. Make File Sequence Diagram
To put data in your path you can either do post request, or use vault command, i prefer Curl
curl --location --request GET 'http://<Vault-Svc-IP>/v1/stream_service/data/dummy_service/config' \\
--header 'X-Vault-Token: <$Your_TOKEN>' \\
--header 'Content-Type: application/json' \\
--data '{
"data": {
"DUMMY_API_KEY": "<$PUT-YOURS-😂😂>",
}
}'
- Setup Agent Injector + Load Env Vars
a. Deployment.yaml
## Your-service-deployment.yaml
replicaCount: 1
fullnameOverride: "dummy-service"
# Source Your Config.txt to get env vars
command: ["/bin/bash"]
args: ['-c', 'source /vault/secrets/config.txt && /init'] ## /init to call your docker entrypoint
envVars: []
ports:
- name: application
containerPort: 9000
protocol: TCP
- name: metrics
containerPort: 9002
protocol: TCP
probes:
readiness:
enabled: true
initialDelaySeconds: 5
path: /api/v1/dummy/health
port: application
image:
repository: <Image> ## you can use busy box image for dummy purpose
pullPolicy: IfNotPresent
tag: "<Tag>"
imagePullSecrets:
- name: ecr-credentials
secret:
enabled: false
serviceAccount:
create: true
annotations: {}
name: ""
# Annotate your pod
podAnnotations:
vault.hashicorp.com/agent-inject: 'true'
vault.hashicorp.com/agent-inject-status: 'update'
vault.hashicorp.com/role: 'vault-role-stream' //The role we define earlier
vault.hashicorp.com/agent-inject-secret-config.txt: 'stream_service/data/dummy_service/config'
# The Path where we have store our secrets
*'stream_service/data/dummy_service/config' in here /data has to be put after our secrets initial path to so that vault can read it*
vault.hashicorp.com/agent-inject-template-config.txt: |
{{- with secret "stream_service/data/dummy_service/config" -}}
{{- range $key, $value := .Data.data }}
export {{ $key }}="{{ $value }}"
{{- end }}
{{- end -}}
service:
type: NodePort
ports:
- name: http
port: 80
targetPort: application
protocol: TCP
- name: metrics
port: 9002
targetPort: metrics
protocol: TCP
resources:
limits:
memory: 200Mi
cpu: 200m
requests:
memory: 200Mi
cpu: 200m
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 80
nodeSelector:
alpha.eksctl.io/nodegroup-name: "<Dummy-nodegroup>"
Check if secrets is being export or not in above config.txt
kubectl exec -it POD -n NAMESPACE -- cat /vault/secrets/config.txt
References
Top comments (0)