From 0e58abfb290520bc4c7625c9fd979d317d2e17cb Mon Sep 17 00:00:00 2001 From: kirby Date: Thu, 22 May 2025 10:52:56 +0200 Subject: [PATCH] adding script dir, vault scripts --- scripts/convert-dotenv-to-json.py | 27 +++++++++ scripts/convert-json-to-dotenv.py | 18 ++++++ vault/scripts/create-app-approle.sh | 30 ++++++++++ vault/scripts/create-app-kubernetes.sh | 58 +++++++++++++++++++ vault/scripts/create-database-user.sh | 16 +++++ vault/scripts/policy-template.hcl | 4 ++ vault/scripts/vault-renew-token.sh | 25 ++++++++ vault/{ => scripts}/vault-snapshot-restore.sh | 0 vault/{ => scripts}/vault-snapshot.conf | 0 vault/scripts/vault-snapshot.sh | 52 +++++++++++++++++ 10 files changed, 230 insertions(+) create mode 100644 scripts/convert-dotenv-to-json.py create mode 100644 scripts/convert-json-to-dotenv.py create mode 100644 vault/scripts/create-app-approle.sh create mode 100644 vault/scripts/create-app-kubernetes.sh create mode 100644 vault/scripts/create-database-user.sh create mode 100644 vault/scripts/policy-template.hcl create mode 100644 vault/scripts/vault-renew-token.sh rename vault/{ => scripts}/vault-snapshot-restore.sh (100%) rename vault/{ => scripts}/vault-snapshot.conf (100%) create mode 100644 vault/scripts/vault-snapshot.sh diff --git a/scripts/convert-dotenv-to-json.py b/scripts/convert-dotenv-to-json.py new file mode 100644 index 0000000..58a5eae --- /dev/null +++ b/scripts/convert-dotenv-to-json.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import json +import sys + +def env_to_json(file_path): + # Open dotenv file + with open(file_path, 'r') as file: + # Store dotenv variables in a dict + data = {} + for line in file: + # Ignore comment and empty lines + if line.startswith('#') or not line.strip(): + continue + # Split key from value + key, value = line.strip().split("=", 1) + data[key] = value.replace('\'', '').replace('"', '') + + # Convert to json + json_data = json.dumps(data, indent=4) + + return json_data + +def main(): + print(env_to_json(sys.argv[1])) + +if __name__ == "__main__": + main() diff --git a/scripts/convert-json-to-dotenv.py b/scripts/convert-json-to-dotenv.py new file mode 100644 index 0000000..0ab4f23 --- /dev/null +++ b/scripts/convert-json-to-dotenv.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import json +import sys + +def json_to_env(json_string): + # Load json to dict + data = json.loads(json_string) + # Store value to string and print corresponding key + env_string = "" + for key, value in data.items(): + env_string += f'{key}={value}\n' + return env_string + +def main(): + print(json_to_env(sys.argv[1])) + +if __name__ == "__main__": + main() diff --git a/vault/scripts/create-app-approle.sh b/vault/scripts/create-app-approle.sh new file mode 100644 index 0000000..baf6100 --- /dev/null +++ b/vault/scripts/create-app-approle.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script create role and policy for app using the approle auth method. + +APP=$1 + +cp ./policy-template.hcl ./policy.hcl +sed -i "s|APPNAME|${APP}|g" ./policy.hcl + +# Creating stg policy +echo "###################################" +echo "Creation policy et app role de stg" +echo "###################################" +sed -i "s|ENV|stg|g" ./policy.hcl +/usr/bin/vault policy write "${APP}"-stg ./policy.hcl +/usr/bin/vault write auth/approle/role/${APP}-stg token_policies="${APP}-stg" +/usr/bin/vault read auth/approle/role/${APP}-stg/role-id +/usr/bin/vault write -f auth/approle/role/${APP}-stg/secret-id + +# Creating prd policy +echo "###################################" +echo "Creation policy et app role de prd" +echo "###################################" +sed -i "s|stg|prd|g" ./policy.hcl +/usr/bin/vault policy write "${APP}"-prd ./policy.hcl +/usr/bin/vault write auth/approle/role/${APP}-prd token_policies="${APP}-prd" +/usr/bin/vault read auth/approle/role/${APP}-prd/role-id +/usr/bin/vault write -f auth/approle/role/${APP}-prd/secret-id + +rm -f ./policy.hcl diff --git a/vault/scripts/create-app-kubernetes.sh b/vault/scripts/create-app-kubernetes.sh new file mode 100644 index 0000000..c1a12d3 --- /dev/null +++ b/vault/scripts/create-app-kubernetes.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# This script will create a role in each kubernetes auth method. +# It will also create a policy based on a template for each environment. + +set -eu + +# Getting app name and namespace from argument +APP=$1 +NAMESPACE=$2 + + +cp ./policy-template.hcl ./policy.hcl +sed -i "s|APPNAME|${APP}|g" ./policy.hcl + +# Creating tst policy +echo "###################################" +echo "Creation policy et app role de tst" +echo "###################################" +CURRENT_ENV="tst" +sed -i "s|ENV|tst|g" ./policy.hcl +/usr/bin/vault policy write "${APP}"-"${CURRENT_ENV}" ./policy.hcl +/usr/bin/vault write auth/testing/kubernetes/role/"${APP}" \ + bound_service_account_names="${APP}" \ + bound_service_account_namespaces="${NAMESPACE}" \ + alias_name_source="serviceaccount_uid" \ + token_no_default_policy=true \ + token_policies="${APP}""-""${CURRENT_ENV}" + +# Creating stg policy +echo "###################################" +echo "Creation policy et app role de stg" +echo "###################################" +CURRENT_ENV="stg" +sed -i "s|tst|stg|g" ./policy.hcl +/usr/bin/vault policy write "${APP}"-"${CURRENT_ENV}" ./policy.hcl +/usr/bin/vault write auth/staging/kubernetes/role/"${APP}" \ + bound_service_account_names="${APP}" \ + bound_service_account_namespaces="${NAMESPACE}" \ + alias_name_source="serviceaccount_uid" \ + token_no_default_policy=true \ + token_policies="${APP}""-""${CURRENT_ENV}" + +# Creating prd policy +echo "###################################" +echo "Creation policy et app role de prd" +echo "###################################" +CURRENT_ENV="prd" +sed -i "s|stg|prd|g" ./policy.hcl +/usr/bin/vault policy write "${APP}"-"${CURRENT_ENV}" ./policy.hcl +/usr/bin/vault write auth/production/kubernetes/role/"${APP}" \ + bound_service_account_names="${APP}" \ + bound_service_account_namespaces="${NAMESPACE}" \ + alias_name_source="serviceaccount_uid" \ + token_no_default_policy=true \ + token_policies="${APP}""-""${CURRENT_ENV}" + +rm -f ./policy.hcl diff --git a/vault/scripts/create-database-user.sh b/vault/scripts/create-database-user.sh new file mode 100644 index 0000000..be12629 --- /dev/null +++ b/vault/scripts/create-database-user.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Usage : ./create-database-user.sh my-api + +USERNAME=$1 +TYPE="mongodb" +PASSWORD=$(apg -a 1 -n 1 -m 24 -x 24 -M LN -E "\''azqwml1i0o") + +ENV="stg" +echo "Putting databases-users/${TYPE}/${ENV}/${USERNAME} with password: ${PASSWORD}" +/usr/bin/vault kv put databases-users/"${TYPE}"/"${ENV}"/"${USERNAME}" password="${PASSWORD}" username="${USERNAME}" + +PASSWORD=$(apg -a 1 -n 1 -m 24 -x 24 -M LN -E "\''azqwml1i0o") +ENV="prd" +echo "Putting databases-users/${TYPE}/${ENV}/${USERNAME} with password: ${PASSWORD}" +/usr/bin/vault kv put databases-users/"${TYPE}"/"${ENV}"/"${USERNAME}" password="${PASSWORD}" username="${USERNAME}" diff --git a/vault/scripts/policy-template.hcl b/vault/scripts/policy-template.hcl new file mode 100644 index 0000000..2839862 --- /dev/null +++ b/vault/scripts/policy-template.hcl @@ -0,0 +1,4 @@ +# In KV2 engine, we need to add /data/ to the path. +path "app/data/APPNAME/ENV/*" { + capabilities = ["read"] +} diff --git a/vault/scripts/vault-renew-token.sh b/vault/scripts/vault-renew-token.sh new file mode 100644 index 0000000..7d2c948 --- /dev/null +++ b/vault/scripts/vault-renew-token.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Script to refresh vault token used in CLI by a tool (rundeck here) + +RUNDECK_TOKEN_PATH="/var/lib/rundeck/.vault-token" +STATUS="0" +STATUSFILE=/var/tmp/batch.vault-renew-token.sh +export DBUS_SESSION_BUS_ADDRESS=/dev/null +export VAULT_ADDR="https://vault.example.com" +source /var/lib/rundeck/vault-renew-token.conf + +set -eu + +function set_error_status() { + echo "[$(date '+%Y%m%d %H%M%S')] : Something went wrong in the script, exiting." | tee -a "${LOGFILE}" + echo "2 vault-snapshot-restore - KO" > ${STATUSFILE} +} + +trap set_error_status ERR + +TOKEN=$(/usr/bin/vault write -field="token" auth/approle/login token_ttl="32d" role_id="${ROLEID}" secret_id="${SECRETID}") +echo "${TOKEN}" > "${RUNDECK_TOKEN_PATH}" + +echo "0 vault-renew-token - OK" > ${STATUSFILE} +exit "${STATUS}" diff --git a/vault/vault-snapshot-restore.sh b/vault/scripts/vault-snapshot-restore.sh similarity index 100% rename from vault/vault-snapshot-restore.sh rename to vault/scripts/vault-snapshot-restore.sh diff --git a/vault/vault-snapshot.conf b/vault/scripts/vault-snapshot.conf similarity index 100% rename from vault/vault-snapshot.conf rename to vault/scripts/vault-snapshot.conf diff --git a/vault/scripts/vault-snapshot.sh b/vault/scripts/vault-snapshot.sh new file mode 100644 index 0000000..029d98b --- /dev/null +++ b/vault/scripts/vault-snapshot.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +BACKUPDIR="/data/backups/vault" +CLASS="STANDARD" +BUCKET="" +ENDPOINT="" +LOGFILE="/data/log/scripts/vault-snapshot.sh" +DATE=$(date +"%Y%m%d") +STATUS="0" +STATUSFILE="/var/tmp/batch.vault-snapshot.sh" +STANDBY="true" + +export VAULT_SKIP_VERIFY="TRUE" + +source /root/.config/vault-snapshot.conf +set -eu + +# Function to handle error during the script. +function set_error_status() { + echo "[$(date '+%Y%m%d %H%M%S')] : Something went wrong in the script, exiting." | tee -a "${LOGFILE}" + echo "2 vault-snapshot-restore - KO" > ${STATUSFILE} +} + +trap set_error_status ERR + +cd "${BACKUPDIR}" || exit + +echo "${DATE} : Récupération du token" | tee -a "${LOGFILE}" +TOKEN=$(/usr/bin/vault write -field="token" auth/approle/login role_id="${ROLEID}" secret_id="${SECRETID}") +export VAULT_TOKEN="${TOKEN}" + +# Check if the node is the active one, if not we stop. +STANDBY=$(/usr/bin/vault read sys/health -format=json | jq '.data.standby') +if [ ! "${STANDBY}" == "false" ]; then + echo "${DATE} : Noeud en standby, on arrête le snapshot" | tee -a "${LOGFILE}" + echo "${DATE} : ###### FIN ######" | tee -a "${LOGFILE}" + echo "0 vault-snapshot - Standby node" > ${STATUSFILE} + exit 0 +fi + +echo "${DATE} : Lancement du snapshot" | tee -a "${LOGFILE}" +/usr/bin/vault operator raft snapshot save "${BACKUPDIR}"/vault-"${DATE}".snap |tee -a "${LOGFILE}" + +echo "${DATE} : Upload du snapshot sur S3" | tee -a "${LOGFILE}" +/usr/local/bin/aws --endpoint-url "${ENDPOINT}" s3 cp "${BACKUPDIR}"/vault-"${DATE}".snap s3://"${BUCKET}"/ --storage-class "${CLASS}" --only-show-errors |tee -a "${LOGFILE}" + +echo "${DATE} : Nettoyage des snapshots de +10 jours" | tee -a "${LOGFILE}" +/usr/bin/find ${BACKUPDIR} -name "*.snap" -mtime 10 -delete + +echo "0 vault-snapshot - OK" > ${STATUSFILE} +echo "${DATE} : ###### FIN ######" | tee -a "${LOGFILE}" +exit ${STATUS}