From 24465cb6f92977b80d8dd900185a973bec937ed3 Mon Sep 17 00:00:00 2001 From: kirby Date: Wed, 28 May 2025 11:23:06 +0200 Subject: [PATCH] add mongodb role --- ansible/roles/mongodb/README.md | 27 ++++ ansible/roles/mongodb/defaults/main.yml | 5 + .../roles/mongodb/files/disable-thp.service | 9 ++ ansible/roles/mongodb/files/local.conf | 2 + ansible/roles/mongodb/files/logrotate.conf | 13 ++ ansible/roles/mongodb/handlers/main.yml | 15 ++ ansible/roles/mongodb/tasks/backup.yml | 124 +++++++++++++++++ ansible/roles/mongodb/tasks/install.yml | 128 ++++++++++++++++++ ansible/roles/mongodb/tasks/main.yml | 13 ++ ansible/roles/mongodb/tasks/supervision.yml | 114 ++++++++++++++++ .../roles/mongodb/templates/mk_mongodb.cfg.j2 | 10 ++ .../roles/mongodb/templates/mongo-keyfile.j2 | 1 + .../roles/mongodb/templates/mongod.conf.j2 | 44 ++++++ .../mongodb/templates/mongodb-dump-full.sh.j2 | 78 +++++++++++ 14 files changed, 583 insertions(+) create mode 100644 ansible/roles/mongodb/README.md create mode 100644 ansible/roles/mongodb/defaults/main.yml create mode 100644 ansible/roles/mongodb/files/disable-thp.service create mode 100644 ansible/roles/mongodb/files/local.conf create mode 100644 ansible/roles/mongodb/files/logrotate.conf create mode 100644 ansible/roles/mongodb/handlers/main.yml create mode 100644 ansible/roles/mongodb/tasks/backup.yml create mode 100644 ansible/roles/mongodb/tasks/install.yml create mode 100644 ansible/roles/mongodb/tasks/main.yml create mode 100644 ansible/roles/mongodb/tasks/supervision.yml create mode 100644 ansible/roles/mongodb/templates/mk_mongodb.cfg.j2 create mode 100644 ansible/roles/mongodb/templates/mongo-keyfile.j2 create mode 100644 ansible/roles/mongodb/templates/mongod.conf.j2 create mode 100755 ansible/roles/mongodb/templates/mongodb-dump-full.sh.j2 diff --git a/ansible/roles/mongodb/README.md b/ansible/roles/mongodb/README.md new file mode 100644 index 0000000..6287524 --- /dev/null +++ b/ansible/roles/mongodb/README.md @@ -0,0 +1,27 @@ +# Installation et configuration de mongoDB + +## Configuration + +La configuration se fait via le fichier ansible/group_vars/{{ nom_du_groupe }}. + +## Variables +* mongodb_replicaset_name : Nom du replicaset configurés entre les serveurs. (Exemple: mongodb-stg) + +## Fonctionnalités + +* Installe les dépendances du rôle et de mongodb, le dépot MongoDB 6, les paquets mongodb. +* Déploie les outils de backups. +* Déploie la configuration relative à la supervision (check, fichier d'authentification et rôle custom). + +## Tags + +* install : installe mongodb, la supervision, les backups et les utilisateurs. +* supervision : met à jour les éléments relatifs à la supervision (check, configuration, rôle custom). +* backup: déploie les outils nécessaires aux sauvegardes (scripts, role, utilisateur, cron). + +## Modification de configuration + +* Mise à jour des éléments de supervision : +``` +ansible-playbook -i hosts-stg playbooks/mongodb.yml -t supervision -l mongodb_stg +``` diff --git a/ansible/roles/mongodb/defaults/main.yml b/ansible/roles/mongodb/defaults/main.yml new file mode 100644 index 0000000..0d91192 --- /dev/null +++ b/ansible/roles/mongodb/defaults/main.yml @@ -0,0 +1,5 @@ +--- + +mongodb_nfs_server_stg: "" +mongodb_nfs_server_prd: "" +mongodb_cmk_url: "" diff --git a/ansible/roles/mongodb/files/disable-thp.service b/ansible/roles/mongodb/files/disable-thp.service new file mode 100644 index 0000000..c943137 --- /dev/null +++ b/ansible/roles/mongodb/files/disable-thp.service @@ -0,0 +1,9 @@ +[Unit] +Description=Disable Transparent Huge Pages (THP) +DefaultDependencies=no +After=sysinit.target local-fs.target +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'echo never | tee /sys/kernel/mm/transparent_hugepage/enabled > /dev/null' +[Install] +WantedBy=basic.target diff --git a/ansible/roles/mongodb/files/local.conf b/ansible/roles/mongodb/files/local.conf new file mode 100644 index 0000000..7426d6e --- /dev/null +++ b/ansible/roles/mongodb/files/local.conf @@ -0,0 +1,2 @@ +vm.swappiness=1 +vm.max_map_count=128000 diff --git a/ansible/roles/mongodb/files/logrotate.conf b/ansible/roles/mongodb/files/logrotate.conf new file mode 100644 index 0000000..409ec6d --- /dev/null +++ b/ansible/roles/mongodb/files/logrotate.conf @@ -0,0 +1,13 @@ +/var/log/mongodb/mongod.log { + daily + rotate 7 + missingok + compress + delaycompress + notifempty + create 640 mongodb mongodb + sharedscripts + postrotate + /bin/kill -SIGUSR1 `cat /var/run/mongodb/mongod.pid 2>/dev/null` >/dev/null 2>&1 + endscript +} diff --git a/ansible/roles/mongodb/handlers/main.yml b/ansible/roles/mongodb/handlers/main.yml new file mode 100644 index 0000000..d9a45c8 --- /dev/null +++ b/ansible/roles/mongodb/handlers/main.yml @@ -0,0 +1,15 @@ +--- + +- name: Systemd daemon_reload + ansible.builtin.systemd_service: + daemon_reload: true + +- name: Restart mongodb + ansible.builtin.systemd_service: + name: mongod + state: restarted + +- name: Restart pbm-agent + ansible.builtin.systemd_service: + name: pbm-agent + state: restarted diff --git a/ansible/roles/mongodb/tasks/backup.yml b/ansible/roles/mongodb/tasks/backup.yml new file mode 100644 index 0000000..da8a4d8 --- /dev/null +++ b/ansible/roles/mongodb/tasks/backup.yml @@ -0,0 +1,124 @@ +--- + +- name: Install dependencies + ansible.builtin.apt: + name: "{{ item }}" + state: present + with_items: + - gnupg2 + - lsb-release + - nfs-common + tags: install,backup + +- name: Ensure nas directory exists + ansible.builtin.file: + path: /nas + state: directory + owner: root + group: root + mode: "0755" + tags: install,backup + +- name: Create backup custom role + community.mongodb.mongodb_role: + login_user: "admin" + login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}" + replica_set: "{{ mongodb_replicaset_name }}" + database: "admin" + name: "pbmAnyAction" + privileges: + - resource: + db: "" + collection: "" + actions: + - "anyAction" + roles: + - role: "backup" + db: "admin" + - role: "clusterMonitor" + db: "admin" + - role: "restore" + db: "admin" + - role: "readWrite" + db: "admin" + state: present + tags: install,backup + +- name: Create backup user + community.mongodb.mongodb_user: + login_user: "admin" + login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}" + replica_set: "{{ mongodb_replicaset_name }}" + database: "admin" + name: "backup" + password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/backup:password') }}" + roles: "pbmAnyAction" + auth_mechanism: "SCRAM-SHA-256" + state: "present" + update_password: on_create + tags: install,backup + +- name: Add nas mounting to fstab + ansible.posix.mount: + src: "{{ mongodb_nfs_server_stg }}:/data/shares/mongodb" + path: "/nas" + fstype: "nfs4" + opts: "rw,noatime,nodiratime,_netdev" + state: present + when: dbenv = "stg" + tags: install,backup,nfs + +- name: Add nas mounting to fstab + ansible.posix.mount: + src: "{{ mongodb_nfs_server_prd }}:/data/shares/mongodb" + path: "/nas" + fstype: "nfs4" + opts: "rw,noatime,nodiratime,_netdev" + state: present + when: dbenv = "prd" + tags: install,backup,nfs + +- name: Ensure scripts directory exists + ansible.builtin.file: + path: /data/scripts + state: directory + owner: root + group: root + mode: "0755" + tags: install,backup + +- name: Deploy backup script + ansible.builtin.template: + src: mongodb-dump-full.sh.j2 + dest: /data/scripts/mongodb-dump-full.sh + owner: root + group: root + mode: "0750" + tags: install,backup + +- name: Add cron to trigger backup + ansible.builtin.cron: + name: "mongodb-dump-full" + weekday: "*" + minute: "0" + hour: "02" + user: root + job: "/data/scripts/mongodb-dump-full.sh -r 14 -d /nas -c" + cron_file: mongodb-dump-full + disabled: true + tags: install,backup + +- name: Add MAILTO variable to cronfile + community.general.cronvar: + name: MAILTO + value: "''" + cron_file: mongodb-dump-full + state: present + tags: install,backup + +- name: Add check batch conf to checkmk + ansible.builtin.lineinfile: + path: /etc/check_mk/mrpe.cfg + line: "#script_mongodb-dump-databases.sh /usr/local/nagios/plugins/check_batch mongodb-dump-full.sh 129600" + state: present + tags: install,backup diff --git a/ansible/roles/mongodb/tasks/install.yml b/ansible/roles/mongodb/tasks/install.yml new file mode 100644 index 0000000..8c99de5 --- /dev/null +++ b/ansible/roles/mongodb/tasks/install.yml @@ -0,0 +1,128 @@ +--- + +- name: Install requirements + ansible.builtin.apt: + name: "{{ item }}" + state: present + with_items: + - gnupg + - python3-pip + tags: install,conf,users + +- name: Installing pymongo via pip + ansible.builtin.pip: + name: + - pymongo + tags: install,conf,users + +- name: Deploy service to disable THP at boot + ansible.builtin.copy: + src: disable-thp.service + dest: /etc/systemd/system/disable-thp.service + owner: root + group: root + mode: "0755" + notify: Systemd daemon_reload + tags: install + +- name: Enable disable-thp service + ansible.builtin.systemd: + name: disable-thp + enabled: true + masked: false + tags: install + +- name: Deploy sysctl conf (max_map_count, swappiness) + ansible.builtin.copy: + src: local.conf + dest: /etc/sysctl.d/local.conf + owner: root + group: root + mode: "0644" + tags: install,conf + +- name: Get mongodb.com gpg key + ansible.builtin.get_url: + url: https://pgp.mongodb.com/server-7.0.asc + dest: /usr/share/keyrings/mongodb-server-7.0.asc + owner: root + group: root + mode: "0644" + tags: install + +- name: Add mongodb.com repository + ansible.builtin.apt_repository: + repo: "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.asc] http://repo.mongodb.org/apt/debian bullseye/mongodb-org/7.0 main" + state: present + tags: install + +- name: Install mongodb + ansible.builtin.apt: + name: mongodb-org + state: present + tags: install + +- name: Holding mongodb packages + ansible.builtin.dpkg_selections: + name: "{{ item }}" + selection: hold + with_items: + - mongodb-org + - mongodb-org-database + - mongodb-org-server + - mongodb-mongosh + - mongodb-org-mongos + - mongodb-org-tools + tags: install + +- name: Ensure permissions are correct on /var/lib/mongodb + ansible.builtin.file: + path: /var/lib/mongodb + owner: mongodb + group: mongodb + mode: "0755" + tags: install + +- name: Start and enable mongodb service + ansible.builtin.systemd: + name: mongod + state: started + enabled: true + tags: install + +- name: Deploy conf file + ansible.builtin.template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + owner: root + group: root + mode: "0644" + tags: install,conf + notify: Restart mongodb + +- name: Deploy keyFile for auth in cluster + ansible.builtin.template: + src: mongo-keyfile.j2 + dest: /etc/mongo-keyfile + owner: mongodb + group: mongodb + mode: "0400" + tags: install + +- name: Deploy logrotate conf file + ansible.builtin.copy: + src: logrotate.conf + dest: /etc/logrotate.d/mongodb + owner: root + group: root + mode: "0644" + tags: install + +- name: Create replicaset + community.mongodb.mongodb_replicaset: + login_user: "admin" + login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}" + login_host: localhost + replica_set: "{{ mongodb_replicaset_name }}" + members: "{{ mongodb_replicaset_members }}" + tags: install diff --git a/ansible/roles/mongodb/tasks/main.yml b/ansible/roles/mongodb/tasks/main.yml new file mode 100644 index 0000000..2e969cf --- /dev/null +++ b/ansible/roles/mongodb/tasks/main.yml @@ -0,0 +1,13 @@ +--- + +- name: Include install tasks + ansible.builtin.include_tasks: install.yml + tags: install + +- name: Include supervision tasks + ansible.builtin.include_tasks: supervision.yml + tags: install,supervision + +- name: Include backup tasks + ansible.builtin.include_tasks: backup.yml + tags: install,backup diff --git a/ansible/roles/mongodb/tasks/supervision.yml b/ansible/roles/mongodb/tasks/supervision.yml new file mode 100644 index 0000000..56eee66 --- /dev/null +++ b/ansible/roles/mongodb/tasks/supervision.yml @@ -0,0 +1,114 @@ +--- + +- name: Deploy checkmk conf template + ansible.builtin.template: + src: mk_mongodb.cfg.j2 + dest: /etc/check_mk/mk_mongodb.cfg + owner: root + group: root + mode: "0644" + tags: install + +- name: Deploy checkmk mongo check + ansible.builtin.get_url: + url: https://{{ mongodb_checkmk_url }}/check_mk/agents/plugins/mk_mongodb.py + dest: /usr/lib/check_mk_agent/plugins/ + owner: root + group: root + mode: "0755" + tags: install + +- name: Deploy supervision role + community.mongodb.mongodb_role: + login_user: "admin" + login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}" + replica_set: "{{ mongodb_replicaset_name }}" + name: supervision + database: admin + privileges: + - resource: + db: "" + collection: "system.version" + actions: + - "collStats" + - resource: + db: "" + collection: "system.keys" + actions: + - "collStats" + - resource: + db: "" + collection: "system.roles" + actions: + - "collStats" + - resource: + db: "" + collection: "system.users" + actions: + - "collStats" + - resource: + db: "" + collection: "system.preimages" + actions: + - "collStats" + - resource: + db: "" + collection: "system.indexBuilds" + actions: + - "collStats" + - resource: + db: "" + collection: "system.rollback.id" + actions: + - "collStats" + - resource: + db: "" + collection: "system.views" + actions: + - "collStats" + - resource: + db: "" + collection: "system.replset" + actions: + - "collStats" + - resource: + db: "" + collection: "replset.initialSyncId" + actions: + - "collStats" + - resource: + db: "" + collection: "replset.election" + actions: + - "collStats" + - resource: + db: "" + collection: "replset.oplogTruncateAfterPoint" + actions: + - "collStats" + - resource: + db: "" + collection: "replset.minvalid" + actions: + - "collStats" + roles: + - role: "clusterMonitor" + db: "admin" + - role: "readAnyDatabase" + db: "admin" + state: present + tags: install,supervision + +- name: Create checkmk mongodb user + community.mongodb.mongodb_user: + login_user: "admin" + login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}" + database: "admin" + name: "checkmk" + password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/checkmk:password') }}" + roles: "supervision" + auth_mechanism: "SCRAM-SHA-256" + replica_set: "{{ mongodb_replicaset_name }}" + state: "present" + update_password: on_create + tags: install,supervision diff --git a/ansible/roles/mongodb/templates/mk_mongodb.cfg.j2 b/ansible/roles/mongodb/templates/mk_mongodb.cfg.j2 new file mode 100644 index 0000000..c5d0eda --- /dev/null +++ b/ansible/roles/mongodb/templates/mk_mongodb.cfg.j2 @@ -0,0 +1,10 @@ +[MONGODB] + +# all keys are optional +host = 127.0.0.1 +# host defaults to localhost +username = checkmk +password = {{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/checkmk:password') }} +auth_source = admin +# auth_source defaults to admin +auth_mechanism = DEFAULT diff --git a/ansible/roles/mongodb/templates/mongo-keyfile.j2 b/ansible/roles/mongodb/templates/mongo-keyfile.j2 new file mode 100644 index 0000000..2c3d2fc --- /dev/null +++ b/ansible/roles/mongodb/templates/mongo-keyfile.j2 @@ -0,0 +1 @@ +{{ lookup('hashi_vault','ansible/data/mongodb/{{ env }}/keyFile:key') }} diff --git a/ansible/roles/mongodb/templates/mongod.conf.j2 b/ansible/roles/mongodb/templates/mongod.conf.j2 new file mode 100644 index 0000000..3528467 --- /dev/null +++ b/ansible/roles/mongodb/templates/mongod.conf.j2 @@ -0,0 +1,44 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +processManagement: + pidFilePath: /var/run/mongodb/mongod.pid + +# Where and how to store data. +storage: + dbPath: /var/lib/mongodb +# engine: +# wiredTiger: + +# where to write logging data. +systemLog: + destination: file + logAppend: true + logRotate: reopen + path: /var/log/mongodb/mongod.log + +# network interfaces +net: + port: 27017 + bindIp: 0.0.0.0 + + +# how the process runs +processManagement: + timeZoneInfo: /usr/share/zoneinfo + +security: + keyFile: /etc/mongo-keyfile + authorization: enabled + +#operationProfiling: + +replication: + replSetName: {{ mongodb_replicaset_name }} + +#sharding: + +setParameter: + transactionLifetimeLimitSeconds: 3600 diff --git a/ansible/roles/mongodb/templates/mongodb-dump-full.sh.j2 b/ansible/roles/mongodb/templates/mongodb-dump-full.sh.j2 new file mode 100755 index 0000000..ba3378a --- /dev/null +++ b/ansible/roles/mongodb/templates/mongodb-dump-full.sh.j2 @@ -0,0 +1,78 @@ +#!/bin/bash + +set -eu + +DATE=$(date +%Y%m%d) +HOSTNAME=$(hostname -s) +STATUS=0 +LOGFILE="/var/tmp/mongodb-dump-databases.log" +HOST="localhost" +COMPRESS=false +USER="backup" +PASSWORD="{{lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/backup:password') }}" +DUMP_OPTIONS="" + +touch ${LOGFILE} + +# +# Fonctions +# + +checkNas() +{ + if [ ! -e "${BACKUPDIR}/.mount" ]; then + echo "${BACKUPDIR} not mounted. Backup aborted." | tee -a ${LOGFILE} + exit 1 + fi +} + +usage() +{ + echo "$0 -r -d -c (compression)" + echo "Exemple : /data/scripts/mongodb-dump-full.sh -r 20 -d /nas -c" +} + +# +# Main +# + +while getopts "hcr:d:" option +do + case "${option}" + in + r) + RETENTION=${OPTARG};; + d) + BACKUPDIR=${OPTARG};; + c) + COMPRESS=true;; + h | *) + usage + exit 1;; + esac +done + +echo "Lancement du dump - Retention : ${RETENTION} - Repertoire : ${BACKUPDIR}" | tee -a ${LOGFILE} + +# check if the node is secondary +SEC=$(mongosh --host=${HOST} --authenticationDatabase admin --username ${USER} --password ${PASSWORD} --eval 'rs.hello().secondary' --quiet) +if [ ! "${SEC}" == "true" ]; then + echo "$(date +%s)|2|Node is not seconday ${LOGFILE}" > /var/tmp/batch."$(basename $0)" + exit 0 +fi + +[ -d "${BACKUPDIR}" ] || mkdir -p "${BACKUPDIR}" + +if [ "${COMPRESS}" ]; then + DUMP_OPTIONS="${DUMP_OPTIONS} --authenticationDatabase=admin --username=${USER} --password=${PASSWORD} --gzip" +else + DUMP_OPTIONS="${DUMP_OPTIONS} --authenticationDatabase=admin --username=${USER} --password=${PASSWORD}" +fi + +# dump +mongodump -v --host=${HOST} ${DUMP_OPTIONS} --archive="${BACKUPDIR}/${DATE}-${HOSTNAME}.gz" |tee -a ${LOGFILE} +STATUS=$? + +# output in statusfile for checkmk +echo "$(date +%s)|${STATUS}|Check log file ${LOGFILE}" > /var/tmp/batch."$(basename "$0")" +echo "Fin du dump - Retention : ${RETENTION} - Repertoire : ${BACKUPDIR}" | tee -a ${LOGFILE}