add mongodb role

ansible/roles/mongodb/README.md

@@ -0,0 +1,27 @@
+# Installation et configuration de mongoDB
+
+## Configuration
+
+La configuration se fait via le fichier ansible/group_vars/{{ nom_du_groupe }}.
+
+## Variables
+* mongodb_replicaset_name : Nom du replicaset configurés entre les serveurs. (Exemple: mongodb-stg)
+
+## Fonctionnalités
+
+* Installe les dépendances du rôle et de mongodb, le dépot MongoDB 6, les paquets mongodb.
+* Déploie les outils de backups.
+* Déploie la configuration relative à la supervision (check, fichier d'authentification et rôle custom).
+
+## Tags
+
+* install : installe mongodb, la supervision, les backups et les utilisateurs.
+* supervision : met à jour les éléments relatifs à la supervision (check, configuration, rôle custom).
+* backup: déploie les outils nécessaires aux sauvegardes (scripts, role, utilisateur, cron).
+
+## Modification de configuration
+
+* Mise à jour des éléments de supervision :
+```
+ansible-playbook -i hosts-stg playbooks/mongodb.yml -t supervision -l mongodb_stg
+```

ansible/roles/mongodb/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+
+mongodb_nfs_server_stg: ""
+mongodb_nfs_server_prd: ""
+mongodb_cmk_url: ""

ansible/roles/mongodb/files/disable-thp.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Disable Transparent Huge Pages (THP)
+DefaultDependencies=no
+After=sysinit.target local-fs.target
+[Service]
+Type=oneshot
+ExecStart=/bin/sh -c 'echo never | tee /sys/kernel/mm/transparent_hugepage/enabled > /dev/null'
+[Install]
+WantedBy=basic.target

ansible/roles/mongodb/files/local.conf

@@ -0,0 +1,2 @@
+vm.swappiness=1
+vm.max_map_count=128000

ansible/roles/mongodb/files/logrotate.conf

@@ -0,0 +1,13 @@
+/var/log/mongodb/mongod.log {
+  daily
+  rotate 7
+  missingok
+  compress
+  delaycompress
+  notifempty
+  create 640 mongodb mongodb
+  sharedscripts
+  postrotate
+    /bin/kill -SIGUSR1 `cat /var/run/mongodb/mongod.pid 2>/dev/null` >/dev/null 2>&1
+  endscript
+}

ansible/roles/mongodb/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+
+- name: Systemd daemon_reload
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+
+- name: Restart mongodb
+  ansible.builtin.systemd_service:
+    name: mongod
+    state: restarted
+
+- name: Restart pbm-agent
+  ansible.builtin.systemd_service:
+    name: pbm-agent
+    state: restarted

ansible/roles/mongodb/tasks/backup.yml

@@ -0,0 +1,124 @@
+---
+
+- name: Install dependencies
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - gnupg2
+    - lsb-release
+    - nfs-common
+  tags: install,backup
+
+- name: Ensure nas directory exists
+  ansible.builtin.file:
+    path: /nas
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  tags: install,backup
+
+- name: Create backup custom role
+  community.mongodb.mongodb_role:
+    login_user: "admin"
+    login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}"
+    replica_set: "{{ mongodb_replicaset_name }}"
+    database: "admin"
+    name: "pbmAnyAction"
+    privileges:
+      - resource:
+          db: ""
+          collection: ""
+        actions:
+          - "anyAction"
+    roles:
+      - role: "backup"
+        db: "admin"
+      - role: "clusterMonitor"
+        db: "admin"
+      - role: "restore"
+        db: "admin"
+      - role: "readWrite"
+        db: "admin"
+    state: present
+  tags: install,backup
+
+- name: Create backup user
+  community.mongodb.mongodb_user:
+    login_user: "admin"
+    login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}"
+    replica_set: "{{ mongodb_replicaset_name }}"
+    database: "admin"
+    name: "backup"
+    password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/backup:password') }}"
+    roles: "pbmAnyAction"
+    auth_mechanism: "SCRAM-SHA-256"
+    state: "present"
+    update_password: on_create
+  tags: install,backup
+
+- name: Add nas mounting to fstab
+  ansible.posix.mount:
+    src: "{{ mongodb_nfs_server_stg }}:/data/shares/mongodb"
+    path: "/nas"
+    fstype: "nfs4"
+    opts: "rw,noatime,nodiratime,_netdev"
+    state: present
+  when: dbenv = "stg"
+  tags: install,backup,nfs
+
+- name: Add nas mounting to fstab
+  ansible.posix.mount:
+    src: "{{ mongodb_nfs_server_prd }}:/data/shares/mongodb"
+    path: "/nas"
+    fstype: "nfs4"
+    opts: "rw,noatime,nodiratime,_netdev"
+    state: present
+  when: dbenv = "prd"
+  tags: install,backup,nfs
+
+- name: Ensure scripts directory exists
+  ansible.builtin.file:
+    path: /data/scripts
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  tags: install,backup
+
+- name: Deploy backup script
+  ansible.builtin.template:
+    src: mongodb-dump-full.sh.j2
+    dest: /data/scripts/mongodb-dump-full.sh
+    owner: root
+    group: root
+    mode: "0750"
+  tags: install,backup
+
+- name: Add cron to trigger backup
+  ansible.builtin.cron:
+    name: "mongodb-dump-full"
+    weekday: "*"
+    minute: "0"
+    hour: "02"
+    user: root
+    job: "/data/scripts/mongodb-dump-full.sh -r 14 -d /nas -c"
+    cron_file: mongodb-dump-full
+    disabled: true
+  tags: install,backup
+
+- name: Add MAILTO variable to cronfile
+  community.general.cronvar:
+    name: MAILTO
+    value: "''"
+    cron_file: mongodb-dump-full
+    state: present
+  tags: install,backup
+
+- name: Add check batch conf to checkmk
+  ansible.builtin.lineinfile:
+    path: /etc/check_mk/mrpe.cfg
+    line: "#script_mongodb-dump-databases.sh /usr/local/nagios/plugins/check_batch mongodb-dump-full.sh 129600"
+    state: present
+  tags: install,backup

ansible/roles/mongodb/tasks/install.yml

@@ -0,0 +1,128 @@
+---
+
+- name: Install requirements
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - gnupg
+    - python3-pip
+  tags: install,conf,users
+
+- name: Installing pymongo via pip
+  ansible.builtin.pip:
+    name:
+      - pymongo
+  tags: install,conf,users
+
+- name: Deploy service to disable THP at boot
+  ansible.builtin.copy:
+    src: disable-thp.service
+    dest: /etc/systemd/system/disable-thp.service
+    owner: root
+    group: root
+    mode: "0755"
+  notify: Systemd daemon_reload
+  tags: install
+
+- name: Enable disable-thp service
+  ansible.builtin.systemd:
+    name: disable-thp
+    enabled: true
+    masked: false
+  tags: install
+
+- name: Deploy sysctl conf (max_map_count, swappiness)
+  ansible.builtin.copy:
+    src: local.conf
+    dest: /etc/sysctl.d/local.conf
+    owner: root
+    group: root
+    mode: "0644"
+  tags: install,conf
+
+- name: Get mongodb.com gpg key
+  ansible.builtin.get_url:
+    url: https://pgp.mongodb.com/server-7.0.asc
+    dest: /usr/share/keyrings/mongodb-server-7.0.asc
+    owner: root
+    group: root
+    mode: "0644"
+  tags: install
+
+- name: Add mongodb.com repository
+  ansible.builtin.apt_repository:
+    repo: "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.asc] http://repo.mongodb.org/apt/debian bullseye/mongodb-org/7.0 main"
+    state: present
+  tags: install
+
+- name: Install mongodb
+  ansible.builtin.apt:
+    name: mongodb-org
+    state: present
+  tags: install
+
+- name: Holding mongodb packages
+  ansible.builtin.dpkg_selections:
+    name: "{{ item }}"
+    selection: hold
+  with_items:
+    - mongodb-org
+    - mongodb-org-database
+    - mongodb-org-server
+    - mongodb-mongosh
+    - mongodb-org-mongos
+    - mongodb-org-tools
+  tags: install
+
+- name: Ensure permissions are correct on /var/lib/mongodb
+  ansible.builtin.file:
+    path: /var/lib/mongodb
+    owner: mongodb
+    group: mongodb
+    mode: "0755"
+  tags: install
+
+- name: Start and enable mongodb service
+  ansible.builtin.systemd:
+    name: mongod
+    state: started
+    enabled: true
+  tags: install
+
+- name: Deploy conf file
+  ansible.builtin.template:
+    src: mongod.conf.j2
+    dest: /etc/mongod.conf
+    owner: root
+    group: root
+    mode: "0644"
+  tags: install,conf
+  notify: Restart mongodb
+
+- name: Deploy keyFile for auth in cluster
+  ansible.builtin.template:
+    src: mongo-keyfile.j2
+    dest: /etc/mongo-keyfile
+    owner: mongodb
+    group: mongodb
+    mode: "0400"
+  tags: install
+
+- name: Deploy logrotate conf file
+  ansible.builtin.copy:
+    src: logrotate.conf
+    dest: /etc/logrotate.d/mongodb
+    owner: root
+    group: root
+    mode: "0644"
+  tags: install
+
+- name: Create replicaset
+  community.mongodb.mongodb_replicaset:
+    login_user: "admin"
+    login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}"
+    login_host: localhost
+    replica_set: "{{ mongodb_replicaset_name }}"
+    members: "{{ mongodb_replicaset_members }}"
+  tags: install

ansible/roles/mongodb/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+
+- name: Include install tasks
+  ansible.builtin.include_tasks: install.yml
+  tags: install
+
+- name: Include supervision tasks
+  ansible.builtin.include_tasks: supervision.yml
+  tags: install,supervision
+
+- name: Include backup tasks
+  ansible.builtin.include_tasks: backup.yml
+  tags: install,backup

ansible/roles/mongodb/tasks/supervision.yml

@@ -0,0 +1,114 @@
+---
+
+- name: Deploy checkmk conf template
+  ansible.builtin.template:
+    src: mk_mongodb.cfg.j2
+    dest: /etc/check_mk/mk_mongodb.cfg
+    owner: root
+    group: root
+    mode: "0644"
+  tags: install
+
+- name: Deploy checkmk mongo check
+  ansible.builtin.get_url:
+    url: https://{{ mongodb_checkmk_url }}/check_mk/agents/plugins/mk_mongodb.py
+    dest: /usr/lib/check_mk_agent/plugins/
+    owner: root
+    group: root
+    mode: "0755"
+  tags: install
+
+- name: Deploy supervision role
+  community.mongodb.mongodb_role:
+    login_user: "admin"
+    login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}"
+    replica_set: "{{ mongodb_replicaset_name }}"
+    name: supervision
+    database: admin
+    privileges:
+      - resource:
+          db: ""
+          collection: "system.version"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.keys"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.roles"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.users"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.preimages"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.indexBuilds"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.rollback.id"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.views"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "system.replset"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "replset.initialSyncId"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "replset.election"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "replset.oplogTruncateAfterPoint"
+        actions:
+          - "collStats"
+      - resource:
+          db: ""
+          collection: "replset.minvalid"
+        actions:
+          - "collStats"
+    roles:
+      - role: "clusterMonitor"
+        db: "admin"
+      - role: "readAnyDatabase"
+        db: "admin"
+    state: present
+  tags: install,supervision
+
+- name: Create checkmk mongodb user
+  community.mongodb.mongodb_user:
+    login_user: "admin"
+    login_password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/admin:password') }}"
+    database: "admin"
+    name: "checkmk"
+    password: "{{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/checkmk:password') }}"
+    roles: "supervision"
+    auth_mechanism: "SCRAM-SHA-256"
+    replica_set: "{{ mongodb_replicaset_name }}"
+    state: "present"
+    update_password: on_create
+  tags: install,supervision

ansible/roles/mongodb/templates/mk_mongodb.cfg.j2

@@ -0,0 +1,10 @@
+[MONGODB]
+
+# all keys are optional
+host = 127.0.0.1
+# host defaults to localhost
+username = checkmk
+password = {{ lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/checkmk:password') }}
+auth_source = admin
+# auth_source defaults to admin
+auth_mechanism = DEFAULT

ansible/roles/mongodb/templates/mongo-keyfile.j2

@@ -0,0 +1 @@
+{{ lookup('hashi_vault','ansible/data/mongodb/{{ env }}/keyFile:key') }}

ansible/roles/mongodb/templates/mongod.conf.j2

@@ -0,0 +1,44 @@
+# mongod.conf
+
+# for documentation of all options, see:
+#   http://docs.mongodb.org/manual/reference/configuration-options/
+
+processManagement:
+  pidFilePath: /var/run/mongodb/mongod.pid
+
+# Where and how to store data.
+storage:
+  dbPath: /var/lib/mongodb
+#  engine:
+#  wiredTiger:
+
+# where to write logging data.
+systemLog:
+  destination: file
+  logAppend: true
+  logRotate: reopen
+  path: /var/log/mongodb/mongod.log
+
+# network interfaces
+net:
+  port: 27017
+  bindIp: 0.0.0.0
+
+
+# how the process runs
+processManagement:
+  timeZoneInfo: /usr/share/zoneinfo
+
+security:
+  keyFile: /etc/mongo-keyfile
+  authorization: enabled
+
+#operationProfiling:
+
+replication:
+  replSetName: {{ mongodb_replicaset_name }}
+
+#sharding:
+
+setParameter:
+    transactionLifetimeLimitSeconds: 3600

ansible/roles/mongodb/templates/mongodb-dump-full.sh.j2

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -eu
+
+DATE=$(date +%Y%m%d)
+HOSTNAME=$(hostname -s)
+STATUS=0
+LOGFILE="/var/tmp/mongodb-dump-databases.log"
+HOST="localhost"
+COMPRESS=false
+USER="backup"
+PASSWORD="{{lookup('community.hashi_vault.hashi_vault', 'ansible/data/mongodb/{{ env }}/users/backup:password') }}"
+DUMP_OPTIONS=""
+
+touch ${LOGFILE}
+
+#
+# Fonctions
+#
+
+checkNas()
+{
+    if [ ! -e "${BACKUPDIR}/.mount" ]; then
+        echo "${BACKUPDIR} not mounted. Backup aborted." | tee -a ${LOGFILE}
+        exit 1
+    fi
+}
+
+usage()
+{
+  echo "$0 -r <retention> -d <repertoire> -c (compression)"
+  echo "Exemple : /data/scripts/mongodb-dump-full.sh -r 20 -d /nas -c"
+}
+
+#
+# Main
+#
+
+while getopts "hcr:d:" option
+do
+  case "${option}"
+  in
+    r)
+      RETENTION=${OPTARG};;
+    d)
+      BACKUPDIR=${OPTARG};;
+    c)
+      COMPRESS=true;;
+    h | *)
+      usage
+      exit 1;;
+  esac
+done
+
+echo "Lancement du dump - Retention : ${RETENTION} - Repertoire : ${BACKUPDIR}" | tee -a ${LOGFILE}
+
+# check if the node is secondary
+SEC=$(mongosh --host=${HOST} --authenticationDatabase admin --username ${USER} --password ${PASSWORD} --eval 'rs.hello().secondary' --quiet)
+if [ ! "${SEC}" == "true" ]; then
+    echo "$(date +%s)|2|Node is not seconday ${LOGFILE}" > /var/tmp/batch."$(basename $0)"
+    exit 0
+fi
+
+[ -d "${BACKUPDIR}" ] || mkdir -p "${BACKUPDIR}"
+
+if [ "${COMPRESS}" ]; then
+    DUMP_OPTIONS="${DUMP_OPTIONS} --authenticationDatabase=admin --username=${USER} --password=${PASSWORD} --gzip"
+else
+    DUMP_OPTIONS="${DUMP_OPTIONS} --authenticationDatabase=admin --username=${USER} --password=${PASSWORD}"
+fi
+
+# dump
+mongodump -v --host=${HOST} ${DUMP_OPTIONS} --archive="${BACKUPDIR}/${DATE}-${HOSTNAME}.gz" |tee -a ${LOGFILE}
+STATUS=$?
+
+# output in statusfile for checkmk
+echo "$(date +%s)|${STATUS}|Check log file ${LOGFILE}" > /var/tmp/batch."$(basename "$0")"
+echo "Fin du dump - Retention : ${RETENTION} - Repertoire : ${BACKUPDIR}" | tee -a ${LOGFILE}