From 50cc047161f894943ea521ac127b4208670f7e21 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 19 Jul 2022 17:09:33 -0500 Subject: [PATCH 01/11] imp: add an includeReplica option for snapshots --- modules/consul-snapshots.nix | 12 ++++++++++++ modules/nomad-snapshots.nix | 12 ++++++++++++ modules/vault-snapshots.nix | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix index f6effb65..b0e4d483 100644 --- a/modules/consul-snapshots.nix +++ b/modules/consul-snapshots.nix @@ -85,6 +85,17 @@ ''; }; + includeReplica = lib.mkOption { + type = with lib.types; bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; + interval = lib.mkOption { type = with lib.types; addCheck str (x: x != ""); default = null; @@ -143,6 +154,7 @@ BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader; + INCLUDE_REPLICA = lib.boolToString cfg.${job}.includeReplica; CONSUL_HTTP_ADDR = cfg.${job}.consulAddress; }; diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix index f5288737..1cac97a0 100644 --- a/modules/nomad-snapshots.nix +++ b/modules/nomad-snapshots.nix @@ -78,6 +78,17 @@ ''; }; + includeReplica = lib.mkOption { + type = with lib.types; bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; + interval = lib.mkOption { type = with lib.types; addCheck str (x: x != ""); default = null; @@ -144,6 +155,7 @@ BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader; + INCLUDE_REPLICA = lib.boolToString cfg.${job}.includeReplica; NOMAD_ADDR = cfg.${job}.nomadAddress; }; diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix index 34eeee17..8ee11e5a 100644 --- a/modules/vault-snapshots.nix +++ b/modules/vault-snapshots.nix @@ -78,6 +78,17 @@ ''; }; + includeReplica = lib.mkOption { + type = with lib.types; bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; + interval = lib.mkOption { type = with lib.types; addCheck str (x: x != ""); default = null; @@ -144,6 +155,7 @@ BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader; + INCLUDE_REPLICA = lib.boolToString cfg.${job}.includeReplica; VAULT_ADDR = cfg.${job}.vaultAddress; VAULT_FORMAT = "json"; }; From 3ca5ac4c9fc9f652d1d2f256406faa218010f97e Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 19 Jul 2022 17:11:05 -0500 Subject: [PATCH 02/11] imp: make overriding snapshotJobConfig defaults easier --- modules/consul-snapshots.nix | 20 ++++++++++++++++---- modules/nomad-snapshots.nix | 20 ++++++++++++++++---- modules/vault-snapshots.nix | 20 ++++++++++++++++---- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix index b0e4d483..71bce45b 100644 --- a/modules/consul-snapshots.nix +++ b/modules/consul-snapshots.nix @@ -226,8 +226,9 @@ in { Modify services.consul-snapshots.custom options to enable and customize. ''; - hourly = lib.mkOption { - type = with lib.types; snapshotJobConfig; + defaultHourlyOpts = lib.mkOption { + type = with lib.types; attrs; + internal = true; default = { enable = true; backupCount = 48; @@ -237,8 +238,9 @@ in { }; }; - daily = lib.mkOption { - type = with lib.types; snapshotJobConfig; + defaultDailyOpts = lib.mkOption { + type = with lib.types; attrs; + internal = true; default = { enable = true; backupCount = 30; @@ -248,6 +250,16 @@ in { }; }; + hourly = lib.mkOption { + type = with lib.types; snapshotJobConfig; + default = cfg.defaultHourlyOpts; + }; + + daily = lib.mkOption { + type = with lib.types; snapshotJobConfig; + default = cfg.defaultDailyOpts; + }; + custom = lib.mkOption { type = with lib.types; snapshotJobConfig; default = { diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix index 1cac97a0..8f7da6a1 100644 --- a/modules/nomad-snapshots.nix +++ b/modules/nomad-snapshots.nix @@ -241,8 +241,9 @@ in { Modify services.nomad-snapshots.custom options to enable and customize. ''; - hourly = lib.mkOption { - type = with lib.types; snapshotJobConfig; + defaultHourlyOpts = lib.mkOption { + type = with lib.types; attrs; + internal = true; default = { enable = true; backupCount = 48; @@ -252,8 +253,9 @@ in { }; }; - daily = lib.mkOption { - type = with lib.types; snapshotJobConfig; + defaultDailyOpts = lib.mkOption { + type = with lib.types; attrs; + internal = true; default = { enable = true; backupCount = 30; @@ -263,6 +265,16 @@ in { }; }; + hourly = lib.mkOption { + type = with lib.types; snapshotJobConfig; + default = cfg.defaultHourlyOpts; + }; + + daily = lib.mkOption { + type = with lib.types; snapshotJobConfig; + default = cfg.defaultDailyOpts; + }; + custom = lib.mkOption { type = with lib.types; snapshotJobConfig; default = { diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix index 8ee11e5a..c200b84c 100644 --- a/modules/vault-snapshots.nix +++ b/modules/vault-snapshots.nix @@ -246,8 +246,9 @@ in { Modify services.vault-snapshots.custom options to enable and customize. ''; - hourly = lib.mkOption { - type = with lib.types; snapshotJobConfig; + defaultHourlyOpts = lib.mkOption { + type = with lib.types; attrs; + internal = true; default = { enable = true; backupCount = 48; @@ -257,8 +258,9 @@ in { }; }; - daily = lib.mkOption { - type = with lib.types; snapshotJobConfig; + defaultDailyOpts = lib.mkOption { + type = with lib.types; attrs; + internal = true; default = { enable = true; backupCount = 30; @@ -268,6 +270,16 @@ in { }; }; + hourly = lib.mkOption { + type = with lib.types; snapshotJobConfig; + default = cfg.defaultHourlyOpts; + }; + + daily = lib.mkOption { + type = with lib.types; snapshotJobConfig; + default = cfg.defaultDailyOpts; + }; + custom = lib.mkOption { type = with lib.types; snapshotJobConfig; default = { From 90e1d669e0224d0b190aadc55dcca7346ff2a1ea Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 19 Jul 2022 17:11:52 -0500 Subject: [PATCH 03/11] refactor: vault snapshot script --- modules/vault-snapshots.nix | 129 ++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix index c200b84c..4c7e9629 100644 --- a/modules/vault-snapshots.nix +++ b/modules/vault-snapshots.nix @@ -148,8 +148,6 @@ }; snapshotService = job: { - path = with pkgs; [coreutils curl findutils gawk hostname jq vault-bin]; - environment = { OWNER = cfg.${job}.owner; BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; @@ -164,70 +162,69 @@ Type = "oneshot"; Restart = "on-failure"; RestartSec = "30s"; - ExecStart = pkgs.writeBashChecked "vault-snapshot-${job}-script" '' - set -exuo pipefail - - SNAP_NAME="$BACKUP_DIR/vault-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" - - applyPerms () { - TARGET="$1" - PERMS="$2" - - chown "$OWNER" "$TARGET" - chmod "$PERMS" "$TARGET" - } - - checkBackupDir () { - if [ ! -d "$BACKUP_DIR" ]; then - mkdir -p "$BACKUP_DIR" - applyPerms "$BACKUP_DIR" 0700 - fi - } - - exportToken () { - set +x - VAULT_TOKEN="$(< ${hashiTokens.vault})" - export VAULT_TOKEN - set -x - } - - isNotLeader () { - [ "$INCLUDE_LEADER" = "true" ] || \ - vault status | jq -e '(.is_self or false) == false' - } - - isNotRaftStorage () { - vault status | jq -e '.storage_type != "raft"' - } - - takeVaultSnapshot () { - vault operator raft snapshot save "$SNAP_NAME" - applyPerms "$SNAP_NAME" 0400 - } - - if isNotRaftStorage; then - echo "Vault storage backend is not raft." - echo "Ensure the appropriate storage backend is being snapshotted, ex: Consul." - exit 0 - fi - - export VAULT_ADDR - exportToken - - if isNotLeader; then - checkBackupDir - takeVaultSnapshot - fi - - find "$BACKUP_DIR" \ - -type f \ - -name "*''${BACKUP_SUFFIX}.snap" \ - -printf "%T@ %p\n" \ - | sort -r -n \ - | tail -n +${toString (cfg.${job}.backupCount + 1)} \ - | awk '{print $2}' \ - | xargs -r rm - ''; + ExecStart = let + name = "vault-snapshot-${job}-script.sh"; + script = pkgs.writeShellApplication { + inherit name; + runtimeInputs = with pkgs; [coreutils findutils gawk hostname jq vault-bin]; + text = '' + set -x + + SNAP_NAME="$BACKUP_DIR/vault-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" + + applyPerms () { + TARGET="$1" + PERMS="$2" + + chown "$OWNER" "$TARGET" + chmod "$PERMS" "$TARGET" + } + + takeVaultSnapshot () { + if [ ! -d "$BACKUP_DIR" ]; then + mkdir -p "$BACKUP_DIR" + applyPerms "$BACKUP_DIR" 0700 + fi + vault operator raft snapshot save "$SNAP_NAME" + applyPerms "$SNAP_NAME" 0400 + } + + set +x + VAULT_TOKEN="$(< ${hashiTokens.vault})" + export VAULT_TOKEN + set -x + + STATUS="$(vault status)" + + if jq -e '.storage_type != "raft"' <<< "$STATUS"; then + echo "Vault storage backend is not raft." + echo "Ensure the appropriate storage backend is being snapshotted." + exit 0 + fi + + if jq -e '(.is_self // false) == true' <<< "$STATUS"; then + ROLE="leader" + else + ROLE="replica" + fi + + if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then + takeVaultSnapshot + elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then + takeVaultSnapshot + fi + + find "$BACKUP_DIR" \ + -type f \ + -name "*''${BACKUP_SUFFIX}.snap" \ + -printf "%T@ %p\n" \ + | sort -r -n \ + | tail -n +${toString (cfg.${job}.backupCount + 1)} \ + | awk '{print $2}' \ + | xargs -r rm + ''; + }; + in "${script}/bin/${name}"; }; }; in { From ad83ddf2b4872d1c828156ceadffb221fa42f298 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 19 Jul 2022 17:12:13 -0500 Subject: [PATCH 04/11] refactor: consul snapshot script --- modules/consul-snapshots.nix | 98 ++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix index 71bce45b..8a6448ad 100644 --- a/modules/consul-snapshots.nix +++ b/modules/consul-snapshots.nix @@ -147,8 +147,6 @@ }; snapshotService = job: { - path = with pkgs; [consul coreutils findutils gawk hostname jq]; - environment = { OWNER = cfg.${job}.owner; BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; @@ -162,52 +160,56 @@ Type = "oneshot"; Restart = "on-failure"; RestartSec = "30s"; - ExecStart = pkgs.writeBashChecked "consul-snapshot-${job}-script" '' - set -exuo pipefail - - SNAP_NAME="$BACKUP_DIR/consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" - - applyPerms () { - TARGET="$1" - PERMS="$2" - - chown "$OWNER" "$TARGET" - chmod "$PERMS" "$TARGET" - } - - checkBackupDir () { - if [ ! -d "$BACKUP_DIR" ]; then - mkdir -p "$BACKUP_DIR" - applyPerms "$BACKUP_DIR" "0700" - fi - } - - isNotLeader () { - [ "$INCLUDE_LEADER" = "true" ] || \ - consul info | grep -E '^\s*leader\s+=\s+false$' - } - - takeConsulSnapshot () { - consul snapshot save "$SNAP_NAME" - applyPerms "$SNAP_NAME" "0400" - } - - export CONSUL_HTTP_ADDR - - if isNotLeader; then - checkBackupDir - takeConsulSnapshot - fi - - find "$BACKUP_DIR" \ - -type f \ - -name "*''${BACKUP_SUFFIX}.snap" \ - -printf "%T@ %p\n" \ - | sort -r -n \ - | tail -n +${toString (cfg.${job}.backupCount + 1)} \ - | awk '{print $2}' \ - | xargs -r rm - ''; + ExecStart = let + name = "consul-snapshot-${job}-script.sh"; + script = pkgs.writeShellApplication { + inherit name; + runtimeInputs = with pkgs; [consul coreutils findutils gawk hostname jq]; + text = '' + set -x + + SNAP_NAME="$BACKUP_DIR/consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" + + applyPerms () { + TARGET="$1" + PERMS="$2" + + chown "$OWNER" "$TARGET" + chmod "$PERMS" "$TARGET" + } + + takeConsulSnapshot () { + if [ ! -d "$BACKUP_DIR" ]; then + mkdir -p "$BACKUP_DIR" + applyPerms "$BACKUP_DIR" "0700" + fi + consul snapshot save "$SNAP_NAME" + applyPerms "$SNAP_NAME" "0400" + } + + if consul info | grep -E '^\s*leader\s+=\s+true$'; then + ROLE="leader" + else + ROLE="replica" + fi + + if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then + takeConsulSnapshot + elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then + takeConsulSnapshot + fi + + find "$BACKUP_DIR" \ + -type f \ + -name "*''${BACKUP_SUFFIX}.snap" \ + -printf "%T@ %p\n" \ + | sort -r -n \ + | tail -n +${toString (cfg.${job}.backupCount + 1)} \ + | awk '{print $2}' \ + | xargs -r rm + ''; + }; + in "${script}/bin/${name}"; }; }; in { From e1b16aae228bacde0a47773c793c3fbff72f7c84 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 19 Jul 2022 17:12:31 -0500 Subject: [PATCH 05/11] refactor: nomad snapshot script --- modules/nomad-snapshots.nix | 123 ++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix index 8f7da6a1..d7c5ee9e 100644 --- a/modules/nomad-snapshots.nix +++ b/modules/nomad-snapshots.nix @@ -148,8 +148,6 @@ }; snapshotService = job: { - path = with pkgs; [coreutils curl findutils gawk hostname jq nomad]; - environment = { OWNER = cfg.${job}.owner; BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; @@ -163,66 +161,69 @@ Type = "oneshot"; Restart = "on-failure"; RestartSec = "30s"; - ExecStart = pkgs.writeBashChecked "nomad-snapshot-${job}-script" '' - set -exuo pipefail - - SNAP_NAME="$BACKUP_DIR/nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" - - applyPerms () { - TARGET="$1" - PERMS="$2" - - chown "$OWNER" "$TARGET" - chmod "$PERMS" "$TARGET" - } - - checkBackupDir () { - if [ ! -d "$BACKUP_DIR" ]; then - mkdir -p "$BACKUP_DIR" - applyPerms "$BACKUP_DIR" "0700" - fi - } - - exportToken () { - if [ ! -f ${hashiTokens.nomad-snapshot} ]; then - echo "Suitable nomad token for snapshotting not found." - echo "Ensure the appropriate token for snapshotting is available."; - exit 0; - else - set +x - NOMAD_TOKEN="$(< ${hashiTokens.nomad-snapshot})" - export NOMAD_TOKEN + ExecStart = let + name = "nomad-snapshot-${job}-script.sh"; + script = pkgs.writeShellApplication { + inherit name; + runtimeInputs = with pkgs; [coreutils findutils gawk hostname jq nomad]; + text = '' set -x - fi - } - - isNotLeader () { - [ "$INCLUDE_LEADER" = "true" ] || \ - nomad agent-info --json | jq -e '.stats.nomad.leader == "false"' - } - - takeNomadSnapshot () { - nomad operator snapshot save "$SNAP_NAME" - applyPerms "$SNAP_NAME" "0400" - } - - export NOMAD_ADDR - exportToken - - if isNotLeader; then - checkBackupDir - takeNomadSnapshot - fi - - find "$BACKUP_DIR" \ - -type f \ - -name "*''${BACKUP_SUFFIX}.snap" \ - -printf "%T@ %p\n" \ - | sort -r -n \ - | tail -n +${toString (cfg.${job}.backupCount + 1)} \ - | awk '{print $2}' \ - | xargs -r rm - ''; + + SNAP_NAME="$BACKUP_DIR/nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" + + applyPerms () { + TARGET="$1" + PERMS="$2" + + chown "$OWNER" "$TARGET" + chmod "$PERMS" "$TARGET" + } + + takeNomadSnapshot () { + if [ ! -d "$BACKUP_DIR" ]; then + mkdir -p "$BACKUP_DIR" + applyPerms "$BACKUP_DIR" "0700" + fi + nomad operator snapshot save "$SNAP_NAME" + applyPerms "$SNAP_NAME" "0400" + } + + if [ ! -f ${hashiTokens.nomad-snapshot} ]; then + echo "Suitable nomad token for snapshotting not found." + echo "Ensure the appropriate token for snapshotting is available."; + exit 0; + else + set +x + NOMAD_TOKEN="$(< ${hashiTokens.nomad-snapshot})" + export NOMAD_TOKEN + set -x + fi + + STATUS="$(nomad agent-info --json)" + + if jq -e '(.stats.nomad.leader // "false") == "true"' <<< "$STATUS"; then + ROLE="leader" + else + ROLE="replica" + fi + + if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then + takeNomadSnapshot + elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then + takeNomadSnapshot + fi + + find "$BACKUP_DIR" \ + -type f \ + -name "*''${BACKUP_SUFFIX}.snap" \ + -printf "%T@ %p\n" \ + | sort -r -n \ + | tail -n +${toString (cfg.${job}.backupCount + 1)} \ + | awk '{print $2}' \ + | xargs -r rm + ''; + }; + in "${script}/bin/${name}"; }; }; in { From 73cf815818ceee693d0f378e7806bfce2337b791 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 19 Jul 2022 17:13:36 -0500 Subject: [PATCH 06/11] fix: disable vault snapshot on replicas temporarily; cleanup lib refs --- modules/consul-snapshots.nix | 89 +++++++++++++++++++----------------- modules/nomad-snapshots.nix | 89 +++++++++++++++++++----------------- modules/vault-snapshots.nix | 89 +++++++++++++++++++----------------- profiles/vault/server.nix | 10 +++- 4 files changed, 147 insertions(+), 130 deletions(-) diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix index 8a6448ad..669c3948 100644 --- a/modules/consul-snapshots.nix +++ b/modules/consul-snapshots.nix @@ -6,19 +6,22 @@ }: let cfg = config.services.consul-snapshots; - snapshotJobConfig = with lib.types; + inherit (lib) boolToString mkEnableOption mkIf mkOption; + inherit (lib.types) addCheck attrs bool int str submodule; + + snapshotJobConfig = submodule { options = { - enable = lib.mkOption { - type = with lib.types; bool; + enable = mkOption { + type = bool; default = true; description = '' Creates a systemd service and timer to automatically save Consul snapshots. ''; }; - backupCount = lib.mkOption { - type = with lib.types; addCheck int (x: x >= 0); + backupCount = mkOption { + type = addCheck int (x: x >= 0); default = null; description = '' The number of snapshots to keep. A sensible value matched to the onCalendar @@ -29,8 +32,8 @@ ''; }; - backupDirPrefix = lib.mkOption { - type = with lib.types; str; + backupDirPrefix = mkOption { + type = str; default = "/var/lib/private/consul/snapshots"; description = '' The top level location to store the snapshots. The actual storage location @@ -43,8 +46,8 @@ ''; }; - backupSuffix = lib.mkOption { - type = with lib.types; addCheck str (x: x != ""); + backupSuffix = mkOption { + type = addCheck str (x: x != ""); default = null; description = '' Sets the saved snapshot filename with a descriptive suffix prior to the file @@ -54,16 +57,16 @@ ''; }; - consulAddress = lib.mkOption { - type = with lib.types; str; + consulAddress = mkOption { + type = str; default = "http://127.0.0.1:8500"; description = '' The local consul server address, including protocol and port. ''; }; - fixedRandomDelay = lib.mkOption { - type = with lib.types; bool; + fixedRandomDelay = mkOption { + type = bool; default = true; description = '' Makes randomizedDelaySec fixed between service restarts if true. @@ -72,8 +75,8 @@ ''; }; - includeLeader = lib.mkOption { - type = with lib.types; bool; + includeLeader = mkOption { + type = bool; default = true; description = '' Whether to include the leader in the servers which will save snapshots. @@ -85,8 +88,8 @@ ''; }; - includeReplica = lib.mkOption { - type = with lib.types; bool; + includeReplica = mkOption { + type = bool; default = true; description = '' Whether to include the replicas in the servers which will save snapshots. @@ -96,8 +99,8 @@ ''; }; - interval = lib.mkOption { - type = with lib.types; addCheck str (x: x != ""); + interval = mkOption { + type = addCheck str (x: x != ""); default = null; description = '' The default onCalendar systemd timer string to trigger snapshot backups. @@ -110,8 +113,8 @@ ''; }; - randomizedDelaySec = lib.mkOption { - type = with lib.types; addCheck int (x: x >= 0); + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); default = 0; description = '' A randomization period to be added to each systemd timer to avoid @@ -125,8 +128,8 @@ ''; }; - owner = lib.mkOption { - type = with lib.types; str; + owner = mkOption { + type = str; default = "consul:consul"; description = '' The user and group to own the snapshot storage directory and snapshot files. @@ -151,8 +154,8 @@ OWNER = cfg.${job}.owner; BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; - INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader; - INCLUDE_REPLICA = lib.boolToString cfg.${job}.includeReplica; + INCLUDE_LEADER = boolToString cfg.${job}.includeLeader; + INCLUDE_REPLICA = boolToString cfg.${job}.includeReplica; CONSUL_HTTP_ADDR = cfg.${job}.consulAddress; }; @@ -215,7 +218,7 @@ in { options = { services.consul-snapshots = { - enable = lib.mkEnableOption '' + enable = mkEnableOption '' Enable Consul snapshots. By default hourly snapshots will be taken and stored for 2 days on each consul server. @@ -228,8 +231,8 @@ in { Modify services.consul-snapshots.custom options to enable and customize. ''; - defaultHourlyOpts = lib.mkOption { - type = with lib.types; attrs; + defaultHourlyOpts = mkOption { + type = attrs; internal = true; default = { enable = true; @@ -240,8 +243,8 @@ in { }; }; - defaultDailyOpts = lib.mkOption { - type = with lib.types; attrs; + defaultDailyOpts = mkOption { + type = attrs; internal = true; default = { enable = true; @@ -252,18 +255,18 @@ in { }; }; - hourly = lib.mkOption { - type = with lib.types; snapshotJobConfig; + hourly = mkOption { + type = snapshotJobConfig; default = cfg.defaultHourlyOpts; }; - daily = lib.mkOption { - type = with lib.types; snapshotJobConfig; + daily = mkOption { + type = snapshotJobConfig; default = cfg.defaultDailyOpts; }; - custom = lib.mkOption { - type = with lib.types; snapshotJobConfig; + custom = mkOption { + type = snapshotJobConfig; default = { enable = false; backupSuffix = "custom"; @@ -272,23 +275,23 @@ in { }; }; - config = lib.mkIf cfg.enable { + config = mkIf cfg.enable { # Hourly snapshot configuration systemd.timers.consul-snapshots-hourly = - lib.mkIf cfg.hourly.enable (snapshotTimer "hourly"); + mkIf cfg.hourly.enable (snapshotTimer "hourly"); systemd.services.consul-snapshots-hourly = - lib.mkIf cfg.hourly.enable (snapshotService "hourly"); + mkIf cfg.hourly.enable (snapshotService "hourly"); # Daily snapshot configuration systemd.timers.consul-snapshots-daily = - lib.mkIf cfg.daily.enable (snapshotTimer "daily"); + mkIf cfg.daily.enable (snapshotTimer "daily"); systemd.services.consul-snapshots-daily = - lib.mkIf cfg.daily.enable (snapshotService "daily"); + mkIf cfg.daily.enable (snapshotService "daily"); # Custom snapshot configuration systemd.timers.consul-snapshots-custom = - lib.mkIf cfg.custom.enable (snapshotTimer "custom"); + mkIf cfg.custom.enable (snapshotTimer "custom"); systemd.services.consul-snapshots-custom = - lib.mkIf cfg.custom.enable (snapshotService "custom"); + mkIf cfg.custom.enable (snapshotService "custom"); }; } diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix index d7c5ee9e..80e2a475 100644 --- a/modules/nomad-snapshots.nix +++ b/modules/nomad-snapshots.nix @@ -7,19 +7,22 @@ }: let cfg = config.services.nomad-snapshots; - snapshotJobConfig = with lib.types; + inherit (lib) boolToString mkEnableOption mkIf mkOption; + inherit (lib.types) addCheck attrs bool int str submodule; + + snapshotJobConfig = submodule { options = { - enable = lib.mkOption { - type = with lib.types; bool; + enable = mkOption { + type = bool; default = true; description = '' Creates a systemd service and timer to automatically save Nomad snapshots. ''; }; - backupCount = lib.mkOption { - type = with lib.types; addCheck int (x: x >= 0); + backupCount = mkOption { + type = addCheck int (x: x >= 0); default = null; description = '' The number of snapshots to keep. A sensible value matched to the onCalendar @@ -30,8 +33,8 @@ ''; }; - backupDirPrefix = lib.mkOption { - type = with lib.types; str; + backupDirPrefix = mkOption { + type = str; default = "/var/lib/private/nomad/snapshots"; description = '' The top level location to store the snapshots. The actual storage location @@ -44,8 +47,8 @@ ''; }; - backupSuffix = lib.mkOption { - type = with lib.types; addCheck str (x: x != ""); + backupSuffix = mkOption { + type = addCheck str (x: x != ""); default = null; description = '' Sets the saved snapshot filename with a descriptive suffix prior to the file @@ -55,8 +58,8 @@ ''; }; - fixedRandomDelay = lib.mkOption { - type = with lib.types; bool; + fixedRandomDelay = mkOption { + type = bool; default = true; description = '' Makes randomizedDelaySec fixed between service restarts if true. @@ -65,8 +68,8 @@ ''; }; - includeLeader = lib.mkOption { - type = with lib.types; bool; + includeLeader = mkOption { + type = bool; default = true; description = '' Whether to include the leader in the servers which will save snapshots. @@ -78,8 +81,8 @@ ''; }; - includeReplica = lib.mkOption { - type = with lib.types; bool; + includeReplica = mkOption { + type = bool; default = true; description = '' Whether to include the replicas in the servers which will save snapshots. @@ -89,8 +92,8 @@ ''; }; - interval = lib.mkOption { - type = with lib.types; addCheck str (x: x != ""); + interval = mkOption { + type = addCheck str (x: x != ""); default = null; description = '' The default onCalendar systemd timer string to trigger snapshot backups. @@ -103,8 +106,8 @@ ''; }; - randomizedDelaySec = lib.mkOption { - type = with lib.types; addCheck int (x: x >= 0); + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); default = 0; description = '' A randomization period to be added to each systemd timer to avoid @@ -118,16 +121,16 @@ ''; }; - owner = lib.mkOption { - type = with lib.types; str; + owner = mkOption { + type = str; default = "root:root"; description = '' The user and group to own the snapshot storage directory and snapshot files. ''; }; - nomadAddress = lib.mkOption { - type = with lib.types; str; + nomadAddress = mkOption { + type = str; default = "https://127.0.0.1:4646"; description = '' The local nomad server address, including protocol and port. @@ -152,8 +155,8 @@ OWNER = cfg.${job}.owner; BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; - INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader; - INCLUDE_REPLICA = lib.boolToString cfg.${job}.includeReplica; + INCLUDE_LEADER = boolToString cfg.${job}.includeLeader; + INCLUDE_REPLICA = boolToString cfg.${job}.includeReplica; NOMAD_ADDR = cfg.${job}.nomadAddress; }; @@ -229,7 +232,7 @@ in { options = { services.nomad-snapshots = { - enable = lib.mkEnableOption '' + enable = mkEnableOption '' Enable Nomad snapshots. By default hourly snapshots will be taken and stored for 2 days on each nomad server. @@ -242,8 +245,8 @@ in { Modify services.nomad-snapshots.custom options to enable and customize. ''; - defaultHourlyOpts = lib.mkOption { - type = with lib.types; attrs; + defaultHourlyOpts = mkOption { + type = attrs; internal = true; default = { enable = true; @@ -254,8 +257,8 @@ in { }; }; - defaultDailyOpts = lib.mkOption { - type = with lib.types; attrs; + defaultDailyOpts = mkOption { + type = attrs; internal = true; default = { enable = true; @@ -266,18 +269,18 @@ in { }; }; - hourly = lib.mkOption { - type = with lib.types; snapshotJobConfig; + hourly = mkOption { + type = snapshotJobConfig; default = cfg.defaultHourlyOpts; }; - daily = lib.mkOption { - type = with lib.types; snapshotJobConfig; + daily = mkOption { + type = snapshotJobConfig; default = cfg.defaultDailyOpts; }; - custom = lib.mkOption { - type = with lib.types; snapshotJobConfig; + custom = mkOption { + type = snapshotJobConfig; default = { enable = false; backupSuffix = "custom"; @@ -286,23 +289,23 @@ in { }; }; - config = lib.mkIf cfg.enable { + config = mkIf cfg.enable { # Hourly snapshot configuration systemd.timers.nomad-snapshots-hourly = - lib.mkIf cfg.hourly.enable (snapshotTimer "hourly"); + mkIf cfg.hourly.enable (snapshotTimer "hourly"); systemd.services.nomad-snapshots-hourly = - lib.mkIf cfg.hourly.enable (snapshotService "hourly"); + mkIf cfg.hourly.enable (snapshotService "hourly"); # Daily snapshot configuration systemd.timers.nomad-snapshots-daily = - lib.mkIf cfg.daily.enable (snapshotTimer "daily"); + mkIf cfg.daily.enable (snapshotTimer "daily"); systemd.services.nomad-snapshots-daily = - lib.mkIf cfg.daily.enable (snapshotService "daily"); + mkIf cfg.daily.enable (snapshotService "daily"); # Custom snapshot configuration systemd.timers.nomad-snapshots-custom = - lib.mkIf cfg.custom.enable (snapshotTimer "custom"); + mkIf cfg.custom.enable (snapshotTimer "custom"); systemd.services.nomad-snapshots-custom = - lib.mkIf cfg.custom.enable (snapshotService "custom"); + mkIf cfg.custom.enable (snapshotService "custom"); }; } diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix index 4c7e9629..488b4335 100644 --- a/modules/vault-snapshots.nix +++ b/modules/vault-snapshots.nix @@ -7,19 +7,22 @@ }: let cfg = config.services.vault-snapshots; - snapshotJobConfig = with lib.types; + inherit (lib) boolToString mkEnableOption mkIf mkOption; + inherit (lib.types) addCheck attrs bool int str submodule; + + snapshotJobConfig = submodule { options = { - enable = lib.mkOption { - type = with lib.types; bool; + enable = mkOption { + type = bool; default = true; description = '' Creates a systemd service and timer to automatically save Vault snapshots. ''; }; - backupCount = lib.mkOption { - type = with lib.types; addCheck int (x: x >= 0); + backupCount = mkOption { + type = addCheck int (x: x >= 0); default = null; description = '' The number of snapshots to keep. A sensible value matched to the onCalendar @@ -30,8 +33,8 @@ ''; }; - backupDirPrefix = lib.mkOption { - type = with lib.types; str; + backupDirPrefix = mkOption { + type = str; default = "/var/lib/private/vault/snapshots"; description = '' The top level location to store the snapshots. The actual storage location @@ -44,8 +47,8 @@ ''; }; - backupSuffix = lib.mkOption { - type = with lib.types; addCheck str (x: x != ""); + backupSuffix = mkOption { + type = addCheck str (x: x != ""); default = null; description = '' Sets the saved snapshot filename with a descriptive suffix prior to the file @@ -55,8 +58,8 @@ ''; }; - fixedRandomDelay = lib.mkOption { - type = with lib.types; bool; + fixedRandomDelay = mkOption { + type = bool; default = true; description = '' Makes randomizedDelaySec fixed between service restarts if true. @@ -65,8 +68,8 @@ ''; }; - includeLeader = lib.mkOption { - type = with lib.types; bool; + includeLeader = mkOption { + type = bool; default = true; description = '' Whether to include the leader in the servers which will save snapshots. @@ -78,8 +81,8 @@ ''; }; - includeReplica = lib.mkOption { - type = with lib.types; bool; + includeReplica = mkOption { + type = bool; default = true; description = '' Whether to include the replicas in the servers which will save snapshots. @@ -89,8 +92,8 @@ ''; }; - interval = lib.mkOption { - type = with lib.types; addCheck str (x: x != ""); + interval = mkOption { + type = addCheck str (x: x != ""); default = null; description = '' The default onCalendar systemd timer string to trigger snapshot backups. @@ -103,8 +106,8 @@ ''; }; - randomizedDelaySec = lib.mkOption { - type = with lib.types; addCheck int (x: x >= 0); + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); default = 0; description = '' A randomization period to be added to each systemd timer to avoid @@ -118,16 +121,16 @@ ''; }; - owner = lib.mkOption { - type = with lib.types; str; + owner = mkOption { + type = str; default = "vault:vault"; description = '' The user and group to own the snapshot storage directory and snapshot files. ''; }; - vaultAddress = lib.mkOption { - type = with lib.types; str; + vaultAddress = mkOption { + type = str; default = "https://127.0.0.1:8200"; description = '' The local vault server address, including protocol and port. @@ -152,8 +155,8 @@ OWNER = cfg.${job}.owner; BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; - INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader; - INCLUDE_REPLICA = lib.boolToString cfg.${job}.includeReplica; + INCLUDE_LEADER = boolToString cfg.${job}.includeLeader; + INCLUDE_REPLICA = boolToString cfg.${job}.includeReplica; VAULT_ADDR = cfg.${job}.vaultAddress; VAULT_FORMAT = "json"; }; @@ -230,7 +233,7 @@ in { options = { services.vault-snapshots = { - enable = lib.mkEnableOption '' + enable = mkEnableOption '' Enable Vault snapshots. By default hourly snapshots will be taken and stored for 2 days on each vault server. @@ -243,8 +246,8 @@ in { Modify services.vault-snapshots.custom options to enable and customize. ''; - defaultHourlyOpts = lib.mkOption { - type = with lib.types; attrs; + defaultHourlyOpts = mkOption { + type = attrs; internal = true; default = { enable = true; @@ -255,8 +258,8 @@ in { }; }; - defaultDailyOpts = lib.mkOption { - type = with lib.types; attrs; + defaultDailyOpts = mkOption { + type = attrs; internal = true; default = { enable = true; @@ -267,18 +270,18 @@ in { }; }; - hourly = lib.mkOption { - type = with lib.types; snapshotJobConfig; + hourly = mkOption { + type = snapshotJobConfig; default = cfg.defaultHourlyOpts; }; - daily = lib.mkOption { - type = with lib.types; snapshotJobConfig; + daily = mkOption { + type = snapshotJobConfig; default = cfg.defaultDailyOpts; }; - custom = lib.mkOption { - type = with lib.types; snapshotJobConfig; + custom = mkOption { + type = snapshotJobConfig; default = { enable = false; backupSuffix = "custom"; @@ -287,23 +290,23 @@ in { }; }; - config = lib.mkIf cfg.enable { + config = mkIf cfg.enable { # Hourly snapshot configuration systemd.timers.vault-snapshots-hourly = - lib.mkIf cfg.hourly.enable (snapshotTimer "hourly"); + mkIf cfg.hourly.enable (snapshotTimer "hourly"); systemd.services.vault-snapshots-hourly = - lib.mkIf cfg.hourly.enable (snapshotService "hourly"); + mkIf cfg.hourly.enable (snapshotService "hourly"); # Daily snapshot configuration systemd.timers.vault-snapshots-daily = - lib.mkIf cfg.daily.enable (snapshotTimer "daily"); + mkIf cfg.daily.enable (snapshotTimer "daily"); systemd.services.vault-snapshots-daily = - lib.mkIf cfg.daily.enable (snapshotService "daily"); + mkIf cfg.daily.enable (snapshotService "daily"); # Custom snapshot configuration systemd.timers.vault-snapshots-custom = - lib.mkIf cfg.custom.enable (snapshotTimer "custom"); + mkIf cfg.custom.enable (snapshotTimer "custom"); systemd.services.vault-snapshots-custom = - lib.mkIf cfg.custom.enable (snapshotService "custom"); + mkIf cfg.custom.enable (snapshotService "custom"); }; } diff --git a/profiles/vault/server.nix b/profiles/vault/server.nix index 84cb5fc7..c3b51f63 100644 --- a/profiles/vault/server.nix +++ b/profiles/vault/server.nix @@ -12,9 +12,17 @@ ]; }; - Switches = { + Switches = let + cfg = config.services.vault-snapshots; + in { services.vault.enable = true; services.vault-snapshots.enable = true; + + # Do not include vault replicas in snapshotting until replica snapshot bug is resolved + # Ref: https://github.com/hashicorp/vault/issues/15258 + services.vault-snapshots.hourly = cfg.defaultHourlyOpts // { includeReplica = false; }; + services.vault-snapshots.daily = cfg.defaultDailyOpts // { includeReplica = false; }; + services.vault.ui = true; }; From e845a860499d9c0f7dd499b8b47ec575f32a6dac Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Wed, 20 Jul 2022 18:06:35 -0500 Subject: [PATCH 07/11] fmt: alejandra --- modules/consul-snapshots.nix | 221 +++++++++--------- modules/nomad-snapshots.nix | 221 +++++++++--------- modules/vault-snapshots.nix | 221 +++++++++--------- .../vault/secrets-provisioning/hashistack.nix | 104 ++++----- .../letsencrypt-ingress.nix | 11 +- 5 files changed, 388 insertions(+), 390 deletions(-) diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix index 669c3948..d17dc35a 100644 --- a/modules/consul-snapshots.nix +++ b/modules/consul-snapshots.nix @@ -9,134 +9,133 @@ inherit (lib) boolToString mkEnableOption mkIf mkOption; inherit (lib.types) addCheck attrs bool int str submodule; - snapshotJobConfig = - submodule { - options = { - enable = mkOption { - type = bool; - default = true; - description = '' - Creates a systemd service and timer to automatically save Consul snapshots. - ''; - }; + snapshotJobConfig = submodule { + options = { + enable = mkOption { + type = bool; + default = true; + description = '' + Creates a systemd service and timer to automatically save Consul snapshots. + ''; + }; - backupCount = mkOption { - type = addCheck int (x: x >= 0); - default = null; - description = '' - The number of snapshots to keep. A sensible value matched to the onCalendar - interval parameter should be used. Examples of sensible suggestions may be: + backupCount = mkOption { + type = addCheck int (x: x >= 0); + default = null; + description = '' + The number of snapshots to keep. A sensible value matched to the onCalendar + interval parameter should be used. Examples of sensible suggestions may be: - 48 backupCount for "hourly" interval (2 days of backups) - 30 backupCount for "daily" interval (1 month of backups) - ''; - }; - - backupDirPrefix = mkOption { - type = str; - default = "/var/lib/private/consul/snapshots"; - description = '' - The top level location to store the snapshots. The actual storage location - of the files will be this prefix path with the snapshot job name appended, - where the job is one of "hourly", "daily" or "custom". + 48 backupCount for "hourly" interval (2 days of backups) + 30 backupCount for "daily" interval (1 month of backups) + ''; + }; - Therefore, saved snapshot files will be found at: + backupDirPrefix = mkOption { + type = str; + default = "/var/lib/private/consul/snapshots"; + description = '' + The top level location to store the snapshots. The actual storage location + of the files will be this prefix path with the snapshot job name appended, + where the job is one of "hourly", "daily" or "custom". - $backupDirPrefix/$job/*.snap - ''; - }; + Therefore, saved snapshot files will be found at: - backupSuffix = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - Sets the saved snapshot filename with a descriptive suffix prior to the file - extension. This will enable selective snapshot job pruning. The form is: + $backupDirPrefix/$job/*.snap + ''; + }; - consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap - ''; - }; + backupSuffix = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + Sets the saved snapshot filename with a descriptive suffix prior to the file + extension. This will enable selective snapshot job pruning. The form is: - consulAddress = mkOption { - type = str; - default = "http://127.0.0.1:8500"; - description = '' - The local consul server address, including protocol and port. - ''; - }; + consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap + ''; + }; - fixedRandomDelay = mkOption { - type = bool; - default = true; - description = '' - Makes randomizedDelaySec fixed between service restarts if true. - This will reduce jitter and allow the interval to remain fixed, - while still allowing start time randomization to avoid leader overload. - ''; - }; + consulAddress = mkOption { + type = str; + default = "http://127.0.0.1:8500"; + description = '' + The local consul server address, including protocol and port. + ''; + }; - includeLeader = mkOption { - type = bool; - default = true; - description = '' - Whether to include the leader in the servers which will save snapshots. - This may reduce load on the leader slightly, but by default snapshot - saves are proxied through the leader anyway. + fixedRandomDelay = mkOption { + type = bool; + default = true; + description = '' + Makes randomizedDelaySec fixed between service restarts if true. + This will reduce jitter and allow the interval to remain fixed, + while still allowing start time randomization to avoid leader overload. + ''; + }; - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; + includeLeader = mkOption { + type = bool; + default = true; + description = '' + Whether to include the leader in the servers which will save snapshots. + This may reduce load on the leader slightly, but by default snapshot + saves are proxied through the leader anyway. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; - includeReplica = mkOption { - type = bool; - default = true; - description = '' - Whether to include the replicas in the servers which will save snapshots. + includeReplica = mkOption { + type = bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; - interval = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - The default onCalendar systemd timer string to trigger snapshot backups. - Any valid systemd OnCalendar string may be used here. Sensible - defaults for backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) - daily: 86400 randomizedDelaySec, 30 backupCount (1 month) - ''; - }; + interval = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + The default onCalendar systemd timer string to trigger snapshot backups. + Any valid systemd OnCalendar string may be used here. Sensible + defaults for backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) + daily: 86400 randomizedDelaySec, 30 backupCount (1 month) + ''; + }; - randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); - default = 0; - description = '' - A randomization period to be added to each systemd timer to avoid - leader overload. By default fixedRandomDelay will also be true to minimize - jitter and maintain fixed interval snapshots. Sensible defaults for - backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) - 86400 randomizedDelaySec for "daily" interval (1 day randomization) - ''; - }; + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); + default = 0; + description = '' + A randomization period to be added to each systemd timer to avoid + leader overload. By default fixedRandomDelay will also be true to minimize + jitter and maintain fixed interval snapshots. Sensible defaults for + backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) + 86400 randomizedDelaySec for "daily" interval (1 day randomization) + ''; + }; - owner = mkOption { - type = str; - default = "consul:consul"; - description = '' - The user and group to own the snapshot storage directory and snapshot files. - ''; - }; + owner = mkOption { + type = str; + default = "consul:consul"; + description = '' + The user and group to own the snapshot storage directory and snapshot files. + ''; }; }; + }; snapshotTimer = job: { partOf = ["consul-snapshots-${job}.service"]; diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix index 80e2a475..64d3d150 100644 --- a/modules/nomad-snapshots.nix +++ b/modules/nomad-snapshots.nix @@ -10,134 +10,133 @@ inherit (lib) boolToString mkEnableOption mkIf mkOption; inherit (lib.types) addCheck attrs bool int str submodule; - snapshotJobConfig = - submodule { - options = { - enable = mkOption { - type = bool; - default = true; - description = '' - Creates a systemd service and timer to automatically save Nomad snapshots. - ''; - }; + snapshotJobConfig = submodule { + options = { + enable = mkOption { + type = bool; + default = true; + description = '' + Creates a systemd service and timer to automatically save Nomad snapshots. + ''; + }; - backupCount = mkOption { - type = addCheck int (x: x >= 0); - default = null; - description = '' - The number of snapshots to keep. A sensible value matched to the onCalendar - interval parameter should be used. Examples of sensible suggestions may be: + backupCount = mkOption { + type = addCheck int (x: x >= 0); + default = null; + description = '' + The number of snapshots to keep. A sensible value matched to the onCalendar + interval parameter should be used. Examples of sensible suggestions may be: - 48 backupCount for "hourly" interval (2 days of backups) - 30 backupCount for "daily" interval (1 month of backups) - ''; - }; - - backupDirPrefix = mkOption { - type = str; - default = "/var/lib/private/nomad/snapshots"; - description = '' - The top level location to store the snapshots. The actual storage location - of the files will be this prefix path with the snapshot job name appended, - where the job is one of "hourly", "daily" or "custom". + 48 backupCount for "hourly" interval (2 days of backups) + 30 backupCount for "daily" interval (1 month of backups) + ''; + }; - Therefore, saved snapshot files will be found at: + backupDirPrefix = mkOption { + type = str; + default = "/var/lib/private/nomad/snapshots"; + description = '' + The top level location to store the snapshots. The actual storage location + of the files will be this prefix path with the snapshot job name appended, + where the job is one of "hourly", "daily" or "custom". - $backupDirPrefix/$job/*.snap - ''; - }; + Therefore, saved snapshot files will be found at: - backupSuffix = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - Sets the saved snapshot filename with a descriptive suffix prior to the file - extension. This will enable selective snapshot job pruning. The form is: + $backupDirPrefix/$job/*.snap + ''; + }; - nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap - ''; - }; + backupSuffix = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + Sets the saved snapshot filename with a descriptive suffix prior to the file + extension. This will enable selective snapshot job pruning. The form is: - fixedRandomDelay = mkOption { - type = bool; - default = true; - description = '' - Makes randomizedDelaySec fixed between service restarts if true. - This will reduce jitter and allow the interval to remain fixed, - while still allowing start time randomization to avoid leader overload. - ''; - }; + nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap + ''; + }; - includeLeader = mkOption { - type = bool; - default = true; - description = '' - Whether to include the leader in the servers which will save snapshots. - This may reduce load on the leader slightly, but by default snapshot - saves are proxied through the leader anyway. + fixedRandomDelay = mkOption { + type = bool; + default = true; + description = '' + Makes randomizedDelaySec fixed between service restarts if true. + This will reduce jitter and allow the interval to remain fixed, + while still allowing start time randomization to avoid leader overload. + ''; + }; - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; + includeLeader = mkOption { + type = bool; + default = true; + description = '' + Whether to include the leader in the servers which will save snapshots. + This may reduce load on the leader slightly, but by default snapshot + saves are proxied through the leader anyway. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; - includeReplica = mkOption { - type = bool; - default = true; - description = '' - Whether to include the replicas in the servers which will save snapshots. + includeReplica = mkOption { + type = bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; - interval = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - The default onCalendar systemd timer string to trigger snapshot backups. - Any valid systemd OnCalendar string may be used here. Sensible - defaults for backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) - daily: 86400 randomizedDelaySec, 30 backupCount (1 month) - ''; - }; + interval = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + The default onCalendar systemd timer string to trigger snapshot backups. + Any valid systemd OnCalendar string may be used here. Sensible + defaults for backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) + daily: 86400 randomizedDelaySec, 30 backupCount (1 month) + ''; + }; - randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); - default = 0; - description = '' - A randomization period to be added to each systemd timer to avoid - leader overload. By default fixedRandomDelay will also be true to minimize - jitter and maintain fixed interval snapshots. Sensible defaults for - backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) - 86400 randomizedDelaySec for "daily" interval (1 day randomization) - ''; - }; + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); + default = 0; + description = '' + A randomization period to be added to each systemd timer to avoid + leader overload. By default fixedRandomDelay will also be true to minimize + jitter and maintain fixed interval snapshots. Sensible defaults for + backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) + 86400 randomizedDelaySec for "daily" interval (1 day randomization) + ''; + }; - owner = mkOption { - type = str; - default = "root:root"; - description = '' - The user and group to own the snapshot storage directory and snapshot files. - ''; - }; + owner = mkOption { + type = str; + default = "root:root"; + description = '' + The user and group to own the snapshot storage directory and snapshot files. + ''; + }; - nomadAddress = mkOption { - type = str; - default = "https://127.0.0.1:4646"; - description = '' - The local nomad server address, including protocol and port. - ''; - }; + nomadAddress = mkOption { + type = str; + default = "https://127.0.0.1:4646"; + description = '' + The local nomad server address, including protocol and port. + ''; }; }; + }; snapshotTimer = job: { partOf = ["nomad-snapshots-${job}.service"]; diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix index 488b4335..8b9696f8 100644 --- a/modules/vault-snapshots.nix +++ b/modules/vault-snapshots.nix @@ -10,134 +10,133 @@ inherit (lib) boolToString mkEnableOption mkIf mkOption; inherit (lib.types) addCheck attrs bool int str submodule; - snapshotJobConfig = - submodule { - options = { - enable = mkOption { - type = bool; - default = true; - description = '' - Creates a systemd service and timer to automatically save Vault snapshots. - ''; - }; + snapshotJobConfig = submodule { + options = { + enable = mkOption { + type = bool; + default = true; + description = '' + Creates a systemd service and timer to automatically save Vault snapshots. + ''; + }; - backupCount = mkOption { - type = addCheck int (x: x >= 0); - default = null; - description = '' - The number of snapshots to keep. A sensible value matched to the onCalendar - interval parameter should be used. Examples of sensible suggestions may be: + backupCount = mkOption { + type = addCheck int (x: x >= 0); + default = null; + description = '' + The number of snapshots to keep. A sensible value matched to the onCalendar + interval parameter should be used. Examples of sensible suggestions may be: - 48 backupCount for "hourly" interval (2 days of backups) - 30 backupCount for "daily" interval (1 month of backups) - ''; - }; - - backupDirPrefix = mkOption { - type = str; - default = "/var/lib/private/vault/snapshots"; - description = '' - The top level location to store the snapshots. The actual storage location - of the files will be this prefix path with the snapshot job name appended, - where the job is one of "hourly", "daily" or "custom". + 48 backupCount for "hourly" interval (2 days of backups) + 30 backupCount for "daily" interval (1 month of backups) + ''; + }; - Therefore, saved snapshot files will be found at: + backupDirPrefix = mkOption { + type = str; + default = "/var/lib/private/vault/snapshots"; + description = '' + The top level location to store the snapshots. The actual storage location + of the files will be this prefix path with the snapshot job name appended, + where the job is one of "hourly", "daily" or "custom". - $backupDirPrefix/$job/*.snap - ''; - }; + Therefore, saved snapshot files will be found at: - backupSuffix = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - Sets the saved snapshot filename with a descriptive suffix prior to the file - extension. This will enable selective snapshot job pruning. The form is: + $backupDirPrefix/$job/*.snap + ''; + }; - vault-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap - ''; - }; + backupSuffix = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + Sets the saved snapshot filename with a descriptive suffix prior to the file + extension. This will enable selective snapshot job pruning. The form is: - fixedRandomDelay = mkOption { - type = bool; - default = true; - description = '' - Makes randomizedDelaySec fixed between service restarts if true. - This will reduce jitter and allow the interval to remain fixed, - while still allowing start time randomization to avoid leader overload. - ''; - }; + vault-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap + ''; + }; - includeLeader = mkOption { - type = bool; - default = true; - description = '' - Whether to include the leader in the servers which will save snapshots. - This may reduce load on the leader slightly, but by default snapshot - saves are proxied through the leader anyway. + fixedRandomDelay = mkOption { + type = bool; + default = true; + description = '' + Makes randomizedDelaySec fixed between service restarts if true. + This will reduce jitter and allow the interval to remain fixed, + while still allowing start time randomization to avoid leader overload. + ''; + }; - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; + includeLeader = mkOption { + type = bool; + default = true; + description = '' + Whether to include the leader in the servers which will save snapshots. + This may reduce load on the leader slightly, but by default snapshot + saves are proxied through the leader anyway. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; - includeReplica = mkOption { - type = bool; - default = true; - description = '' - Whether to include the replicas in the servers which will save snapshots. + includeReplica = mkOption { + type = bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; - interval = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - The default onCalendar systemd timer string to trigger snapshot backups. - Any valid systemd OnCalendar string may be used here. Sensible - defaults for backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) - daily: 86400 randomizedDelaySec, 30 backupCount (1 month) - ''; - }; + interval = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + The default onCalendar systemd timer string to trigger snapshot backups. + Any valid systemd OnCalendar string may be used here. Sensible + defaults for backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) + daily: 86400 randomizedDelaySec, 30 backupCount (1 month) + ''; + }; - randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); - default = 0; - description = '' - A randomization period to be added to each systemd timer to avoid - leader overload. By default fixedRandomDelay will also be true to minimize - jitter and maintain fixed interval snapshots. Sensible defaults for - backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) - 86400 randomizedDelaySec for "daily" interval (1 day randomization) - ''; - }; + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); + default = 0; + description = '' + A randomization period to be added to each systemd timer to avoid + leader overload. By default fixedRandomDelay will also be true to minimize + jitter and maintain fixed interval snapshots. Sensible defaults for + backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) + 86400 randomizedDelaySec for "daily" interval (1 day randomization) + ''; + }; - owner = mkOption { - type = str; - default = "vault:vault"; - description = '' - The user and group to own the snapshot storage directory and snapshot files. - ''; - }; + owner = mkOption { + type = str; + default = "vault:vault"; + description = '' + The user and group to own the snapshot storage directory and snapshot files. + ''; + }; - vaultAddress = mkOption { - type = str; - default = "https://127.0.0.1:8200"; - description = '' - The local vault server address, including protocol and port. - ''; - }; + vaultAddress = mkOption { + type = str; + default = "https://127.0.0.1:8200"; + description = '' + The local vault server address, including protocol and port. + ''; }; }; + }; snapshotTimer = job: { partOf = ["vault-snapshots-${job}.service"]; diff --git a/profiles/vault/secrets-provisioning/hashistack.nix b/profiles/vault/secrets-provisioning/hashistack.nix index 2a150997..d28a6cfd 100644 --- a/profiles/vault/secrets-provisioning/hashistack.nix +++ b/profiles/vault/secrets-provisioning/hashistack.nix @@ -1,5 +1,10 @@ -{ config, lib, pkgs, hashiTokens, ... }: -let +{ + config, + lib, + pkgs, + hashiTokens, + ... +}: let roles = let agentCommand = runtimeInputs: namePrefix: cmds: let script = pkgs.writeShellApplication { @@ -17,9 +22,9 @@ let # in { command = [ "${script}/bin/${namePrefix}.sh" ]; }; reload = service: - agentCommand [ pkgs.systemd ] "reload-${service}" "systemctl --no-block try-reload-or-restart ${service} || true"; + agentCommand [pkgs.systemd] "reload-${service}" "systemctl --no-block try-reload-or-restart ${service} || true"; restart = service: - agentCommand [ pkgs.systemd ] "restart-${service}" "systemctl --no-block try-restart ${service} || true"; + agentCommand [pkgs.systemd] "restart-${service}" "systemctl --no-block try-restart ${service} || true"; in { core = rec { inherit reload restart; @@ -39,20 +44,18 @@ let ''; consulAgent = - if config.services.vault-agent.disableTokenRotation.consulAgent then - '' - {{ with secret "kv/bootstrap/static-tokens/cores/consul-server-agent" }}{{ .Data.data.token }}{{ end }}'' - else - '' - {{ with secret "consul/creds/consul-server-agent" }}{{ .Data.token }}{{ end }}''; + if config.services.vault-agent.disableTokenRotation.consulAgent + then '' + {{ with secret "kv/bootstrap/static-tokens/cores/consul-server-agent" }}{{ .Data.data.token }}{{ end }}'' + else '' + {{ with secret "consul/creds/consul-server-agent" }}{{ .Data.token }}{{ end }}''; consulDefault = - if config.services.vault-agent.disableTokenRotation.consulDefault then - '' - {{ with secret "kv/bootstrap/static-tokens/cores/consul-server-default" }}{{ .Data.data.token }}{{ end }}'' - else - '' - {{ with secret "consul/creds/consul-server-default" }}{{ .Data.token }}{{ end }}''; + if config.services.vault-agent.disableTokenRotation.consulDefault + then '' + {{ with secret "kv/bootstrap/static-tokens/cores/consul-server-default" }}{{ .Data.data.token }}{{ end }}'' + else '' + {{ with secret "consul/creds/consul-server-default" }}{{ .Data.token }}{{ end }}''; consulACL = '' { @@ -70,20 +73,18 @@ let inherit reload restart; consulAgent = - if config.services.vault-agent.disableTokenRotation.consulAgent then - '' - {{ with secret "kv/bootstrap/static-tokens/clients/consul-agent" }}{{ .Data.data.token }}{{ end }}'' - else - '' - {{ with secret "consul/creds/consul-agent" }}{{ .Data.token }}{{ end }}''; + if config.services.vault-agent.disableTokenRotation.consulAgent + then '' + {{ with secret "kv/bootstrap/static-tokens/clients/consul-agent" }}{{ .Data.data.token }}{{ end }}'' + else '' + {{ with secret "consul/creds/consul-agent" }}{{ .Data.token }}{{ end }}''; consulDefault = - if config.services.vault-agent.disableTokenRotation.consulDefault then - '' - {{ with secret "kv/bootstrap/static-tokens/clients/consul-default" }}{{ .Data.data.token }}{{ end }}'' - else - '' - {{ with secret "consul/creds/consul-default" }}{{ .Data.token }}{{ end }}''; + if config.services.vault-agent.disableTokenRotation.consulDefault + then '' + {{ with secret "kv/bootstrap/static-tokens/clients/consul-default" }}{{ .Data.data.token }}{{ end }}'' + else '' + {{ with secret "consul/creds/consul-default" }}{{ .Data.token }}{{ end }}''; consulNomad = consulDefault; @@ -112,8 +113,7 @@ let inherit (roles.client) consulAgent consulNomad; consulDefault = '' {{ with secret "consul/creds/consul-default" }}{{ .Data.token }}{{ end }}''; - traefik = - ''{{ with secret "consul/creds/traefik" }}{{ .Data.token }}{{ end }}''; + traefik = ''{{ with secret "consul/creds/traefik" }}{{ .Data.token }}{{ end }}''; consulACL = '' { @@ -131,8 +131,7 @@ let inherit (roles.client) consulAgent consulNomad; consulDefault = '' {{ with secret "consul/creds/consul-default" }}{{ .Data.token }}{{ end }}''; - traefik = - ''{{ with secret "consul/creds/traefik" }}{{ .Data.token }}{{ end }}''; + traefik = ''{{ with secret "consul/creds/traefik" }}{{ .Data.token }}{{ end }}''; consulACL = '' { @@ -153,13 +152,15 @@ let isRouting = roleName == "routing"; in { services.vault-agent = { - sinks = [{ - sink = { - type = "file"; - config = { path = hashiTokens.vault; }; - perms = "0644"; - }; - }]; + sinks = [ + { + sink = { + type = "file"; + config = {path = hashiTokens.vault;}; + perms = "0644"; + }; + } + ]; templates = { "${hashiTokens.consul-default}" = lib.mkIf config.services.consul.enable { @@ -172,27 +173,24 @@ in { contents = role.consulNomad; }; - "${hashiTokens.nomad-autoscaler}" = - lib.mkIf config.services.nomad-autoscaler.enable { - command = role.reload "nomad-autoscaler.service"; - contents = role.nomadAutoscaler; - }; + "${hashiTokens.nomad-autoscaler}" = lib.mkIf config.services.nomad-autoscaler.enable { + command = role.reload "nomad-autoscaler.service"; + contents = role.nomadAutoscaler; + }; - "${hashiTokens.nomad-snapshot}" = - lib.mkIf config.services.nomad-snapshots.enable { - contents = role.nomadSnapshot; - }; + "${hashiTokens.nomad-snapshot}" = lib.mkIf config.services.nomad-snapshots.enable { + contents = role.nomadSnapshot; + }; "${hashiTokens.consuld-json}" = lib.mkIf config.services.consul.enable { command = role.restart "consul.service"; contents = role.consulACL; }; - "${hashiTokens.nomadd-consul-json}" = - lib.mkIf (config.services.nomad.enable && isClient) { - command = role.restart "nomad.service"; - contents = role.nomadConsul; - }; + "${hashiTokens.nomadd-consul-json}" = lib.mkIf (config.services.nomad.enable && isClient) { + command = role.restart "nomad.service"; + contents = role.nomadConsul; + }; "${hashiTokens.traefik}" = lib.mkIf isRouting { contents = role.traefik; diff --git a/profiles/vault/secrets-provisioning/letsencrypt-ingress.nix b/profiles/vault/secrets-provisioning/letsencrypt-ingress.nix index f9326a88..767d7bbf 100644 --- a/profiles/vault/secrets-provisioning/letsencrypt-ingress.nix +++ b/profiles/vault/secrets-provisioning/letsencrypt-ingress.nix @@ -1,10 +1,13 @@ -{ config, lib, pkgs, letsencryptCertMaterial, ... }: let - +{ + config, + lib, + pkgs, + letsencryptCertMaterial, + ... +}: let # assumes: routing has uploaded letsencrypt cert material - reload = service: "${pkgs.systemd}/bin/systemctl --no-block try-reload-or-restart ${service}"; restart = service: "${pkgs.systemd}/bin/systemctl --no-block try-restart ${service}"; - in { services.vault-agent.templates = { ${letsencryptCertMaterial.certFile} = { From cebdb1949b1418d65efff6aa561a64c613b53768 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Wed, 20 Jul 2022 20:04:50 -0500 Subject: [PATCH 08/11] imp: switch to nushell for snapshot history rotation --- modules/consul-snapshots.nix | 19 ++++++++++--------- modules/nomad-snapshots.nix | 19 ++++++++++--------- modules/vault-snapshots.nix | 19 ++++++++++--------- overlay.nix | 1 + 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix index d17dc35a..80dd21c3 100644 --- a/modules/consul-snapshots.nix +++ b/modules/consul-snapshots.nix @@ -166,7 +166,7 @@ name = "consul-snapshot-${job}-script.sh"; script = pkgs.writeShellApplication { inherit name; - runtimeInputs = with pkgs; [consul coreutils findutils gawk hostname jq]; + runtimeInputs = with pkgs; [consul coreutils hostname jq nushell]; text = '' set -x @@ -201,14 +201,15 @@ takeConsulSnapshot fi - find "$BACKUP_DIR" \ - -type f \ - -name "*''${BACKUP_SUFFIX}.snap" \ - -printf "%T@ %p\n" \ - | sort -r -n \ - | tail -n +${toString (cfg.${job}.backupCount + 1)} \ - | awk '{print $2}' \ - | xargs -r rm + # shellcheck disable=SC2016 + nu -c ' + ls $"($env.BACKUP_DIR)" + | where name =~ $"($env.BACKUP_SUFFIX).snap$" + | where type == file + | sort-by modified + | drop ${toString cfg.${job}.backupCount} + | each {|f| rm $"($f.name)" | echo $"Deleted: ($f.name)"} + ' ''; }; in "${script}/bin/${name}"; diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix index 64d3d150..1a0d9e89 100644 --- a/modules/nomad-snapshots.nix +++ b/modules/nomad-snapshots.nix @@ -167,7 +167,7 @@ name = "nomad-snapshot-${job}-script.sh"; script = pkgs.writeShellApplication { inherit name; - runtimeInputs = with pkgs; [coreutils findutils gawk hostname jq nomad]; + runtimeInputs = with pkgs; [coreutils hostname jq nomad nushell]; text = '' set -x @@ -215,14 +215,15 @@ takeNomadSnapshot fi - find "$BACKUP_DIR" \ - -type f \ - -name "*''${BACKUP_SUFFIX}.snap" \ - -printf "%T@ %p\n" \ - | sort -r -n \ - | tail -n +${toString (cfg.${job}.backupCount + 1)} \ - | awk '{print $2}' \ - | xargs -r rm + # shellcheck disable=SC2016 + nu -c ' + ls $"($env.BACKUP_DIR)" + | where name =~ $"($env.BACKUP_SUFFIX).snap$" + | where type == file + | sort-by modified + | drop ${toString cfg.${job}.backupCount} + | each {|f| rm $"($f.name)" | echo $"Deleted: ($f.name)"} + ' ''; }; in "${script}/bin/${name}"; diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix index 8b9696f8..2e17840a 100644 --- a/modules/vault-snapshots.nix +++ b/modules/vault-snapshots.nix @@ -168,7 +168,7 @@ name = "vault-snapshot-${job}-script.sh"; script = pkgs.writeShellApplication { inherit name; - runtimeInputs = with pkgs; [coreutils findutils gawk hostname jq vault-bin]; + runtimeInputs = with pkgs; [coreutils hostname jq nushell vault-bin]; text = '' set -x @@ -216,14 +216,15 @@ takeVaultSnapshot fi - find "$BACKUP_DIR" \ - -type f \ - -name "*''${BACKUP_SUFFIX}.snap" \ - -printf "%T@ %p\n" \ - | sort -r -n \ - | tail -n +${toString (cfg.${job}.backupCount + 1)} \ - | awk '{print $2}' \ - | xargs -r rm + # shellcheck disable=SC2016 + nu -c ' + ls $"($env.BACKUP_DIR)" + | where name =~ $"($env.BACKUP_SUFFIX).snap$" + | where type == file + | sort-by modified + | drop ${toString cfg.${job}.backupCount} + | each {|f| rm $"($f.name)" | echo $"Deleted: ($f.name)"} + ' ''; }; in "${script}/bin/${name}"; diff --git a/overlay.nix b/overlay.nix index f1cca889..89d0355a 100644 --- a/overlay.nix +++ b/overlay.nix @@ -18,6 +18,7 @@ in (nixpkgs-unstable.legacyPackages.${prev.system}) grafana # 9.0.1 grafana-loki # 2.5.0 + nushell # 0.63.0 podman # 4.1.1 vector ; # 0.22.2 From 2e8876193206eb1adc1ee4b9064a3409e2dc466f Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 21 Jul 2022 15:34:30 -0500 Subject: [PATCH 09/11] cleanup: remove deprecated pkgs --- overlay.nix | 3 --- pkgs/filebeat.nix | 22 ---------------------- pkgs/haproxy-auth-request.nix | 22 ---------------------- pkgs/haproxy-cors.nix | 22 ---------------------- 4 files changed, 69 deletions(-) delete mode 100644 pkgs/filebeat.nix delete mode 100644 pkgs/haproxy-auth-request.nix delete mode 100644 pkgs/haproxy-cors.nix diff --git a/overlay.nix b/overlay.nix index 89d0355a..92c8a3d6 100644 --- a/overlay.nix +++ b/overlay.nix @@ -47,11 +47,8 @@ in # during metal deploy resulting in OCI jobs being killed or behaving unexpectedly inherit (nixpkgs-docker.legacyPackages.${prev.system}) docker containerd; # v20.10.15 - filebeat = final.callPackage ./pkgs/filebeat.nix {}; glusterfs = final.callPackage ./pkgs/glusterfs.nix {}; grpcdump = prev.callPackage ./pkgs/grpcdump.nix {}; - haproxy-auth-request = prev.callPackage ./pkgs/haproxy-auth-request.nix {}; - haproxy-cors = prev.callPackage ./pkgs/haproxy-cors.nix {}; mill = prev.callPackage ./pkgs/mill.nix {}; nomad = prev.callPackage ./pkgs/nomad.nix {buildGoModule = prev.buildGo117Module;}; nomad-autoscaler = prev.callPackage ./pkgs/nomad-autoscaler.nix {}; diff --git a/pkgs/filebeat.nix b/pkgs/filebeat.nix deleted file mode 100644 index 6baea6f1..00000000 --- a/pkgs/filebeat.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ - lib, - buildGoModule, - fetchFromGitHub, - systemd, -}: -buildGoModule rec { - pname = "beats"; - version = "7.12.0"; - - src = fetchFromGitHub { - owner = "elastic"; - repo = "beats"; - rev = "v${version}"; - sha256 = "sha256-CnTEZQ3exZSn8vhDguWhALLySh4lmXEPTWdAqWPU4bI="; - }; - - vendorSha256 = "sha256-Pd8jE7fAYQ/Js39X+8d1ojcGzxAg5MQkYqY2PB8CXa4="; - - subPackages = ["filebeat"]; - buildInputs = [systemd.dev]; -} diff --git a/pkgs/haproxy-auth-request.nix b/pkgs/haproxy-auth-request.nix deleted file mode 100644 index ba4cd23a..00000000 --- a/pkgs/haproxy-auth-request.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ - stdenv, - fetchgit, -}: -stdenv.mkDerivation { - pname = "haproxy-auth-request"; - version = "2020-08-23-unstable"; - - src = fetchgit { - url = "https://github.com/TimWolla/haproxy-auth-request.git"; - rev = "c3c9349166fb4aa9a9b3964267f3eaa03117c3a3"; - sha256 = "03vy7hj6xynclnshhmiydnisi6bfglnqkzrkja8snkiigcd9lab0"; - fetchSubmodules = true; - }; - - DESTDIR = placeholder "out"; - - postInstall = '' - mkdir -p $out/usr/share/haproxy/haproxy-lua-http - cp $out/usr/share/haproxy/http.lua $out/usr/share/haproxy/haproxy-lua-http/http.lua - ''; -} diff --git a/pkgs/haproxy-cors.nix b/pkgs/haproxy-cors.nix deleted file mode 100644 index c61f3522..00000000 --- a/pkgs/haproxy-cors.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ - stdenv, - fetchgit, -}: -stdenv.mkDerivation { - pname = "haproxy-lua-cors"; - version = "2020-10-22-unstable"; - - src = fetchgit { - url = "https://github.com/haproxytech/haproxy-lua-cors.git"; - rev = "0cd674749f98657f9a86dde2abacb4bb61eac438"; - sha256 = "15mnhlqa8ipy3bryvy7v9srnvzj0qqcsd3rjvagxn5658hg97r9d"; - fetchSubmodules = true; - }; - - DESTDIR = placeholder "out"; - - installPhase = '' - mkdir -p $out/usr/share/haproxy/haproxy-lua-cors - cp lib/cors.lua $out/usr/share/haproxy/haproxy-lua-cors/cors.lua - ''; -} From 6e12b9c40734310e4eddc3caa6c74f854ab126c0 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 21 Jul 2022 15:49:23 -0500 Subject: [PATCH 10/11] refactor: multiple snapshot modules into one hashi-snapshot module --- modules/consul-snapshots.nix | 297 ------------- modules/hashi-snapshots.nix | 416 ++++++++++++++++++ modules/nomad-snapshots.nix | 311 ------------- modules/vault-snapshots.nix | 312 ------------- profiles/consul/server.nix | 2 +- profiles/nomad/server.nix | 2 +- .../vault/secrets-provisioning/hashistack.nix | 2 +- profiles/vault/server.nix | 8 +- 8 files changed, 423 insertions(+), 927 deletions(-) delete mode 100644 modules/consul-snapshots.nix create mode 100644 modules/hashi-snapshots.nix delete mode 100644 modules/nomad-snapshots.nix delete mode 100644 modules/vault-snapshots.nix diff --git a/modules/consul-snapshots.nix b/modules/consul-snapshots.nix deleted file mode 100644 index 80dd21c3..00000000 --- a/modules/consul-snapshots.nix +++ /dev/null @@ -1,297 +0,0 @@ -{ - config, - pkgs, - lib, - ... -}: let - cfg = config.services.consul-snapshots; - - inherit (lib) boolToString mkEnableOption mkIf mkOption; - inherit (lib.types) addCheck attrs bool int str submodule; - - snapshotJobConfig = submodule { - options = { - enable = mkOption { - type = bool; - default = true; - description = '' - Creates a systemd service and timer to automatically save Consul snapshots. - ''; - }; - - backupCount = mkOption { - type = addCheck int (x: x >= 0); - default = null; - description = '' - The number of snapshots to keep. A sensible value matched to the onCalendar - interval parameter should be used. Examples of sensible suggestions may be: - - 48 backupCount for "hourly" interval (2 days of backups) - 30 backupCount for "daily" interval (1 month of backups) - ''; - }; - - backupDirPrefix = mkOption { - type = str; - default = "/var/lib/private/consul/snapshots"; - description = '' - The top level location to store the snapshots. The actual storage location - of the files will be this prefix path with the snapshot job name appended, - where the job is one of "hourly", "daily" or "custom". - - Therefore, saved snapshot files will be found at: - - $backupDirPrefix/$job/*.snap - ''; - }; - - backupSuffix = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - Sets the saved snapshot filename with a descriptive suffix prior to the file - extension. This will enable selective snapshot job pruning. The form is: - - consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap - ''; - }; - - consulAddress = mkOption { - type = str; - default = "http://127.0.0.1:8500"; - description = '' - The local consul server address, including protocol and port. - ''; - }; - - fixedRandomDelay = mkOption { - type = bool; - default = true; - description = '' - Makes randomizedDelaySec fixed between service restarts if true. - This will reduce jitter and allow the interval to remain fixed, - while still allowing start time randomization to avoid leader overload. - ''; - }; - - includeLeader = mkOption { - type = bool; - default = true; - description = '' - Whether to include the leader in the servers which will save snapshots. - This may reduce load on the leader slightly, but by default snapshot - saves are proxied through the leader anyway. - - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; - - includeReplica = mkOption { - type = bool; - default = true; - description = '' - Whether to include the replicas in the servers which will save snapshots. - - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; - - interval = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - The default onCalendar systemd timer string to trigger snapshot backups. - Any valid systemd OnCalendar string may be used here. Sensible - defaults for backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) - daily: 86400 randomizedDelaySec, 30 backupCount (1 month) - ''; - }; - - randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); - default = 0; - description = '' - A randomization period to be added to each systemd timer to avoid - leader overload. By default fixedRandomDelay will also be true to minimize - jitter and maintain fixed interval snapshots. Sensible defaults for - backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) - 86400 randomizedDelaySec for "daily" interval (1 day randomization) - ''; - }; - - owner = mkOption { - type = str; - default = "consul:consul"; - description = '' - The user and group to own the snapshot storage directory and snapshot files. - ''; - }; - }; - }; - - snapshotTimer = job: { - partOf = ["consul-snapshots-${job}.service"]; - timerConfig = { - OnCalendar = cfg.${job}.interval; - RandomizedDelaySec = cfg.${job}.randomizedDelaySec; - FixedRandomDelay = cfg.${job}.fixedRandomDelay; - AccuracySec = "1us"; - }; - wantedBy = ["timers.target"]; - }; - - snapshotService = job: { - environment = { - OWNER = cfg.${job}.owner; - BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; - BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; - INCLUDE_LEADER = boolToString cfg.${job}.includeLeader; - INCLUDE_REPLICA = boolToString cfg.${job}.includeReplica; - CONSUL_HTTP_ADDR = cfg.${job}.consulAddress; - }; - - serviceConfig = { - Type = "oneshot"; - Restart = "on-failure"; - RestartSec = "30s"; - ExecStart = let - name = "consul-snapshot-${job}-script.sh"; - script = pkgs.writeShellApplication { - inherit name; - runtimeInputs = with pkgs; [consul coreutils hostname jq nushell]; - text = '' - set -x - - SNAP_NAME="$BACKUP_DIR/consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" - - applyPerms () { - TARGET="$1" - PERMS="$2" - - chown "$OWNER" "$TARGET" - chmod "$PERMS" "$TARGET" - } - - takeConsulSnapshot () { - if [ ! -d "$BACKUP_DIR" ]; then - mkdir -p "$BACKUP_DIR" - applyPerms "$BACKUP_DIR" "0700" - fi - consul snapshot save "$SNAP_NAME" - applyPerms "$SNAP_NAME" "0400" - } - - if consul info | grep -E '^\s*leader\s+=\s+true$'; then - ROLE="leader" - else - ROLE="replica" - fi - - if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then - takeConsulSnapshot - elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then - takeConsulSnapshot - fi - - # shellcheck disable=SC2016 - nu -c ' - ls $"($env.BACKUP_DIR)" - | where name =~ $"($env.BACKUP_SUFFIX).snap$" - | where type == file - | sort-by modified - | drop ${toString cfg.${job}.backupCount} - | each {|f| rm $"($f.name)" | echo $"Deleted: ($f.name)"} - ' - ''; - }; - in "${script}/bin/${name}"; - }; - }; -in { - options = { - services.consul-snapshots = { - enable = mkEnableOption '' - Enable Consul snapshots. - - By default hourly snapshots will be taken and stored for 2 days on each consul server. - Modify services.consul-snapshots.hourly options to customize or disable. - - By default daily snapshots will be taken and stored for 1 month on each consul server. - Modify services.consul-snapshots.daily options to customize or disable. - - By default customized snapshots are disabled. - Modify services.consul-snapshots.custom options to enable and customize. - ''; - - defaultHourlyOpts = mkOption { - type = attrs; - internal = true; - default = { - enable = true; - backupCount = 48; - backupSuffix = "hourly"; - interval = "hourly"; - randomizedDelaySec = 3600; - }; - }; - - defaultDailyOpts = mkOption { - type = attrs; - internal = true; - default = { - enable = true; - backupCount = 30; - backupSuffix = "daily"; - interval = "daily"; - randomizedDelaySec = 86400; - }; - }; - - hourly = mkOption { - type = snapshotJobConfig; - default = cfg.defaultHourlyOpts; - }; - - daily = mkOption { - type = snapshotJobConfig; - default = cfg.defaultDailyOpts; - }; - - custom = mkOption { - type = snapshotJobConfig; - default = { - enable = false; - backupSuffix = "custom"; - }; - }; - }; - }; - - config = mkIf cfg.enable { - # Hourly snapshot configuration - systemd.timers.consul-snapshots-hourly = - mkIf cfg.hourly.enable (snapshotTimer "hourly"); - systemd.services.consul-snapshots-hourly = - mkIf cfg.hourly.enable (snapshotService "hourly"); - - # Daily snapshot configuration - systemd.timers.consul-snapshots-daily = - mkIf cfg.daily.enable (snapshotTimer "daily"); - systemd.services.consul-snapshots-daily = - mkIf cfg.daily.enable (snapshotService "daily"); - - # Custom snapshot configuration - systemd.timers.consul-snapshots-custom = - mkIf cfg.custom.enable (snapshotTimer "custom"); - systemd.services.consul-snapshots-custom = - mkIf cfg.custom.enable (snapshotService "custom"); - }; -} diff --git a/modules/hashi-snapshots.nix b/modules/hashi-snapshots.nix new file mode 100644 index 00000000..ad3886ee --- /dev/null +++ b/modules/hashi-snapshots.nix @@ -0,0 +1,416 @@ +{ + config, + pkgs, + lib, + hashiTokens, + ... +}: let + cfg = config.services.hashi-snapshots; + + inherit (lib) boolToString listToAttrs mkEnableOption mkIf mkMerge mkOption nameValuePair toUpper; + inherit (lib.types) addCheck attrs bool enum int str submodule; + + snapshotJobConfig = submodule { + options = { + enable = mkOption { + type = bool; + default = true; + description = '' + Creates a systemd service and timer to automatically save Vault snapshots. + ''; + }; + + backupCount = mkOption { + type = addCheck int (x: x >= 0); + default = null; + description = '' + The number of snapshots to keep. A sensible value matched to the onCalendar + interval parameter should be used. Examples of sensible suggestions may be: + + 48 backupCount for "hourly" interval (2 days of backups) + 30 backupCount for "daily" interval (1 month of backups) + ''; + }; + + backupDirPrefix = mkOption { + type = str; + default = null; + description = '' + The top level location to store the snapshots. The actual storage location + of the files will be this prefix path with the snapshot job name appended, + where the job is one of "hourly", "daily" or "custom". + + Therefore, saved snapshot files will be found at: + + $backupDirPrefix/$job/*.snap + ''; + }; + + backupSuffix = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + Sets the saved snapshot filename with a descriptive suffix prior to the file + extension. This will enable selective snapshot job pruning. The form is: + + $HASHI_SERVICE-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap + ''; + }; + + fixedRandomDelay = mkOption { + type = bool; + default = true; + description = '' + Makes randomizedDelaySec fixed between service restarts if true. + This will reduce jitter and allow the interval to remain fixed, + while still allowing start time randomization to avoid leader overload. + ''; + }; + + hashiAddress = mkOption { + type = str; + default = null; + description = '' + The local Hashicorp service server address, including protocol and port. + ''; + }; + + includeLeader = mkOption { + type = bool; + default = true; + description = '' + Whether to include the leader in the servers which will save snapshots. + This may reduce load on the leader slightly, but by default snapshot + saves are proxied through the leader anyway. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; + + includeReplica = mkOption { + type = bool; + default = true; + description = '' + Whether to include the replicas in the servers which will save snapshots. + + Reducing leader load from snapshots may be best done by fixed time + snapshot randomization so snapshot concurrency remains 1. + ''; + }; + + interval = mkOption { + type = addCheck str (x: x != ""); + default = null; + description = '' + The default onCalendar systemd timer string to trigger snapshot backups. + Any valid systemd OnCalendar string may be used here. Sensible + defaults for backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) + daily: 86400 randomizedDelaySec, 30 backupCount (1 month) + ''; + }; + + randomizedDelaySec = mkOption { + type = addCheck int (x: x >= 0); + default = 0; + description = '' + A randomization period to be added to each systemd timer to avoid + leader overload. By default fixedRandomDelay will also be true to minimize + jitter and maintain fixed interval snapshots. Sensible defaults for + backupCount and randomizedDelaySec should match this parameter. + Examples of sensible suggestions may be: + + 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) + 86400 randomizedDelaySec for "daily" interval (1 day randomization) + ''; + }; + + owner = mkOption { + type = str; + default = null; + description = '' + The user and group to own the snapshot storage directory and snapshot files. + ''; + }; + }; + }; + + snapshotTimer = hashiService: job: { + partOf = ["${hashiService}-snapshots-${job}.service"]; + timerConfig = { + OnCalendar = cfg.${hashiService}.${job}.interval; + RandomizedDelaySec = cfg.${hashiService}.${job}.randomizedDelaySec; + FixedRandomDelay = cfg.${hashiService}.${job}.fixedRandomDelay; + AccuracySec = "1us"; + }; + wantedBy = ["timers.target"]; + }; + + snapshotService = hashiService: job: { + environment = { + OWNER = cfg.${hashiService}.${job}.owner; + BACKUP_DIR = "${cfg.${hashiService}.${job}.backupDirPrefix}/${job}"; + BACKUP_SUFFIX = "-${cfg.${hashiService}.${job}.backupSuffix}"; + HASHI_SERVICE = hashiService; + INCLUDE_LEADER = boolToString cfg.${hashiService}.${job}.includeLeader; + INCLUDE_REPLICA = boolToString cfg.${hashiService}.${job}.includeReplica; + "${toUpper hashiService}_ADDR" = mkIf (hashiService != "consul") cfg.${hashiService}.${job}.hashiAddress; + "${toUpper hashiService}_HTTP_ADDR" = mkIf (hashiService == "consul") cfg.${hashiService}.${job}.hashiAddress; + "${toUpper hashiService}_FORMAT" = mkIf (hashiService == "vault") "json"; + }; + + serviceConfig = { + Type = "oneshot"; + Restart = "on-failure"; + RestartSec = "30s"; + ExecStart = let + name = "${hashiService}-snapshot-${job}-script.sh"; + script = snapshotScripts.${hashiService} job; + in "${script}/bin/${name}"; + }; + }; + + snapshotScripts = let + mkSnapshotScript = { + hashiService, + job, + extraInputs, + snapshotCmd, + envPrep, + roleCmd, + }: + pkgs.writeShellApplication { + name = "${hashiService}-snapshot-${job}-script.sh"; + runtimeInputs = with pkgs; [coreutils hostname nushell] ++ extraInputs; + text = '' + set -x + + SNAP_NAME="$BACKUP_DIR/$HASHI_SERVICE-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ$BACKUP_SUFFIX").snap" + + applyPerms () { + TARGET="$1" + PERMS="$2" + + chown "$OWNER" "$TARGET" + chmod "$PERMS" "$TARGET" + } + + takeSnapshot () { + if [ ! -d "$BACKUP_DIR" ]; then + mkdir -p "$BACKUP_DIR" + applyPerms "$BACKUP_DIR" "0700" + fi + ${snapshotCmd} + applyPerms "$SNAP_NAME" "0400" + } + + ${envPrep} + + if ${roleCmd}; then + ROLE="leader" + else + ROLE="replica" + fi + + if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then + takeSnapshot + elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then + takeSnapshot + fi + + # shellcheck disable=SC2016 + nu -c ' + ls $"($env.BACKUP_DIR)" + | where name =~ $"($env.BACKUP_SUFFIX).snap$" + | where type == file + | sort-by modified + | drop ${toString cfg.${hashiService}.${job}.backupCount} + | each {|f| rm $"($f.name)"; echo $"Deleted: ($f.name)"} + ' + ''; + }; + in { + consul = job: + mkSnapshotScript { + inherit job; + hashiService = "consul"; + extraInputs = with pkgs; [consul gnugrep]; + snapshotCmd = ''consul snapshot save "$SNAP_NAME"''; + envPrep = ""; + roleCmd = "consul info | grep -E '^\\s*leader\\s+=\\s+true$'"; + }; + + nomad = job: + mkSnapshotScript { + inherit job; + hashiService = "nomad"; + extraInputs = with pkgs; [jq nomad]; + snapshotCmd = ''nomad operator snapshot save "$SNAP_NAME"''; + envPrep = '' + set +x + NOMAD_TOKEN="$(< ${hashiTokens.nomad-snapshot})" + export NOMAD_TOKEN + set -x + + STATUS="$(nomad agent-info --json)" + ''; + roleCmd = ''jq -e '(.stats.nomad.leader // "false") == "true"' <<< "$STATUS"''; + }; + + vault = job: + mkSnapshotScript { + inherit job; + hashiService = "vault"; + extraInputs = with pkgs; [jq vault-bin]; + snapshotCmd = ''vault operator raft snapshot save "$SNAP_NAME"''; + envPrep = '' + set +x + VAULT_TOKEN="$(< ${hashiTokens.vault})" + export VAULT_TOKEN + set -x + + STATUS="$(vault status)" + + if jq -e '.storage_type != "raft"' <<< "$STATUS"; then + echo "Vault storage backend is not raft." + echo "Ensure the appropriate storage backend is being snapshotted." + exit 0 + fi + ''; + roleCmd = ''jq -e '(.is_self // false) == true' <<< "$STATUS"''; + }; + }; +in { + options = let + snapshotDescription = hashiService: '' + By default hourly snapshots will be taken and stored for 2 days on each snapshotted server. + Modify services.hashi-snapshots.${hashiService}.hourly options to customize or disable. + + By default daily snapshots will be taken and stored for 1 month on each snapshotted server. + Modify services.hashi-snapshots.${hashiService}.daily options to customize or disable. + + By default customized snapshots are disabled. + Modify services.hashi-snapshots.${hashiService}.custom options to enable and customize. + ''; + in { + services.hashi-snapshots = + { + enableConsul = mkEnableOption '' + Enable Consul snapshots. + ${snapshotDescription "consul"} + ''; + + enableNomad = mkEnableOption '' + Enable Nomad snapshots. + ${snapshotDescription "nomad"} + ''; + + enableVault = mkEnableOption '' + Enable Vault snapshots. + ${snapshotDescription "vault"} + ''; + + defaultHashiOpts = mkOption { + type = attrs; + internal = true; + default = { + consul = { + backupDirPrefix = "/var/lib/private/consul/snapshots"; + hashiAddress = "http://127.0.0.1:8500"; + owner = "consul:consul"; + }; + nomad = { + backupDirPrefix = "/var/lib/private/nomad/snapshots"; + hashiAddress = "https://127.0.0.1:4646"; + owner = "root:root"; + }; + vault = { + backupDirPrefix = "/var/lib/private/vault/snapshots"; + hashiAddress = "https://127.0.0.1:8200"; + owner = "vault:vault"; + }; + }; + }; + + defaultHourlyOpts = mkOption { + type = attrs; + internal = true; + default = { + enable = true; + backupCount = 48; + backupSuffix = "hourly"; + interval = "hourly"; + randomizedDelaySec = 3600; + }; + }; + + defaultDailyOpts = mkOption { + type = attrs; + internal = true; + default = { + enable = true; + backupCount = 30; + backupSuffix = "daily"; + interval = "daily"; + randomizedDelaySec = 86400; + }; + }; + } + // ( + listToAttrs + (map + ( + hashiService: + nameValuePair + hashiService + { + hourly = mkOption { + type = snapshotJobConfig; + default = cfg.defaultHourlyOpts // cfg.defaultHashiOpts.${hashiService}; + }; + + daily = mkOption { + type = snapshotJobConfig; + default = cfg.defaultDailyOpts // cfg.defaultHashiOpts.${hashiService}; + }; + + custom = mkOption { + type = snapshotJobConfig; + default = + { + enable = false; + backupSuffix = "custom"; + } + // cfg.defaultHashiOpts.${hashiService}; + }; + } + ) + ["consul" "nomad" "vault"]) + ); + }; + + config = let + mkSnapshotJobSet = hashiService: job: { + systemd.timers."${hashiService}-snapshots-${job}" = + mkIf cfg.${hashiService}.${job}.enable (snapshotTimer hashiService job); + + systemd.services."${hashiService}-snapshots-${job}" = + mkIf cfg.${hashiService}.${job}.enable (snapshotService hashiService job); + }; + + mkSnapshotServices = hashiService: + mkMerge [ + (mkSnapshotJobSet hashiService "hourly") + (mkSnapshotJobSet hashiService "daily") + (mkSnapshotJobSet hashiService "custom") + ]; + in + mkMerge [ + (mkIf cfg.enableConsul (mkSnapshotServices "consul")) + (mkIf cfg.enableNomad (mkSnapshotServices "nomad")) + (mkIf cfg.enableVault (mkSnapshotServices "vault")) + ]; +} diff --git a/modules/nomad-snapshots.nix b/modules/nomad-snapshots.nix deleted file mode 100644 index 1a0d9e89..00000000 --- a/modules/nomad-snapshots.nix +++ /dev/null @@ -1,311 +0,0 @@ -{ - config, - pkgs, - lib, - hashiTokens, - ... -}: let - cfg = config.services.nomad-snapshots; - - inherit (lib) boolToString mkEnableOption mkIf mkOption; - inherit (lib.types) addCheck attrs bool int str submodule; - - snapshotJobConfig = submodule { - options = { - enable = mkOption { - type = bool; - default = true; - description = '' - Creates a systemd service and timer to automatically save Nomad snapshots. - ''; - }; - - backupCount = mkOption { - type = addCheck int (x: x >= 0); - default = null; - description = '' - The number of snapshots to keep. A sensible value matched to the onCalendar - interval parameter should be used. Examples of sensible suggestions may be: - - 48 backupCount for "hourly" interval (2 days of backups) - 30 backupCount for "daily" interval (1 month of backups) - ''; - }; - - backupDirPrefix = mkOption { - type = str; - default = "/var/lib/private/nomad/snapshots"; - description = '' - The top level location to store the snapshots. The actual storage location - of the files will be this prefix path with the snapshot job name appended, - where the job is one of "hourly", "daily" or "custom". - - Therefore, saved snapshot files will be found at: - - $backupDirPrefix/$job/*.snap - ''; - }; - - backupSuffix = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - Sets the saved snapshot filename with a descriptive suffix prior to the file - extension. This will enable selective snapshot job pruning. The form is: - - nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap - ''; - }; - - fixedRandomDelay = mkOption { - type = bool; - default = true; - description = '' - Makes randomizedDelaySec fixed between service restarts if true. - This will reduce jitter and allow the interval to remain fixed, - while still allowing start time randomization to avoid leader overload. - ''; - }; - - includeLeader = mkOption { - type = bool; - default = true; - description = '' - Whether to include the leader in the servers which will save snapshots. - This may reduce load on the leader slightly, but by default snapshot - saves are proxied through the leader anyway. - - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; - - includeReplica = mkOption { - type = bool; - default = true; - description = '' - Whether to include the replicas in the servers which will save snapshots. - - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; - - interval = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - The default onCalendar systemd timer string to trigger snapshot backups. - Any valid systemd OnCalendar string may be used here. Sensible - defaults for backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) - daily: 86400 randomizedDelaySec, 30 backupCount (1 month) - ''; - }; - - randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); - default = 0; - description = '' - A randomization period to be added to each systemd timer to avoid - leader overload. By default fixedRandomDelay will also be true to minimize - jitter and maintain fixed interval snapshots. Sensible defaults for - backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) - 86400 randomizedDelaySec for "daily" interval (1 day randomization) - ''; - }; - - owner = mkOption { - type = str; - default = "root:root"; - description = '' - The user and group to own the snapshot storage directory and snapshot files. - ''; - }; - - nomadAddress = mkOption { - type = str; - default = "https://127.0.0.1:4646"; - description = '' - The local nomad server address, including protocol and port. - ''; - }; - }; - }; - - snapshotTimer = job: { - partOf = ["nomad-snapshots-${job}.service"]; - timerConfig = { - OnCalendar = cfg.${job}.interval; - RandomizedDelaySec = cfg.${job}.randomizedDelaySec; - FixedRandomDelay = cfg.${job}.fixedRandomDelay; - AccuracySec = "1us"; - }; - wantedBy = ["timers.target"]; - }; - - snapshotService = job: { - environment = { - OWNER = cfg.${job}.owner; - BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; - BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; - INCLUDE_LEADER = boolToString cfg.${job}.includeLeader; - INCLUDE_REPLICA = boolToString cfg.${job}.includeReplica; - NOMAD_ADDR = cfg.${job}.nomadAddress; - }; - - serviceConfig = { - Type = "oneshot"; - Restart = "on-failure"; - RestartSec = "30s"; - ExecStart = let - name = "nomad-snapshot-${job}-script.sh"; - script = pkgs.writeShellApplication { - inherit name; - runtimeInputs = with pkgs; [coreutils hostname jq nomad nushell]; - text = '' - set -x - - SNAP_NAME="$BACKUP_DIR/nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" - - applyPerms () { - TARGET="$1" - PERMS="$2" - - chown "$OWNER" "$TARGET" - chmod "$PERMS" "$TARGET" - } - - takeNomadSnapshot () { - if [ ! -d "$BACKUP_DIR" ]; then - mkdir -p "$BACKUP_DIR" - applyPerms "$BACKUP_DIR" "0700" - fi - nomad operator snapshot save "$SNAP_NAME" - applyPerms "$SNAP_NAME" "0400" - } - - if [ ! -f ${hashiTokens.nomad-snapshot} ]; then - echo "Suitable nomad token for snapshotting not found." - echo "Ensure the appropriate token for snapshotting is available."; - exit 0; - else - set +x - NOMAD_TOKEN="$(< ${hashiTokens.nomad-snapshot})" - export NOMAD_TOKEN - set -x - fi - - STATUS="$(nomad agent-info --json)" - - if jq -e '(.stats.nomad.leader // "false") == "true"' <<< "$STATUS"; then - ROLE="leader" - else - ROLE="replica" - fi - - if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then - takeNomadSnapshot - elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then - takeNomadSnapshot - fi - - # shellcheck disable=SC2016 - nu -c ' - ls $"($env.BACKUP_DIR)" - | where name =~ $"($env.BACKUP_SUFFIX).snap$" - | where type == file - | sort-by modified - | drop ${toString cfg.${job}.backupCount} - | each {|f| rm $"($f.name)" | echo $"Deleted: ($f.name)"} - ' - ''; - }; - in "${script}/bin/${name}"; - }; - }; -in { - options = { - services.nomad-snapshots = { - enable = mkEnableOption '' - Enable Nomad snapshots. - - By default hourly snapshots will be taken and stored for 2 days on each nomad server. - Modify services.nomad-snapshots.hourly options to customize or disable. - - By default daily snapshots will be taken and stored for 1 month on each nomad server. - Modify services.nomad-snapshots.daily options to customize or disable. - - By default customized snapshots are disabled. - Modify services.nomad-snapshots.custom options to enable and customize. - ''; - - defaultHourlyOpts = mkOption { - type = attrs; - internal = true; - default = { - enable = true; - backupCount = 48; - backupSuffix = "hourly"; - interval = "hourly"; - randomizedDelaySec = 3600; - }; - }; - - defaultDailyOpts = mkOption { - type = attrs; - internal = true; - default = { - enable = true; - backupCount = 30; - backupSuffix = "daily"; - interval = "daily"; - randomizedDelaySec = 86400; - }; - }; - - hourly = mkOption { - type = snapshotJobConfig; - default = cfg.defaultHourlyOpts; - }; - - daily = mkOption { - type = snapshotJobConfig; - default = cfg.defaultDailyOpts; - }; - - custom = mkOption { - type = snapshotJobConfig; - default = { - enable = false; - backupSuffix = "custom"; - }; - }; - }; - }; - - config = mkIf cfg.enable { - # Hourly snapshot configuration - systemd.timers.nomad-snapshots-hourly = - mkIf cfg.hourly.enable (snapshotTimer "hourly"); - systemd.services.nomad-snapshots-hourly = - mkIf cfg.hourly.enable (snapshotService "hourly"); - - # Daily snapshot configuration - systemd.timers.nomad-snapshots-daily = - mkIf cfg.daily.enable (snapshotTimer "daily"); - systemd.services.nomad-snapshots-daily = - mkIf cfg.daily.enable (snapshotService "daily"); - - # Custom snapshot configuration - systemd.timers.nomad-snapshots-custom = - mkIf cfg.custom.enable (snapshotTimer "custom"); - systemd.services.nomad-snapshots-custom = - mkIf cfg.custom.enable (snapshotService "custom"); - }; -} diff --git a/modules/vault-snapshots.nix b/modules/vault-snapshots.nix deleted file mode 100644 index 2e17840a..00000000 --- a/modules/vault-snapshots.nix +++ /dev/null @@ -1,312 +0,0 @@ -{ - config, - pkgs, - lib, - hashiTokens, - ... -}: let - cfg = config.services.vault-snapshots; - - inherit (lib) boolToString mkEnableOption mkIf mkOption; - inherit (lib.types) addCheck attrs bool int str submodule; - - snapshotJobConfig = submodule { - options = { - enable = mkOption { - type = bool; - default = true; - description = '' - Creates a systemd service and timer to automatically save Vault snapshots. - ''; - }; - - backupCount = mkOption { - type = addCheck int (x: x >= 0); - default = null; - description = '' - The number of snapshots to keep. A sensible value matched to the onCalendar - interval parameter should be used. Examples of sensible suggestions may be: - - 48 backupCount for "hourly" interval (2 days of backups) - 30 backupCount for "daily" interval (1 month of backups) - ''; - }; - - backupDirPrefix = mkOption { - type = str; - default = "/var/lib/private/vault/snapshots"; - description = '' - The top level location to store the snapshots. The actual storage location - of the files will be this prefix path with the snapshot job name appended, - where the job is one of "hourly", "daily" or "custom". - - Therefore, saved snapshot files will be found at: - - $backupDirPrefix/$job/*.snap - ''; - }; - - backupSuffix = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - Sets the saved snapshot filename with a descriptive suffix prior to the file - extension. This will enable selective snapshot job pruning. The form is: - - vault-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ")-$backupSuffix.snap - ''; - }; - - fixedRandomDelay = mkOption { - type = bool; - default = true; - description = '' - Makes randomizedDelaySec fixed between service restarts if true. - This will reduce jitter and allow the interval to remain fixed, - while still allowing start time randomization to avoid leader overload. - ''; - }; - - includeLeader = mkOption { - type = bool; - default = true; - description = '' - Whether to include the leader in the servers which will save snapshots. - This may reduce load on the leader slightly, but by default snapshot - saves are proxied through the leader anyway. - - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; - - includeReplica = mkOption { - type = bool; - default = true; - description = '' - Whether to include the replicas in the servers which will save snapshots. - - Reducing leader load from snapshots may be best done by fixed time - snapshot randomization so snapshot concurrency remains 1. - ''; - }; - - interval = mkOption { - type = addCheck str (x: x != ""); - default = null; - description = '' - The default onCalendar systemd timer string to trigger snapshot backups. - Any valid systemd OnCalendar string may be used here. Sensible - defaults for backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - hourly: 3600 randomizedDelaySec, 48 backupCount (2 days) - daily: 86400 randomizedDelaySec, 30 backupCount (1 month) - ''; - }; - - randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); - default = 0; - description = '' - A randomization period to be added to each systemd timer to avoid - leader overload. By default fixedRandomDelay will also be true to minimize - jitter and maintain fixed interval snapshots. Sensible defaults for - backupCount and randomizedDelaySec should match this parameter. - Examples of sensible suggestions may be: - - 3600 randomizedDelaySec for "hourly" interval (1 hr randomization) - 86400 randomizedDelaySec for "daily" interval (1 day randomization) - ''; - }; - - owner = mkOption { - type = str; - default = "vault:vault"; - description = '' - The user and group to own the snapshot storage directory and snapshot files. - ''; - }; - - vaultAddress = mkOption { - type = str; - default = "https://127.0.0.1:8200"; - description = '' - The local vault server address, including protocol and port. - ''; - }; - }; - }; - - snapshotTimer = job: { - partOf = ["vault-snapshots-${job}.service"]; - timerConfig = { - OnCalendar = cfg.${job}.interval; - RandomizedDelaySec = cfg.${job}.randomizedDelaySec; - FixedRandomDelay = cfg.${job}.fixedRandomDelay; - AccuracySec = "1us"; - }; - wantedBy = ["timers.target"]; - }; - - snapshotService = job: { - environment = { - OWNER = cfg.${job}.owner; - BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}"; - BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}"; - INCLUDE_LEADER = boolToString cfg.${job}.includeLeader; - INCLUDE_REPLICA = boolToString cfg.${job}.includeReplica; - VAULT_ADDR = cfg.${job}.vaultAddress; - VAULT_FORMAT = "json"; - }; - - serviceConfig = { - Type = "oneshot"; - Restart = "on-failure"; - RestartSec = "30s"; - ExecStart = let - name = "vault-snapshot-${job}-script.sh"; - script = pkgs.writeShellApplication { - inherit name; - runtimeInputs = with pkgs; [coreutils hostname jq nushell vault-bin]; - text = '' - set -x - - SNAP_NAME="$BACKUP_DIR/vault-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap" - - applyPerms () { - TARGET="$1" - PERMS="$2" - - chown "$OWNER" "$TARGET" - chmod "$PERMS" "$TARGET" - } - - takeVaultSnapshot () { - if [ ! -d "$BACKUP_DIR" ]; then - mkdir -p "$BACKUP_DIR" - applyPerms "$BACKUP_DIR" 0700 - fi - vault operator raft snapshot save "$SNAP_NAME" - applyPerms "$SNAP_NAME" 0400 - } - - set +x - VAULT_TOKEN="$(< ${hashiTokens.vault})" - export VAULT_TOKEN - set -x - - STATUS="$(vault status)" - - if jq -e '.storage_type != "raft"' <<< "$STATUS"; then - echo "Vault storage backend is not raft." - echo "Ensure the appropriate storage backend is being snapshotted." - exit 0 - fi - - if jq -e '(.is_self // false) == true' <<< "$STATUS"; then - ROLE="leader" - else - ROLE="replica" - fi - - if [ "$ROLE" = "leader" ] && [ "$INCLUDE_LEADER" = "true" ]; then - takeVaultSnapshot - elif [ "$ROLE" = "replica" ] && [ "$INCLUDE_REPLICA" = "true" ]; then - takeVaultSnapshot - fi - - # shellcheck disable=SC2016 - nu -c ' - ls $"($env.BACKUP_DIR)" - | where name =~ $"($env.BACKUP_SUFFIX).snap$" - | where type == file - | sort-by modified - | drop ${toString cfg.${job}.backupCount} - | each {|f| rm $"($f.name)" | echo $"Deleted: ($f.name)"} - ' - ''; - }; - in "${script}/bin/${name}"; - }; - }; -in { - options = { - services.vault-snapshots = { - enable = mkEnableOption '' - Enable Vault snapshots. - - By default hourly snapshots will be taken and stored for 2 days on each vault server. - Modify services.vault-snapshots.hourly options to customize or disable. - - By default daily snapshots will be taken and stored for 1 month on each vault server. - Modify services.vault-snapshots.daily options to customize or disable. - - By default customized snapshots are disabled. - Modify services.vault-snapshots.custom options to enable and customize. - ''; - - defaultHourlyOpts = mkOption { - type = attrs; - internal = true; - default = { - enable = true; - backupCount = 48; - backupSuffix = "hourly"; - interval = "hourly"; - randomizedDelaySec = 3600; - }; - }; - - defaultDailyOpts = mkOption { - type = attrs; - internal = true; - default = { - enable = true; - backupCount = 30; - backupSuffix = "daily"; - interval = "daily"; - randomizedDelaySec = 86400; - }; - }; - - hourly = mkOption { - type = snapshotJobConfig; - default = cfg.defaultHourlyOpts; - }; - - daily = mkOption { - type = snapshotJobConfig; - default = cfg.defaultDailyOpts; - }; - - custom = mkOption { - type = snapshotJobConfig; - default = { - enable = false; - backupSuffix = "custom"; - }; - }; - }; - }; - - config = mkIf cfg.enable { - # Hourly snapshot configuration - systemd.timers.vault-snapshots-hourly = - mkIf cfg.hourly.enable (snapshotTimer "hourly"); - systemd.services.vault-snapshots-hourly = - mkIf cfg.hourly.enable (snapshotService "hourly"); - - # Daily snapshot configuration - systemd.timers.vault-snapshots-daily = - mkIf cfg.daily.enable (snapshotTimer "daily"); - systemd.services.vault-snapshots-daily = - mkIf cfg.daily.enable (snapshotService "daily"); - - # Custom snapshot configuration - systemd.timers.vault-snapshots-custom = - mkIf cfg.custom.enable (snapshotTimer "custom"); - systemd.services.vault-snapshots-custom = - mkIf cfg.custom.enable (snapshotService "custom"); - }; -} diff --git a/profiles/consul/server.nix b/profiles/consul/server.nix index cc9f65e5..f4a73ae5 100644 --- a/profiles/consul/server.nix +++ b/profiles/consul/server.nix @@ -8,7 +8,7 @@ Imports = {imports = [./common.nix ./policies.nix];}; Switches = { - services.consul-snapshots.enable = true; + services.hashi-snapshots.enableConsul = true; services.consul.server = true; services.consul.ui = true; }; diff --git a/profiles/nomad/server.nix b/profiles/nomad/server.nix index 18a62323..dd4583f2 100644 --- a/profiles/nomad/server.nix +++ b/profiles/nomad/server.nix @@ -8,7 +8,7 @@ Switches = { services.nomad.server.enabled = true; - services.nomad-snapshots.enable = true; + services.hashi-snapshots.enableNomad = true; }; Config = let diff --git a/profiles/vault/secrets-provisioning/hashistack.nix b/profiles/vault/secrets-provisioning/hashistack.nix index d28a6cfd..2a9a0e1f 100644 --- a/profiles/vault/secrets-provisioning/hashistack.nix +++ b/profiles/vault/secrets-provisioning/hashistack.nix @@ -178,7 +178,7 @@ in { contents = role.nomadAutoscaler; }; - "${hashiTokens.nomad-snapshot}" = lib.mkIf config.services.nomad-snapshots.enable { + "${hashiTokens.nomad-snapshot}" = lib.mkIf config.services.hashi-snapshots.enableNomad { contents = role.nomadSnapshot; }; diff --git a/profiles/vault/server.nix b/profiles/vault/server.nix index c3b51f63..6eb1a20e 100644 --- a/profiles/vault/server.nix +++ b/profiles/vault/server.nix @@ -13,15 +13,15 @@ }; Switches = let - cfg = config.services.vault-snapshots; + cfg = config.services.hashi-snapshots; in { services.vault.enable = true; - services.vault-snapshots.enable = true; + services.hashi-snapshots.enableVault = true; # Do not include vault replicas in snapshotting until replica snapshot bug is resolved # Ref: https://github.com/hashicorp/vault/issues/15258 - services.vault-snapshots.hourly = cfg.defaultHourlyOpts // { includeReplica = false; }; - services.vault-snapshots.daily = cfg.defaultDailyOpts // { includeReplica = false; }; + services.hashi-snapshots.vault.hourly = cfg.defaultHourlyOpts // cfg.defaultHashiOpts.vault // {includeReplica = false;}; + services.hashi-snapshots.vault.daily = cfg.defaultDailyOpts // cfg.defaultHashiOpts.vault // {includeReplica = false;}; services.vault.ui = true; }; From 8875afc95ff185e690bcda7d09326d75c80839fa Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 26 Jul 2022 12:12:57 -0500 Subject: [PATCH 11/11] cleanup: pr suggestions --- modules/hashi-snapshots.nix | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/hashi-snapshots.nix b/modules/hashi-snapshots.nix index ad3886ee..05d5ead1 100644 --- a/modules/hashi-snapshots.nix +++ b/modules/hashi-snapshots.nix @@ -8,7 +8,7 @@ cfg = config.services.hashi-snapshots; inherit (lib) boolToString listToAttrs mkEnableOption mkIf mkMerge mkOption nameValuePair toUpper; - inherit (lib.types) addCheck attrs bool enum int str submodule; + inherit (lib.types) attrs bool enum int ints nonEmptyStr str submodule; snapshotJobConfig = submodule { options = { @@ -21,7 +21,7 @@ }; backupCount = mkOption { - type = addCheck int (x: x >= 0); + type = ints.unsigned; default = null; description = '' The number of snapshots to keep. A sensible value matched to the onCalendar @@ -47,7 +47,7 @@ }; backupSuffix = mkOption { - type = addCheck str (x: x != ""); + type = nonEmptyStr; default = null; description = '' Sets the saved snapshot filename with a descriptive suffix prior to the file @@ -100,7 +100,7 @@ }; interval = mkOption { - type = addCheck str (x: x != ""); + type = nonEmptyStr; default = null; description = '' The default onCalendar systemd timer string to trigger snapshot backups. @@ -114,7 +114,7 @@ }; randomizedDelaySec = mkOption { - type = addCheck int (x: x >= 0); + type = ints.unsigned; default = 0; description = '' A randomization period to be added to each systemd timer to avoid @@ -251,11 +251,11 @@ snapshotCmd = ''nomad operator snapshot save "$SNAP_NAME"''; envPrep = '' set +x - NOMAD_TOKEN="$(< ${hashiTokens.nomad-snapshot})" + NOMAD_TOKEN=$(< ${hashiTokens.nomad-snapshot}) export NOMAD_TOKEN set -x - STATUS="$(nomad agent-info --json)" + STATUS=$(nomad agent-info --json) ''; roleCmd = ''jq -e '(.stats.nomad.leader // "false") == "true"' <<< "$STATUS"''; }; @@ -268,11 +268,11 @@ snapshotCmd = ''vault operator raft snapshot save "$SNAP_NAME"''; envPrep = '' set +x - VAULT_TOKEN="$(< ${hashiTokens.vault})" + VAULT_TOKEN=$(< ${hashiTokens.vault}) export VAULT_TOKEN set -x - STATUS="$(vault status)" + STATUS=$(vault status) if jq -e '.storage_type != "raft"' <<< "$STATUS"; then echo "Vault storage backend is not raft."