Backup Monitoring

#!/bin/bash
set -o pipefail  # fail if pg_dump fails, not just gzip
WEBHOOK="https://app.alphorn.dev/n/V1StGXR8_Z5jdHi6B-myT"
ALPHORN_API_KEY="${ALPHORN_API_KEY:-alp_...}"  # or export it in the environment
DB_NAME="myapp"
BACKUP_DIR="/backups"
BACKUP_FILE="$BACKUP_DIR/${DB_NAME}_$(date +%Y%m%d_%H%M%S).sql.gz"
START_TIME=$(date +%s)

# Run backup
pg_dump -U postgres "$DB_NAME" | gzip > "$BACKUP_FILE"
EXIT_CODE=$?
END_TIME=$(date +%s)
DURATION=$(( END_TIME - START_TIME ))

if [ $EXIT_CODE -eq 0 ]; then
  SIZE=$(du -sh "$BACKUP_FILE" | cut -f1)
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{
      \"title\": \"Backup completed: $DB_NAME\",
      \"message\": \"Size: $SIZE, Duration: ${DURATION}s\nFile: $BACKUP_FILE\",
      \"priority\": 2,
      \"tags\": [\"backup\", \"database\", \"success\"]
    }"
else
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{
      \"title\": \"Backup FAILED: $DB_NAME\",
      \"message\": \"pg_dump exited with code $EXIT_CODE after ${DURATION}s\",
      \"priority\": 5,
      \"tags\": [\"backup\", \"database\", \"failure\"]
    }"
fi

# Clean up old backups (keep last 7) - only after a successful run,
# so failed runs never push good backups out
if [ $EXIT_CODE -eq 0 ]; then
  ls -t "$BACKUP_DIR"/${DB_NAME}_*.sql.gz | tail -n +8 | xargs rm -f
fi

#!/bin/bash
set -o pipefail  # fail if mysqldump fails, not just gzip
WEBHOOK="https://app.alphorn.dev/n/V1StGXR8_Z5jdHi6B-myT"
ALPHORN_API_KEY="${ALPHORN_API_KEY:-alp_...}"  # or export it in the environment
DB_NAME="myapp"
BACKUP_FILE="/backups/${DB_NAME}_$(date +%Y%m%d).sql.gz"

mysqldump -u root "$DB_NAME" | gzip > "$BACKUP_FILE"

if [ $? -eq 0 ]; then
  SIZE=$(du -sh "$BACKUP_FILE" | cut -f1)
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{\"title\":\"MySQL backup OK: $DB_NAME\",\"message\":\"Size: $SIZE\",\"priority\":2,\"tags\":[\"backup\",\"success\"]}"
else
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{\"title\":\"MySQL backup FAILED: $DB_NAME\",\"message\":\"mysqldump or gzip failed, check the server logs\",\"priority\":5,\"tags\":[\"backup\",\"failure\"]}"
fi

EXPECTED_MIN_MB=100
ACTUAL_MB=$(du -m "$BACKUP_FILE" | cut -f1)

if [ "$ACTUAL_MB" -lt "$EXPECTED_MIN_MB" ]; then
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{
      \"title\": \"Backup size anomaly: $DB_NAME\",
      \"message\": \"Backup is only ${ACTUAL_MB}MB (expected at least ${EXPECTED_MIN_MB}MB). Possible data loss.\",
      \"priority\": 5,
      \"tags\": [\"backup\", \"anomaly\"]
    }"
fi

#!/bin/bash
WEBHOOK="https://app.alphorn.dev/n/V1StGXR8_Z5jdHi6B-myT"
ALPHORN_API_KEY="${ALPHORN_API_KEY:-alp_...}"  # or export it in the environment
REPO="/backups/restic-repo"
PATHS="/home /etc /var/lib"
START_TIME=$(date +%s)

OUTPUT=$(restic -r "$REPO" backup $PATHS 2>&1)
EXIT_CODE=$?
END_TIME=$(date +%s)
DURATION=$(( END_TIME - START_TIME ))

if [ $EXIT_CODE -eq 0 ]; then
  # Extract snapshot summary from restic output
  SUMMARY=$(echo "$OUTPUT" | tail -3)
  # jq escapes the multi-line output into valid JSON
  jq -n --arg d "$DURATION" --arg s "$SUMMARY" \
    '{title: "Restic backup completed", message: "Duration: \($d)s\n\($s)", priority: 2, tags: ["backup", "restic", "success"]}' |
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    --data-binary @-
else
  jq -n --arg c "$EXIT_CODE" --arg d "$DURATION" --arg o "$OUTPUT" \
    '{title: "Restic backup FAILED", message: "Exit code \($c) after \($d)s\n\($o)", priority: 5, tags: ["backup", "restic", "failure"]}' |
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    --data-binary @-
fi

restic -r "$REPO" forget --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
if [ $? -ne 0 ]; then
  curl -fsS -X POST "$WEBHOOK" \
    -H "Authorization: Bearer $ALPHORN_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{\"title\":\"Restic prune FAILED\",\"message\":\"restic forget --prune exited non-zero for $REPO\",\"priority\":5,\"tags\":[\"backup\",\"restic\",\"failure\"]}"
fi

#!/bin/bash
WEBHOOK="https://app.alphorn.dev/n/V1StGXR8_Z5jdHi6B-myT"
ALPHORN_API_KEY="${ALPHORN_API_KEY:-alp_...}"  # or export it in the environment
REPO="/backups/borg-repo"
ARCHIVE="$REPO::$(hostname)-$(date +%Y%m%d_%H%M%S)"
PATHS="/home /etc /var/lib"
START_TIME=$(date +%s)

OUTPUT=$(borg create --stats "$ARCHIVE" $PATHS 2>&1)
EXIT_CODE=$?
END_TIME=$(date +%s)
DURATION=$(( END_TIME - START_TIME ))

# Borg exit codes: 0 = success, 1 = warning (e.g. a file changed while being read), 2+ = error
if [ $EXIT_CODE -eq 0 ]; then
  TITLE="Borg backup completed"; PRIORITY=2; TAG="success"
elif [ $EXIT_CODE -eq 1 ]; then
  TITLE="Borg backup completed with warnings"; PRIORITY=3; TAG="warning"
else
  TITLE="Borg backup FAILED (exit $EXIT_CODE)"; PRIORITY=5; TAG="failure"
fi

# jq escapes the multi-line borg output into valid JSON
jq -n --arg t "$TITLE" --arg d "$DURATION" --arg o "$OUTPUT" --argjson p "$PRIORITY" --arg tag "$TAG" \
  '{title: $t, message: "Duration: \($d)s\n\($o)", priority: $p, tags: ["backup", "borg", $tag]}' |
curl -fsS -X POST "$WEBHOOK" \
  -H "Authorization: Bearer $ALPHORN_API_KEY" \
  -H "Content-Type: application/json" \
  --data-binary @-

# Prune old archives, then free the space (borg compact needs Borg 1.2+)
borg prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6 "$REPO"
borg compact "$REPO"