Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .github/workflows/secrets-toolkit.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: Secrets Toolkit

on:
pull_request:
paths:
- "scripts/secrets-*.sh"
- "scripts/tests/**"
- "secrets/**"
- ".github/workflows/secrets-toolkit.yml"
push:
branches:
- "feature/secrets-collect"
paths:
- "scripts/secrets-*.sh"
- "scripts/tests/**"
- "secrets/**"
- ".github/workflows/secrets-toolkit.yml"

jobs:
verify:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Install shellcheck
run: sudo apt-get update && sudo apt-get install -y shellcheck

- name: Run toolkit checks
run: bash scripts/tests/run-all.sh
367 changes: 367 additions & 0 deletions scripts/secrets-collect.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,367 @@
#!/bin/bash
# Collect pulled/local secrets into Bitwarden-ready value blocks.
# Values are written only to .secrets-recovery/bitwarden-blocks.txt.
set -euo pipefail

umask 077

ROOT_DIR="${SECRETS_ROOT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
RECOVERY_DIR="${SECRETS_RECOVERY_DIR:-${ROOT_DIR}/.secrets-recovery}"
PULLED_DIR="${SECRETS_PULLED_DIR:-${RECOVERY_DIR}/pulled}"
MAPPING_FILE="${SECRETS_COLLECT_MAPPING_FILE:-${ROOT_DIR}/secrets/collect-mapping.yaml}"
OUTPUT_FILE="${SECRETS_OUTPUT_FILE:-${RECOVERY_DIR}/bitwarden-blocks.txt}"

usage() {
cat <<USAGE
Usage: ./scripts/secrets-collect.sh [--help]

Reads:
.secrets-recovery/pulled/
secrets/collect-mapping.yaml

Writes:
.secrets-recovery/bitwarden-blocks.txt

The script never prints secret values to stdout.
USAGE
}

if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
usage
exit 0
fi

if [[ "$#" -gt 0 ]]; then
echo "Unexpected argument: $1" >&2
usage >&2
exit 2
fi

if [[ ! -d "${PULLED_DIR}" ]]; then
cat >&2 <<EOF
Missing ${PULLED_DIR}.

Run the pull step first:
./scripts/secrets-pull.sh
EOF
exit 1
fi

if [[ ! -f "${MAPPING_FILE}" ]]; then
echo "Missing mapping file: ${MAPPING_FILE}" >&2
exit 1
fi

mkdir -p "${RECOVERY_DIR}"

flatten_path() {
local raw_path="$1"
local flattened="${raw_path#/}"
flattened="${flattened//\//__}"
printf '%s' "${flattened}"
}

expand_local_path() {
local raw_path="$1"
case "${raw_path}" in
"~")
printf '%s' "${HOME}"
;;
~/*)
printf '%s/%s' "${HOME}" "${raw_path#~/}"
;;
*)
printf '%s' "${raw_path}"
;;
esac
}

pulled_file_path() {
local server="$1"
local remote_file="$2"
printf '%s/%s/%s' "${PULLED_DIR}" "${server}" "$(flatten_path "${remote_file}")"
}

trim_string() {
local value="$1"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
printf '%s' "${value}"
}

strip_matching_quotes() {
local value="$1"
if [[ "${value}" == \"*\" && "${value}" == *\" ]]; then
value="${value:1:${#value}-2}"
elif [[ "${value}" == \'*\' && "${value}" == *\' ]]; then
value="${value:1:${#value}-2}"
fi
printf '%s' "${value}"
}

lookup_assignment_value() {
local file_path="$1"
local key="$2"
local line
local value

if [[ ! -f "${file_path}" ]]; then
return 1
fi

line="$(awk -v key="${key}" '
BEGIN {
pattern = "^[[:space:]]*" key "="
}
$0 ~ pattern {
result = $0
}
END {
if (result != "") {
print result
} else {
exit 1
}
}
' "${file_path}")" || return 1

value="${line#*=}"
value="$(trim_string "${value}")"
value="${value%\\}"
value="$(trim_string "${value}")"
value="$(strip_matching_quotes "${value}")"
printf '%s' "${value}"
}

extract_postgres_password() {
local url_value="$1"

python3 - "$url_value" <<'PY'
import sys
from urllib.parse import unquote, urlparse

url = sys.argv[1]
parsed = urlparse(url)
password = parsed.password
if password is None:
sys.exit(1)
print(unquote(password), end="")
PY
}

parse_mapping() {
awk '
function trim(value) {
gsub(/^[[:space:]]+|[[:space:]]+$/, "", value)
gsub(/^"|"$/, "", value)
return value
}
function value_after_colon() {
sub(/^[^:]+:[[:space:]]*/, "")
return trim($0)
}
function reset_record() {
bw_item = tag = optional = source_type = server = file = path = var = extract = vars = lines = ""
in_vars = 0
in_lines = 0
}
function flush_record() {
if (bw_item != "") {
print bw_item "\037" tag "\037" optional "\037" source_type "\037" server "\037" file "\037" path "\037" var "\037" extract "\037" vars "\037" lines
}
}
/^- bw_item:/ {
flush_record()
reset_record()
bw_item = value_after_colon()
next
}
/^ tag:/ { tag = value_after_colon(); next }
/^ optional:/ { optional = value_after_colon(); next }
/^ source:/ { next }
/^ type:/ { source_type = value_after_colon(); in_vars = 0; in_lines = 0; next }
/^ server:/ { server = value_after_colon(); in_vars = 0; in_lines = 0; next }
/^ file:/ { file = value_after_colon(); in_vars = 0; in_lines = 0; next }
/^ path:/ { path = value_after_colon(); in_vars = 0; in_lines = 0; next }
/^ var:/ { var = value_after_colon(); in_vars = 0; in_lines = 0; next }
/^ extract:/ { extract = value_after_colon(); in_vars = 0; in_lines = 0; next }
/^ vars:/ { in_vars = 1; in_lines = 0; next }
/^ lines:/ { in_lines = 1; in_vars = 0; next }
in_vars && /^ - / {
value = $0
sub(/^ -[[:space:]]*/, "", value)
value = trim(value)
vars = vars (vars == "" ? "" : ",") value
next
}
in_lines && /^ - / {
value = $0
sub(/^ -[[:space:]]*/, "", value)
value = trim(value)
lines = lines (lines == "" ? "" : "\034") value
next
}
!/^ - / {
in_vars = 0
in_lines = 0
}
BEGIN { reset_record() }
END { flush_record() }
' "${MAPPING_FILE}"
}

write_block() {
local out_file="$1"
local item_name="$2"
local tag_name="$3"
local body="$4"

{
printf '### %s [%s]\n' "${item_name}" "${tag_name}"
printf '%s\n' "${body}"
printf '\n'
} >> "${out_file}"
}

emit_missing_notice() {
local out_file="$1"
shift
local missing_items=("$@")
local joined

if [[ "${#missing_items[@]}" -eq 0 ]]; then
return
fi

joined="$(printf '%s, ' "${missing_items[@]}")"
joined="${joined%, }"

{
printf '# PARTIAL: %s\n' "${joined}"
printf '\n'
} >> "${out_file}"
}

tmp_output="$(mktemp "${RECOVERY_DIR}/bitwarden-blocks.XXXXXX")"
trap 'rm -f "${tmp_output}"' EXIT
chmod 600 "${tmp_output}"

written_count=0
missing_items=()

while IFS=$'\037' read -r bw_item tag optional source_type server file path var extract vars_csv lines_raw; do
[[ -n "${bw_item}" ]] || continue

source_file=""
block_body=""

case "${source_type}" in
env_file)
source_file="$(pulled_file_path "${server}" "${file}")"
if ! block_value="$(lookup_assignment_value "${source_file}" "${var}")"; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: ${source_file} -> ${var}" >&2
continue
fi
block_body="${var}=${block_value}"
;;
env_file_group)
source_file="$(pulled_file_path "${server}" "${file}")"
if [[ ! -f "${source_file}" ]]; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: ${source_file}" >&2
continue
fi

block_lines=()
IFS=',' read -r -a vars_list <<< "${vars_csv}"
for member_var in "${vars_list[@]}"; do
if ! member_value="$(lookup_assignment_value "${source_file}" "${member_var}")"; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: ${source_file} -> ${member_var}" >&2
block_lines=()
break
fi
block_lines+=("${member_var}=${member_value}")
done

if [[ "${#block_lines[@]}" -eq 0 ]]; then
continue
fi

block_body="$(printf '%s\n' "${block_lines[@]}")"
block_body="${block_body%$'\n'}"
;;
extract_from_url)
source_file="$(pulled_file_path "${server}" "${file}")"
if ! url_value="$(lookup_assignment_value "${source_file}" "${var}")"; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: ${source_file} -> ${var}" >&2
continue
fi
if [[ "${extract}" != "postgres_password" ]]; then
echo "Unsupported extract kind '${extract}' for ${bw_item}" >&2
exit 1
fi
if ! extracted_value="$(extract_postgres_password "${url_value}")"; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: unable to extract password from ${var}" >&2
continue
fi
block_body="${var}=${url_value}"$'\n'"# extracted password -> ${extracted_value}"
;;
whole_file)
source_file="$(pulled_file_path "${server}" "${file}")"
if [[ ! -f "${source_file}" ]]; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: ${source_file}" >&2
continue
fi
block_body="$(<"${source_file}")"
;;
local_file)
source_file="$(expand_local_path "${path}")"
if [[ ! -f "${source_file}" ]]; then
if [[ "${optional}" == "true" ]]; then
missing_items+=("${bw_item}")
fi
echo "Missing ${bw_item}: ${source_file}" >&2
continue
fi
block_body="$(<"${source_file}")"
;;
literal_lines)
IFS=$'\034' read -r -a literal_lines <<< "${lines_raw}"
block_body="$(printf '%s\n' "${literal_lines[@]}")"
block_body="${block_body%$'\n'}"
;;
*)
echo "Unsupported source type '${source_type}' for ${bw_item}" >&2
exit 1
;;
esac

write_block "${tmp_output}" "${bw_item}" "${tag}" "${block_body}"
written_count=$((written_count + 1))
done < <(parse_mapping)

if [[ "${#missing_items[@]}" -gt 0 ]]; then
emit_missing_notice "${tmp_output}" "${missing_items[@]}"
fi
mv "${tmp_output}" "${OUTPUT_FILE}"
trap - EXIT

chmod 600 "${OUTPUT_FILE}"
echo "${written_count} items written to ${OUTPUT_FILE}"
Loading
Loading