diff --git a/.gitignore b/.gitignore index d091128..4a26663 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ elastic deploy/*_data deploy/docker-images scripts/download-image.sh -deploy/preset.zip \ No newline at end of file +deploy/preset.zip +node_modules \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fb025ed..74d21c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,11 @@ COPY . . #2 -RUN find scripts/ -type f -name "*.sh" -exec sed -i 's/\r$//' {} + && \ +RUN find scripts/ -type f \( -name "*.sh" -o -name "*.js" \) -exec sed -i 's/\r$//' {} + && \ sed -i 's/\r$//' gradlew +RUN bash scripts/download_scripts/download_data.sh $PRESET_ENCRYPTED_URI $PRESET_ENCRYPTED_PASSWORD + ENV IS_LOCAL_HOST=false ENV IS_DOCKER=true diff --git a/Dockerfile.patched b/Dockerfile.patched index 00ef247..b332113 100644 --- a/Dockerfile.patched +++ b/Dockerfile.patched @@ -110,6 +110,8 @@ EOF RUN find scripts/ -type f -name "*.sh" -exec sed -i 's/\r$//' {} + && \ sed -i 's/\r$//' gradlew +RUN bash scripts/download_scripts/download_data.sh $PRESET_ENCRYPTED_URI $PRESET_ENCRYPTED_PASSWORD + ENV IS_LOCAL_HOST=false ENV IS_DOCKER=true diff --git a/deploy/Dockerfile.koyeb b/deploy/Dockerfile.koyeb index 9a00cd6..8572e9a 100644 --- a/deploy/Dockerfile.koyeb +++ b/deploy/Dockerfile.koyeb @@ -18,8 +18,6 @@ COPY . . RUN find scripts/ -type f -name "*.sh" -exec sed -i 's/\r$//' {} + && \ sed -i 's/\r$//' gradlew -RUN bash scripts/download_scripts/download_data.sh $PRESET_ENCRYPTED_URI $PRESET_ENCRYPTED_PASSWORD - RUN bash scripts/download_scripts/manual-download-docker-image.sh #mock host environment diff --git a/dev.env b/dev.env index ef81e0b..816a673 100644 --- a/dev.env +++ b/dev.env @@ -51,3 +51,5 @@ PRESET_WORKING_DIR='../deploy' PRESET_SCRIPT='sh ./flush-database-singleton.sh ../cli/build/install/cli/bin/cli ./preset_data' PRESET_ENCRYPTED_URI= PRESET_ENCRYPTED_PASSWORD= + +PUSH_TOKEN= diff --git a/dev.win.env b/dev.win.env index 0b0ad0d..fa2181d 100644 --- a/dev.win.env +++ b/dev.win.env @@ -51,3 +51,5 @@ PRESET_WORKING_DIR='../deploy' PRESET_SCRIPT='sh ./flush-database-singleton.sh ../cli/build/install/cli/bin/cli ./preset_data' PRESET_ENCRYPTED_URI= PRESET_ENCRYPTED_PASSWORD= + +PUSH_TOKEN= diff --git a/env-filter b/env-filter index 582895c..b40bb9c 100644 --- a/env-filter +++ b/env-filter @@ -44,3 +44,4 @@ PRESET_WORKING_DIR PRESET_SCRIPT PRESET_ENCRYPTED_URI PRESET_ENCRYPTED_PASSWORD +PUSH_TOKEN diff --git a/scripts/tool_scripts/check-env-filter.sh b/scripts/tool_scripts/check-env-filter.sh index 7ecdea5..6bd54b6 100644 --- a/scripts/tool_scripts/check-env-filter.sh +++ b/scripts/tool_scripts/check-env-filter.sh @@ -29,12 +29,6 @@ while IFS='=' read -r key value; do fi done < "$env_file" -# 验证 env_keys 和 env_filter_keys 是否按顺序匹配 -if [ ${#env_keys[@]} -ne ${#env_filter_keys[@]} ]; then - echo "env $env_file 和 env-filter 文件中的变量数量不匹配! ${#env_keys[@]} - ${#env_filter_keys[@]}" - exit 1 -fi - # 按行对比 for i in "${!env_keys[@]}"; do if [ "${env_keys[$i]}" != "${env_filter_keys[$i]}" ]; then @@ -43,4 +37,25 @@ for i in "${!env_keys[@]}"; do fi done + + +# 验证 env_keys 和 env_filter_keys 是否按顺序匹配 +if [ ${#env_keys[@]} -ne ${#env_filter_keys[@]} ]; then + echo "env $env_file 和 env-filter 文件中的变量数量不匹配! ${#env_keys[@]} - ${#env_filter_keys[@]}" + echo "Only in env:" + for item in "${env_keys[@]}"; do + if [[ ! " ${env_filter_keys[@]} " =~ " ${item} " ]]; then + echo "$item" + fi + done + + echo "Only in env-filter:" + for item in "${env_filter_keys[@]}"; do + if [[ ! " ${env_keys[@]} " =~ " ${item} " ]]; then + echo "$item" + fi + done + exit 1 +fi + echo "环境变量与 env-filter 中的定义完全按顺序匹配!" diff --git a/scripts/tool_scripts/encrypt-secret.js b/scripts/tool_scripts/encrypt-secret.js new file mode 100644 index 0000000..c639feb --- /dev/null +++ b/scripts/tool_scripts/encrypt-secret.js @@ -0,0 +1,27 @@ +const sodium = require('libsodium-wrappers') + +const args = process.argv.slice(2); + +if (args.length != 2) { + console.error("args not match", args.length) + return +} + +const key = args[0] +const secret = args[1] + +//Check if libsodium is ready and then proceed. +sodium.ready.then(() => { + // Convert the secret and key to a Uint8Array. + let binkey = sodium.from_base64(key, sodium.base64_variants.ORIGINAL) + let binsec = sodium.from_string(secret) + + // Encrypt the secret using libsodium + let encBytes = sodium.crypto_box_seal(binsec, binkey) + + // Convert the encrypted Uint8Array to Base64 + let output = sodium.to_base64(encBytes, sodium.base64_variants.ORIGINAL) + + // Print the output + console.log(output) +}); diff --git a/scripts/tool_scripts/package-lock.json b/scripts/tool_scripts/package-lock.json new file mode 100644 index 0000000..7c2f420 --- /dev/null +++ b/scripts/tool_scripts/package-lock.json @@ -0,0 +1,29 @@ +{ + "name": "playground", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "playground", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "libsodium-wrappers": "^0.7.15" + } + }, + "node_modules/libsodium": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/libsodium/-/libsodium-0.7.15.tgz", + "integrity": "sha512-sZwRknt/tUpE2AwzHq3jEyUU5uvIZHtSssktXq7owd++3CSgn8RGrv6UZJJBpP7+iBghBqe7Z06/2M31rI2NKw==" + }, + "node_modules/libsodium-wrappers": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/libsodium-wrappers/-/libsodium-wrappers-0.7.15.tgz", + "integrity": "sha512-E4anqJQwcfiC6+Yrl01C1m8p99wEhLmJSs0VQqST66SbQXXBoaJY0pF4BNjRYa/sOQAxx6lXAaAFIlx+15tXJQ==", + "dependencies": { + "libsodium": "^0.7.15" + } + } + } +} diff --git a/scripts/tool_scripts/package.json b/scripts/tool_scripts/package.json new file mode 100644 index 0000000..cd1c433 --- /dev/null +++ b/scripts/tool_scripts/package.json @@ -0,0 +1,14 @@ +{ + "name": "playground", + "version": "1.0.0", + "description": "", + "main": "encrypt-secret.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "libsodium-wrappers": "^0.7.15" + } +} \ No newline at end of file diff --git a/scripts/tool_scripts/push-env-to-github.sh b/scripts/tool_scripts/push-env-to-github.sh new file mode 100644 index 0000000..821127b --- /dev/null +++ b/scripts/tool_scripts/push-env-to-github.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +set -e + +FLAVOR=$1 + +if [ -z "$FLAVOR" ]; then + echo "FLAVOR must be set." + exit 1 +fi + +# Variables +OWNER="storytellerF" # 替换为仓库的拥有者 +REPO="A" # 替换为你的仓库名 +ENVIRONMENT="$FLAVOR" # 替换为目标环境名 +ENV_FILE="$FLAVOR.env" # .env 文件路径 +PUSH_TOKEN_KEY="PUSH_TOKEN" +PUSH_TOKEN=$(grep "^${PUSH_TOKEN_KEY}=" "$ENV_FILE" | cut -d '=' -f2-) # 替换为你的 GitHub 访问令牌 + +if [ -z "$PUSH_TOKEN" ]; then + echo "PUSH_TOKEN must be set." + exit 1 +fi + +check_and_set_variable() { + # 接收一个参数作为环境变量的键 + local env_key="$1" + + # 定义敏感关键词 + local keywords=("TOKEN" "PASS" "PASSWORD" "KEY") + + # 检查环境变量键是否包含任何敏感关键字 + for keyword in "${keywords[@]}"; do + if [[ $env_key == *"$keyword"* ]]; then + echo "环境变量键 '$env_key' 包含敏感关键字 '$keyword'。" + return 1 # 返回 1 + fi + done + + # 如果没有敏感关键字,返回 0 + echo "环境变量键 '$env_key' 不包含敏感关键字。" + return 0 # 返回 0 +} + +# Function to check if the environment exists +check_environment_exists() { + local env_name="$1" + response=$(curl -fL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $PUSH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$OWNER/$REPO/environments") + echo "$response" | jq -e ".environments[] | select(.name == \"$env_name\")" > /dev/null +} + +# Check if the environment exists +if check_environment_exists "$ENVIRONMENT"; then + echo "Environment $ENVIRONMENT exists. Proceeding to add secret..." +else + echo "Environment $ENVIRONMENT does not exist. Skipping secrets." + exit 1 +fi + +# Step 1: Get the public key for the environment +PUBLIC_KEY_INFO=$(curl -fsL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $PUSH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$OWNER/$REPO/environments/$ENVIRONMENT/secrets/public-key") + +PUBLIC_KEY=$(echo "$PUBLIC_KEY_INFO" | jq -r .key) +KEY_ID=$(echo "$PUBLIC_KEY_INFO" | jq -r .key_id) + +echo "$PUBLIC_KEY $KEY_ID" + +# Read the .env file +while IFS='=' read -r secret_name secret_value; do + # Skip empty lines and comments + [[ -z "$secret_name" || "$secret_name" == \#* ]] && continue + if [ -z "$secret_value" ]; then + echo "$secret_name is empty, skip." + continue + fi + + check_and_set_variable $secret_name + result=$? + + # 检查返回结果 + if [ $result -eq 0 ]; then + exists_name=$(curl -sL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $PUSH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$OWNER/$REPO/environments/$ENVIRONMENT/variables/$secret_name" | jq -r '.name') + if [ "$exists_name" = "$secret_name" ]; then + + curl -L \ + -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $PUSH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"name\":\"$secret_name\",\"value\":\"$secret_value\"}" \ + "https://api.github.com/repos/$OWNER/$REPO/environments/$ENVIRONMENT/variables/$secret_name" + else + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $PUSH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"name\":\"$secret_name\",\"value\":\"$secret_value\"}" \ + "https://api.github.com/repos/$OWNER/$REPO/environments/$ENVIRONMENT/variables" + fi + else + ENCRYPTED_SECRET=$(node scripts/tool_scripts/encrypt-secret.js $PUBLIC_KEY "$secret_value") + + curl -L -X PUT \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $PUSH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"encrypted_value\":\"$ENCRYPTED_SECRET\",\"key_id\":\"$KEY_ID\"}" \ + "https://api.github.com/repos/$OWNER/$REPO/environments/$ENVIRONMENT/secrets/$secret_name" + + echo "Secret $secret_name has been added to environment $ENVIRONMENT." + fi + + sleep 2 + +done < "$ENV_FILE" + diff --git a/scripts/tool_scripts/save-env.sh b/scripts/tool_scripts/save-env.sh new file mode 100644 index 0000000..c425703 --- /dev/null +++ b/scripts/tool_scripts/save-env.sh @@ -0,0 +1,33 @@ +if [ -z "$FLAVOR" ]; then + echo "FLAVOR must set." + exit 1 +fi +export FLAVOR=generated-$FLAVOR +OUTPUT_FILE="$FLAVOR.env" + +# 定义 env-filter 文件名 +env_filter_file="env-filter" + +# 创建或清空 .env 文件 +> $OUTPUT_FILE + +# 遍历 env-filter 文件中的每一行 +while IFS= read -r key; do + # 去除可能的 \r 符号(处理 Windows 换行符) + key=$(echo "$key" | tr -d '\r') + upper_key=$(echo "$key" | tr '[:lower:]' '[:upper:]') + + # 获取对应的环境变量值 + value="${!key}" + # 对反斜杠进行转义 + value=$(echo "$value" | sed 's/\\/\\\\/g') + + # 如果值中包含空格,则用引号包裹 + if echo "$value" | grep -q ' '; then + echo "$upper_key=\"$value\"" >> $OUTPUT_FILE + else + echo "$upper_key=$value" >> $OUTPUT_FILE + fi +done < "$env_filter_file" + +echo "$FLAVOR.env 文件已生成。" diff --git a/server/src/test/resources/.env b/server/src/test/resources/.env index c5868e0..6ad0f33 100644 --- a/server/src/test/resources/.env +++ b/server/src/test/resources/.env @@ -51,3 +51,5 @@ PRESET_WORKING_DIR= PRESET_SCRIPT= PRESET_ENCRYPTED_URI= PRESET_ENCRYPTED_PASSWORD= + +PUSH_TOKEN=