Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,25 @@
# ── 代理端口 ─────────────────────────────────────────────
PORT=8787

# ── 双节点生产拓扑 ────────────────────────────────────────
# 日本主服务器:
# ── 生产节点拓扑 ─────────────────────────────────────────
# 日本主服务器(当前生产只有这一台)
# SERVER_ROLE=primary
# RELAY_NODE_ID=jp-primary
# RELAY_PRIMARY_ORIGIN=https://app.kibotalk.app
# RELAY_CN_ORIGIN=http://123.99.200.156:8443
# RELAY_CN_NODE_ID=cn-relay
# RELAY_CN_ENABLED=true
# RELAY_NODES_JSON=[]
#
# 国内中转服务器
# 可选中转服务器(例如香港或新加坡)
# SERVER_ROLE=relay
# RELAY_NODE_ID=cn-relay
# RELAY_NODE_ID=sg-relay
# RELAY_PRIMARY_ORIGIN=https://app.kibotalk.app
# RELAY_OUTBOX_PATH=/app/data/usage-outbox.json
# RELAY_ACCEPT_NEW_SESSIONS=true
# RELAY_HTTP_PORT=8443
# 主服务器用 JSON 发布一个或多个中转节点:
# RELAY_NODES_JSON=[{"id":"sg-relay","origin":"https://sg-relay.kibotalk.app"}]
#
# Ed25519:日本保存私钥,两台都保存公钥。值可以是 PEM,或 PEM 的 base64。
# Ed25519:主服务器保存私钥,所有节点保存公钥。值可以是 PEM,或 PEM 的 base64。
# RELAY_TOKEN_PRIVATE_KEY=replace-on-primary-only
# RELAY_TOKEN_PUBLIC_KEY=replace-on-both-nodes
# RELAY_TOKEN_PUBLIC_KEY=replace-on-all-nodes
# 节点间心跳和用量回传凭据(与用户/提供商密钥分离):
# RELAY_NODE_SECRET=replace-with-a-long-random-secret

Expand Down
88 changes: 15 additions & 73 deletions .github/workflows/deploy-production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,12 @@ jobs:
CN_DEPLOY_SSH_KEY: ${{ secrets.CN_DEPLOY_SSH_KEY }}
CN_SERVER_HOST: ${{ secrets.CN_SERVER_HOST }}
CN_SERVER_USER: ${{ secrets.CN_SERVER_USER }}
CN_RELAY_ORIGIN: ${{ secrets.CN_RELAY_ORIGIN }}
JP_STT_DASHSCOPE_API_KEY: ${{ secrets.JP_STT_DASHSCOPE_API_KEY }}
JP_STT_DASHSCOPE_WS_URL: ${{ secrets.JP_STT_DASHSCOPE_WS_URL }}
run: |
for secret_name in \
DEPLOY_SSH_KEY SERVER_HOST SERVER_USER \
CN_DEPLOY_SSH_KEY CN_SERVER_HOST CN_SERVER_USER CN_RELAY_ORIGIN \
CN_DEPLOY_SSH_KEY CN_SERVER_HOST CN_SERVER_USER \
JP_STT_DASHSCOPE_API_KEY JP_STT_DASHSCOPE_WS_URL; do
if [ -z "${!secret_name}" ]; then
echo "::error::Missing Actions secret: ${secret_name}"
Expand All @@ -49,32 +48,26 @@ jobs:
install -d -m 700 ~/.ssh
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519_primary
printf '%s\n' "$CN_DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519_cn
chmod 600 ~/.ssh/id_ed25519_primary ~/.ssh/id_ed25519_cn
chmod 600 ~/.ssh/id_ed25519_primary
chmod 600 ~/.ssh/id_ed25519_cn
ssh-keyscan -H "$SERVER_HOST" >> ~/.ssh/known_hosts
ssh-keyscan -H "$CN_SERVER_HOST" >> ~/.ssh/known_hosts
- name: Configure primary routes and providers
env:
SERVER_HOST: ${{ secrets.SERVER_HOST }}
SERVER_USER: ${{ secrets.SERVER_USER }}
CN_RELAY_ORIGIN: ${{ secrets.CN_RELAY_ORIGIN }}
JP_STT_DASHSCOPE_API_KEY: ${{ secrets.JP_STT_DASHSCOPE_API_KEY }}
JP_STT_DASHSCOPE_WS_URL: ${{ secrets.JP_STT_DASHSCOPE_WS_URL }}
run: |
case "$CN_RELAY_ORIGIN" in
http://*|https://*) ;;
*) echo "::error::CN_RELAY_ORIGIN must be an absolute HTTP(S) origin"; exit 1 ;;
esac
case "$JP_STT_DASHSCOPE_WS_URL" in
wss://*.ap-northeast-1.maas.aliyuncs.com/api-ws/v1/realtime) ;;
*) echo "::error::JP_STT_DASHSCOPE_WS_URL must use the Alibaba Cloud Tokyo realtime endpoint"; exit 1 ;;
esac
printf '%s\n%s\n%s\n' \
"$CN_RELAY_ORIGIN" \
printf '%s\n%s\n' \
"$JP_STT_DASHSCOPE_API_KEY" \
"$JP_STT_DASHSCOPE_WS_URL" \
| ssh -i ~/.ssh/id_ed25519_primary "${SERVER_USER}@${SERVER_HOST}" '
set -eu
IFS= read -r origin
IFS= read -r stt_key
IFS= read -r stt_url
set_env() {
Expand All @@ -87,7 +80,7 @@ jobs:
printf "%s=%s\n" "$name" "$value" >> /opt/kibotalk/.env
fi
}
set_env RELAY_CN_ORIGIN "$origin"
set_env RELAY_NODES_JSON "[]"
set_env STT_DASHSCOPE_API_KEY "$stt_key"
set_env STT_DASHSCOPE_WS_URL "$stt_url"
chmod 600 /opt/kibotalk/.env
Expand All @@ -96,36 +89,18 @@ jobs:
env:
SERVER_HOST: ${{ secrets.SERVER_HOST }}
SERVER_USER: ${{ secrets.SERVER_USER }}
CN_SERVER_HOST: ${{ secrets.CN_SERVER_HOST }}
CN_SERVER_USER: ${{ secrets.CN_SERVER_USER }}
run: |
ssh -i ~/.ssh/id_ed25519_primary "${SERVER_USER}@${SERVER_HOST}" '
set -eu
test -s /opt/kibotalk/.env
for name in \
RELAY_PRIMARY_ORIGIN RELAY_CN_ORIGIN RELAY_CN_ENABLED \
RELAY_PRIMARY_ORIGIN RELAY_NODES_JSON \
RELAY_TOKEN_PRIVATE_KEY RELAY_TOKEN_PUBLIC_KEY RELAY_NODE_SECRET \
STT_ACTIVE STT_DASHSCOPE_API_KEY STT_DASHSCOPE_WS_URL \
LLM_ACTIVE; do
grep -q "^${name}=." /opt/kibotalk/.env
done
'
ssh -i ~/.ssh/id_ed25519_cn "${CN_SERVER_USER}@${CN_SERVER_HOST}" '
set -eu
test -s /opt/kibotalk-relay/.env
for name in \
RELAY_PRIMARY_ORIGIN RELAY_ACCEPT_NEW_SESSIONS \
RELAY_TOKEN_PUBLIC_KEY RELAY_NODE_SECRET \
STT_ACTIVE LLM_ACTIVE; do
grep -q "^${name}=." /opt/kibotalk-relay/.env
done
relay_port=$(sed -n "s/^RELAY_HTTP_PORT=//p" /opt/kibotalk-relay/.env)
relay_port=${relay_port:-8443}
case "$relay_port" in
8443|9443) ;;
*) echo "RELAY_HTTP_PORT must be 8443 or 9443" >&2; exit 1 ;;
esac
'
- uses: pnpm/action-setup@v4
with:
version: 10.33.0
Expand All @@ -141,22 +116,15 @@ jobs:
- name: Build API image
run: |
docker build --platform linux/amd64 --tag "kibotalk-api:${GITHUB_SHA}" .
docker build --platform linux/amd64 --target relay \
--tag "kibotalk-relay-api:${GITHUB_SHA}" .
docker build --platform linux/amd64 --tag kibotalk-caddy:2.10 --file infra/production/Caddy.Dockerfile .
docker build --platform linux/amd64 --tag kibotalk-postgres:17 --file infra/production/Postgres.Dockerfile .
touch infra/production/.env infra/relay/.env
touch infra/production/.env
(
cd infra/production
POSTGRES_PASSWORD=validation docker compose -f compose.yaml \
config --quiet --no-env-resolution --no-path-resolution
)
(
cd infra/relay
RELAY_HTTP_PORT=8443 docker compose -f compose.yaml \
config --quiet --no-env-resolution --no-path-resolution
)
rm infra/production/.env infra/relay/.env
rm infra/production/.env
docker run --rm \
--volume "$PWD/infra/production/Caddyfile:/etc/caddy/Caddyfile:ro" \
kibotalk-caddy:2.10 \
Expand All @@ -166,9 +134,6 @@ jobs:
kibotalk-caddy:2.10 \
kibotalk-postgres:17 \
| gzip -1 > /tmp/kibotalk-images.tar.gz
docker save \
"kibotalk-relay-api:${GITHUB_SHA}" \
| gzip -1 > /tmp/kibotalk-relay-images.tar.gz
- name: Stage on-device model mirror
run: |
pnpm --filter @kibotalk/desktop download-models
Expand All @@ -182,17 +147,6 @@ jobs:
scp -i ~/.ssh/id_ed25519_primary /tmp/kibotalk-images.tar.gz "${SERVER_USER}@${SERVER_HOST}:/opt/kibotalk/incoming/kibotalk-images.tar.gz"
scp -i ~/.ssh/id_ed25519_primary /tmp/kibotalk-models.tar.gz "${SERVER_USER}@${SERVER_HOST}:/opt/kibotalk/incoming/kibotalk-models.tar.gz"
scp -i ~/.ssh/id_ed25519_primary infra/production/compose.yaml infra/production/Caddyfile "${SERVER_USER}@${SERVER_HOST}:/opt/kibotalk/"
- name: Upload relay payload
env:
CN_SERVER_HOST: ${{ secrets.CN_SERVER_HOST }}
CN_SERVER_USER: ${{ secrets.CN_SERVER_USER }}
run: |
ssh -i ~/.ssh/id_ed25519_cn "${CN_SERVER_USER}@${CN_SERVER_HOST}" \
"mkdir -p /opt/kibotalk-relay/incoming"
scp -i ~/.ssh/id_ed25519_cn /tmp/kibotalk-relay-images.tar.gz \
"${CN_SERVER_USER}@${CN_SERVER_HOST}:/opt/kibotalk-relay/incoming/kibotalk-images.tar.gz"
scp -i ~/.ssh/id_ed25519_cn infra/relay/compose.yaml \
"${CN_SERVER_USER}@${CN_SERVER_HOST}:/opt/kibotalk-relay/"
- name: Activate release
env:
SERVER_HOST: ${{ secrets.SERVER_HOST }}
Expand Down Expand Up @@ -220,27 +174,19 @@ jobs:
fi
docker compose --env-file .env -f compose.yaml up -d --pull never
"
- name: Activate relay release
- name: Deactivate legacy China relay
env:
CN_SERVER_HOST: ${{ secrets.CN_SERVER_HOST }}
CN_SERVER_USER: ${{ secrets.CN_SERVER_USER }}
run: |
# GITHUB_SHA must expand on the runner before this script reaches the server.
# shellcheck disable=SC2029
ssh -i ~/.ssh/id_ed25519_cn "${CN_SERVER_USER}@${CN_SERVER_HOST}" "
ssh -i ~/.ssh/id_ed25519_cn "${CN_SERVER_USER}@${CN_SERVER_HOST}" '
set -eu
gzip -dc /opt/kibotalk-relay/incoming/kibotalk-images.tar.gz | docker load
cd /opt/kibotalk-relay
if grep -q '^KIBOTALK_RELAY_IMAGE=' .env; then
sed -i 's|^KIBOTALK_RELAY_IMAGE=.*|KIBOTALK_RELAY_IMAGE=kibotalk-relay-api:${GITHUB_SHA}|' .env
else
printf '\nKIBOTALK_RELAY_IMAGE=kibotalk-relay-api:${GITHUB_SHA}\n' >> .env
if [ -f /opt/kibotalk-relay/compose.yaml ]; then
cd /opt/kibotalk-relay
docker compose --env-file .env -f compose.yaml down
fi
docker compose --env-file .env -f compose.yaml up -d --pull never
"
'
- name: Verify deployment
env:
CN_RELAY_ORIGIN: ${{ secrets.CN_RELAY_ORIGIN }}
run: |
for _attempt in $(seq 1 24); do
if curl --fail --silent --show-error https://app.kibotalk.app/health \
Expand All @@ -253,11 +199,7 @@ jobs:
&& curl --fail --silent --show-error https://kibotalk.app/en/ >/dev/null \
&& test "$(curl --silent --output /dev/null --write-out '%{redirect_url}' https://www.kibotalk.app/)" = "https://kibotalk.app/" \
&& curl --fail --silent --show-error https://advx.kibotalk.app/ \
| grep --quiet '正在清理旧版网页数据' \
&& curl --fail --silent --show-error "${CN_RELAY_ORIGIN}/health" \
| grep --quiet '"ok":true' \
&& curl --fail --silent --show-error "${CN_RELAY_ORIGIN}/api/latency" \
| grep --quiet '"nodeId":"cn-relay"'; then
| grep --quiet '正在清理旧版网页数据'; then
exit 0
fi
sleep 5
Expand Down
4 changes: 2 additions & 2 deletions apps/api/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ import {
releaseRelayLlm,
} from './relay-session-state'
import { providerHealthy } from './provider-health'
import { serverRole } from './server-role'
import { relayNodeId, serverRole } from './server-role'

export const app = new Hono()
const role = serverRole()
Expand Down Expand Up @@ -135,7 +135,7 @@ app.get('/health', async (context) => {
return context.json({
ok: providerHealthy(),
role,
nodeId: process.env.RELAY_NODE_ID ?? 'cn-relay',
nodeId: relayNodeId(),
providers: providerHealthy() ? 'ok' : 'error',
version: process.env.APP_VERSION ?? 'development',
}, providerHealthy() ? 200 : 503)
Expand Down
59 changes: 47 additions & 12 deletions apps/api/src/relay-nodes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ const DEFAULT_PROBE = {
} as const
const HEARTBEAT_MAX_AGE_SECONDS = 45

type ConfiguredRelay = {
id: string
origin: string
}

function normalizeOrigin(value: string, name: string): string {
let url: URL
try {
Expand All @@ -27,30 +32,60 @@ function normalizeOrigin(value: string, name: string): string {
return url.origin
}

function configuredSecondaryRelays(
value: string | undefined,
primaryNodeId: string,
): RelayNode[] {
if (!value?.trim()) return []
let parsed: unknown
try {
parsed = JSON.parse(value)
} catch {
throw new Error('RELAY_NODES_JSON must be valid JSON')
}
if (!Array.isArray(parsed)) {
throw new Error('RELAY_NODES_JSON must be an array')
}
const seen = new Set([primaryNodeId])
return parsed.map((entry, index) => {
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
throw new Error(`RELAY_NODES_JSON[${index}] must be an object`)
}
const relay = entry as Partial<ConfiguredRelay>
const id = typeof relay.id === 'string' ? relay.id.trim() : ''
const origin = typeof relay.origin === 'string' ? relay.origin.trim() : ''
if (!id || !origin) {
throw new Error(`RELAY_NODES_JSON[${index}] must contain id and origin`)
}
if (seen.has(id)) {
throw new Error(`RELAY_NODES_JSON contains duplicate node id: ${id}`)
}
seen.add(id)
return {
id,
origin: normalizeOrigin(origin, `RELAY_NODES_JSON[${index}].origin`),
role: 'relay' as const,
acceptingNewSessions: true,
}
})
}

export function configuredRelayNodes(
env: NodeJS.ProcessEnv = process.env,
): RelayNode[] {
const primaryNodeId = primaryRelayNodeId(env)
const primaryOrigin = normalizeOrigin(
env.RELAY_PRIMARY_ORIGIN
?? env.PUBLIC_APP_URL
?? `http://localhost:${env.PORT ?? 8787}`,
'RELAY_PRIMARY_ORIGIN',
)
const nodes: RelayNode[] = [{
id: primaryRelayNodeId(env),
return [{
id: primaryNodeId,
origin: primaryOrigin,
role: 'primary',
acceptingNewSessions: relayAcceptingNewSessions(env),
}]
if (env.RELAY_CN_ENABLED !== 'false' && env.RELAY_CN_ORIGIN) {
nodes.push({
id: env.RELAY_CN_NODE_ID?.trim() || 'cn-relay',
origin: normalizeOrigin(env.RELAY_CN_ORIGIN, 'RELAY_CN_ORIGIN'),
role: 'relay',
acceptingNewSessions: true,
})
}
return nodes
}, ...configuredSecondaryRelays(env.RELAY_NODES_JSON, primaryNodeId)]
}

export async function availableRelayNodeList(
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/server-role.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export function serverRole(env: NodeJS.ProcessEnv = process.env): ServerRole {
export function relayNodeId(env: NodeJS.ProcessEnv = process.env): string {
const value = env.RELAY_NODE_ID?.trim()
if (value) return value
return serverRole(env) === 'primary' ? 'jp-primary' : 'cn-relay'
return serverRole(env) === 'primary' ? 'jp-primary' : 'relay-node'
}

export function primaryRelayNodeId(env: NodeJS.ProcessEnv = process.env): string {
Expand Down
Loading
Loading