diff --git a/.env.ci b/.env.ci index 6e56e86c..90c4af60 100644 --- a/.env.ci +++ b/.env.ci @@ -1,5 +1,15 @@ DATABASE_URL=postgresql://root:root@localhost:5432/lifecycle +APP_DB_HOST=localhost +APP_DB_PORT=5432 +APP_DB_USER=root +APP_DB_PASSWORD=root +APP_DB_NAME=lifecycle +APP_DB_SSL=false REDIS_URL=redis://localhost:6379 +APP_REDIS_HOST=localhost +APP_REDIS_PORT=6379 +APP_REDIS_PASSWORD= +APP_REDIS_TLS=false GITHUB_WEBHOOK_SECRET='o1o1o1o1' CODEFRESH_API_KEY='o1o1o1o1o1' GITHUB_PRIVATE_KEY='o1o1o1o1o1' diff --git a/.env.example b/.env.example index 3ef42462..4ecee4d4 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,29 @@ +# DATABASE CONFIGURATION +# Option 1: Individual database variables (recommended) +APP_DB_HOST='localhost' +APP_DB_PORT='5434' +APP_DB_USER='lifecycle' +APP_DB_PASSWORD='lifecycle' +APP_DB_NAME='lifecycle' +APP_DB_SSL='false' + +# Option 2: Legacy DATABASE_URL format (for backward compatibility) +# DEPRECATED: This format will be removed in future releases. Please use individual APP_DB_* variables above. # You need the production db password below # DATABASE_URL='postgresql://lifecycle:@localhost:5432/lifecycle' # You may need to update the port DATABASE_URL='postgresql://lifecycle:lifecycle@localhost:5434/lifecycle' -DATABASE_HOST='localhost' -DATABASE_NAME='lifecycle' -DATABASE_PASSWORD='lifecycle' -# REDIS +# REDIS CONFIGURATION +# Option 1: Individual Redis variables (recommended) +APP_REDIS_HOST='localhost' +APP_REDIS_PORT='6379' +APP_REDIS_PASSWORD='' +APP_REDIS_TLS='false' + +# Option 2: Legacy REDIS_URL format (for backward compatibility) +# DEPRECATED: This format will be removed in future releases. Please use individual APP_REDIS_* variables above. REDIS_URL='redis://localhost:6379' -REDIS_PORT='6379' # GITHUB # This is all created while you're creating your GitHub App and Test repository diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..4bf7a27c --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +# Ignore Helm templates +helm/**/templates/*.yaml +helm/**/templates/*.yml \ No newline at end of file diff --git a/Tiltfile b/Tiltfile index 350da070..10c49aa9 100644 --- a/Tiltfile +++ b/Tiltfile @@ -103,8 +103,15 @@ docker_build_with_restart( entrypoint=["/app_setup_entrypoint.sh"], dockerfile="sysops/dockerfiles/tilt.app.dockerfile", build_args={ - "DATABASE_URL": "postgresql://lifecycle:lifecycle@local-postgres.{}.svc.cluster.local:5432/lifecycle".format(app_namespace), - "REDIS_URL": "redis://redis-master.{}.svc.cluster.local:6379".format(app_namespace), + "APP_DB_HOST": "local-postgres.{}.svc.cluster.local".format(app_namespace), + "APP_DB_PORT": "5432", + "APP_DB_USER": "lifecycle", + "APP_DB_PASSWORD": "lifecycle", + "APP_DB_NAME": "lifecycle", + "APP_DB_SSL": "false", + "APP_REDIS_HOST": "redis-master.{}.svc.cluster.local".format(app_namespace), + "APP_REDIS_PORT": "6379", + "APP_REDIS_PASSWORD": "", }, live_update=[ sync("./src", "/app/src"), diff --git a/helm/web-app/templates/secret.yaml b/helm/web-app/templates/secret.yaml index 4506ecea..19901673 100644 --- a/helm/web-app/templates/secret.yaml +++ b/helm/web-app/templates/secret.yaml @@ -41,4 +41,42 @@ data: GITHUB_APP_ID: {{ .Values.secrets.githubAppId | default "not_setup" | b64enc | quote }} GITHUB_CLIENT_ID: {{ .Values.secrets.githubClientId | default "not_setup" | b64enc | quote }} GITHUB_APP_INSTALLATION_ID: {{ .Values.secrets.githubInstallationId | default "not_setup" | b64enc | quote }} + # Database secrets + {{- if .Values.secrets.databaseUrl }} + DATABASE_URL: {{ .Values.secrets.databaseUrl | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appDbHost }} + APP_DB_HOST: {{ .Values.secrets.appDbHost | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appDbPort }} + APP_DB_PORT: {{ .Values.secrets.appDbPort | toString | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appDbUser }} + APP_DB_USER: {{ .Values.secrets.appDbUser | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appDbPassword }} + APP_DB_PASSWORD: {{ .Values.secrets.appDbPassword | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appDbName }} + APP_DB_NAME: {{ .Values.secrets.appDbName | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appDbSsl }} + APP_DB_SSL: {{ .Values.secrets.appDbSsl | toString | b64enc | quote }} + {{- end }} + # Redis + {{- if .Values.secrets.redisUrl }} + REDIS_URL: {{ .Values.secrets.redisUrl | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appRedisHost }} + APP_REDIS_HOST: {{ .Values.secrets.appRedisHost | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appRedisPort }} + APP_REDIS_PORT: {{ .Values.secrets.appRedisPort | toString | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appRedisPassword }} + APP_REDIS_PASSWORD: {{ .Values.secrets.appRedisPassword | b64enc | quote }} + {{- end }} + {{- if .Values.secrets.appRedisTls }} + APP_REDIS_TLS: {{ .Values.secrets.appRedisTls | toString | b64enc | quote }} + {{- end }} {{- end }} diff --git a/knexfile.ts b/knexfile.ts index 8961d2ab..13ded8bf 100644 --- a/knexfile.ts +++ b/knexfile.ts @@ -18,16 +18,37 @@ import 'dotenv/config'; import { ConnectionString } from 'connection-string'; import { merge } from 'lodash'; -const { NODE_ENV, DATABASE_URL } = process.env; -const { hosts, user, password, path, port = 5432, params} = new ConnectionString(DATABASE_URL); -const host = hosts?.[0]?.name; -const database = path?.[0]; -const ssl = params?.ssl == "true" ? { rejectUnauthorized: false } : false +const { NODE_ENV, DATABASE_URL, APP_DB_HOST, APP_DB_PORT, APP_DB_USER, APP_DB_PASSWORD, APP_DB_NAME, APP_DB_SSL } = + process.env; -// console.log('Running database migrations with the following arguments 🏎️', { -// envValues: { NODE_ENV, DATABASE_URL }, -// resolvedValues: { host, user, password, database, port }, -// }); +let host: string | undefined; +let user: string | undefined; +let password: string | undefined; +let database: string | undefined; +let port: number; +let ssl: boolean | { rejectUnauthorized: boolean }; + +if (APP_DB_HOST && APP_DB_USER && APP_DB_PASSWORD && APP_DB_NAME) { + host = APP_DB_HOST; + user = APP_DB_USER; + password = APP_DB_PASSWORD; + database = APP_DB_NAME; + port = APP_DB_PORT ? parseInt(APP_DB_PORT, 10) : 5432; + ssl = APP_DB_SSL === 'true' ? { rejectUnauthorized: false } : false; +} else if (DATABASE_URL) { + // Fall back to parsing DATABASE_URL will be removed in future releases + const parsed = new ConnectionString(DATABASE_URL); + host = parsed.hosts?.[0]?.name; + user = parsed.user; + password = parsed.password; + database = parsed.path?.[0]; + port = parsed.port || 5432; + ssl = parsed.params?.ssl == 'true' ? { rejectUnauthorized: false } : false; +} else { + throw new Error( + 'Database configuration not found. Please provide either DATABASE_URL or individual APP_DB_* environment variables.' + ); +} const defaults = { client: 'pg', diff --git a/next.config.js b/next.config.js index c47c8910..d1ea9297 100644 --- a/next.config.js +++ b/next.config.js @@ -26,6 +26,12 @@ module.exports = { APP_ENV: process.env.APP_ENV, CODEFRESH_API_KEY: process.env.CODEFRESH_API_KEY, DATABASE_URL: process.env.DATABASE_URL, + APP_DB_HOST: process.env.APP_DB_HOST, + APP_DB_PORT: process.env.APP_DB_PORT, + APP_DB_USER: process.env.APP_DB_USER, + APP_DB_PASSWORD: process.env.APP_DB_PASSWORD, + APP_DB_NAME: process.env.APP_DB_NAME, + APP_DB_SSL: process.env.APP_DB_SSL, FASTLY_TOKEN: process.env.FASTLY_TOKEN, GITHUB_API_REQUEST_INTERVAL: process.env.GITHUB_API_REQUEST_INTERVAL, GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET, @@ -39,7 +45,10 @@ module.exports = { LOG_LEVEL: process.env.LOG_LEVEL, MAX_GITHUB_API_REQUEST: process.env.MAX_GITHUB_API_REQUEST, REDIS_URL: process.env.REDIS_URL, - REDIS_PORT: process.env.REDIS_PORT, + APP_REDIS_HOST: process.env.APP_REDIS_HOST, + APP_REDIS_PORT: process.env.APP_REDIS_PORT, + APP_REDIS_PASSWORD: process.env.APP_REDIS_PASSWORD, + APP_REDIS_TLS: process.env.APP_REDIS_TLS, GITHUB_APP_INSTALLATION_ID: process.env.GITHUB_APP_INSTALLATION_ID, PINO_PRETTY: process.env.PINO_PRETTY, ENVIRONMENT: process.env.ENVIRONMENT, diff --git a/scripts/k8-start.sh b/scripts/k8-start.sh index 92662d9e..4f8f5279 100755 --- a/scripts/k8-start.sh +++ b/scripts/k8-start.sh @@ -13,15 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. - set -e cd /app # check and error if required env vars are not set required_vars=( - DATABASE_URL - REDIS_URL GITHUB_APP_ID GITHUB_CLIENT_ID GITHUB_APP_INSTALLATION_ID @@ -30,7 +27,32 @@ required_vars=( GITHUB_WEBHOOK_SECRET ) +db_configured=false +if [ -n "$APP_DB_HOST" ] && [ -n "$APP_DB_USER" ] && [ -n "$APP_DB_PASSWORD" ] && [ -n "$APP_DB_NAME" ]; then + db_configured=true +elif [ -n "$DATABASE_URL" ]; then + echo "⚠️ Using legacy DATABASE_URL configuration (falling back)" + db_configured=true +fi + +redis_configured=false +if [ -n "$APP_REDIS_HOST" ]; then + redis_configured=true +elif [ -n "$REDIS_URL" ]; then + echo "⚠️ Using legacy REDIS_URL configuration (falling back)" + redis_configured=true +fi + missing=() + +if [ "$db_configured" = false ]; then + missing+=("DATABASE_URL or APP_DB_* variables") +fi + +if [ "$redis_configured" = false ]; then + missing+=("REDIS_URL or APP_REDIS_* variables") +fi + for v in "${required_vars[@]}"; do if [ -z "${!v}" ]; then missing+=("$v") diff --git a/src/server/lib/redisClient.ts b/src/server/lib/redisClient.ts index e761c717..daefd4b9 100644 --- a/src/server/lib/redisClient.ts +++ b/src/server/lib/redisClient.ts @@ -16,7 +16,7 @@ import Redis from 'ioredis'; import Redlock from 'redlock'; -import { REDIS_URL } from 'shared/config'; +import { REDIS_URL, APP_REDIS_HOST, APP_REDIS_PORT, APP_REDIS_PASSWORD, APP_REDIS_TLS } from 'shared/config'; import rootLogger from './logger'; const logger = rootLogger.child({ @@ -32,7 +32,31 @@ export class RedisClient { private readonly bclients: Redis[] = []; private constructor() { - this.redis = new Redis(REDIS_URL); + if (APP_REDIS_HOST) { + const redisConfig: any = { + host: APP_REDIS_HOST, + port: APP_REDIS_PORT ? parseInt(APP_REDIS_PORT, 10) : 6379, + }; + + if (APP_REDIS_PASSWORD) { + redisConfig.password = APP_REDIS_PASSWORD; + } + + if (APP_REDIS_TLS === 'true') { + redisConfig.tls = { + rejectUnauthorized: false, + }; + } + + this.redis = new Redis(redisConfig); + } else if (REDIS_URL) { + this.redis = new Redis(REDIS_URL); + } else { + throw new Error( + 'Redis configuration not found. Please provide either REDIS_URL or individual APP_REDIS_* environment variables.' + ); + } + this.subscriber = this.redis.duplicate(); this.redlock = new Redlock([this.redis], { driftFactor: 0.01, diff --git a/src/shared/config.ts b/src/shared/config.ts index 970df907..9485c2b9 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -33,16 +33,16 @@ const getServerRuntimeConfig = (key: string, fallback?: any): any => { const getProp = (config: Record, key: string, fallback?: any): any => { const value = config[key]; - if (!!value || !!fallback) { - return value || fallback; - } else { - // The literal fallback value of "false" is valid. All other falsy fallbacks are not. - if (fallback === false) return fallback; + if (value !== undefined && value !== null) { + return value; + } + if (fallback !== undefined) { + return fallback; + } - if ('yes' === process.env.BUILD_MODE) return ''; + if ('yes' === process.env.BUILD_MODE) return ''; - throw new Error(`Required config missing: '${key}'`); - } + throw new Error(`Required config missing: '${key}'`); }; export const APP_ENV = getServerRuntimeConfig('APP_ENV', 'development'); @@ -51,8 +51,18 @@ export const IS_STG = APP_ENV === 'staging'; export const IS_DEV = APP_ENV !== 'production'; export const TMP_PATH = `/tmp/lifecycle`; +/** + * @deprecated Use individual APP_DB_* environment variables instead (APP_DB_HOST, APP_DB_USER, APP_DB_PASSWORD, APP_DB_NAME). This will be removed in future releases. + */ export const DATABASE_URL = getServerRuntimeConfig('DATABASE_URL'); +export const APP_DB_HOST = getServerRuntimeConfig('APP_DB_HOST', ''); +export const APP_DB_PORT = getServerRuntimeConfig('APP_DB_PORT', 5432); +export const APP_DB_USER = getServerRuntimeConfig('APP_DB_USER', 'lifecycle'); +export const APP_DB_PASSWORD = getServerRuntimeConfig('APP_DB_PASSWORD', 'lifecycle'); +export const APP_DB_NAME = getServerRuntimeConfig('APP_DB_NAME', ''); +export const APP_DB_SSL = getServerRuntimeConfig('APP_DB_SSL', ''); + export const LIFECYCLE_UI_HOSTHAME_WITH_SCHEME = getServerRuntimeConfig( 'LIFECYCLE_UI_HOSTHAME_WITH_SCHEME', 'REPLACE_ME_WITH_UI_URL' @@ -63,8 +73,16 @@ export const GITHUB_CLIENT_ID = getServerRuntimeConfig('GITHUB_CLIENT_ID'); export const GITHUB_CLIENT_SECRET = getServerRuntimeConfig('GITHUB_CLIENT_SECRET'); export const LIFECYCLE_MODE = getServerRuntimeConfig('LIFECYCLE_MODE'); + +/** + * @deprecated Use individual APP_REDIS_* environment variables instead (APP_REDIS_HOST, APP_REDIS_PORT, APP_REDIS_PASSWORD). This will be removed in future releases. + */ export const REDIS_URL = getServerRuntimeConfig('REDIS_URL'); -export const REDIS_PORT = getServerRuntimeConfig('REDIS_PORT', 6379); + +export const APP_REDIS_HOST = getServerRuntimeConfig('APP_REDIS_HOST', ''); +export const APP_REDIS_PORT = getServerRuntimeConfig('APP_REDIS_PORT', 6379); +export const APP_REDIS_PASSWORD = getServerRuntimeConfig('APP_REDIS_PASSWORD', ''); +export const APP_REDIS_TLS = getServerRuntimeConfig('APP_REDIS_TLS', 'false'); export const GITHUB_PRIVATE_KEY = getServerRuntimeConfig('GITHUB_PRIVATE_KEY') .replace(/\\n/g, '\n') diff --git a/sysops/dockerfiles/tilt.app.Dockerfile b/sysops/dockerfiles/tilt.app.Dockerfile index fb0db11a..556f4ee7 100644 --- a/sysops/dockerfiles/tilt.app.Dockerfile +++ b/sysops/dockerfiles/tilt.app.Dockerfile @@ -45,11 +45,25 @@ RUN pnpm install --frozen-lockfile COPY . . -ARG DATABASE_URL -ARG REDIS_URL +ARG APP_DB_HOST +ARG APP_DB_PORT +ARG APP_DB_USER +ARG APP_DB_PASSWORD +ARG APP_DB_NAME +ARG APP_DB_SSL +ARG APP_REDIS_HOST +ARG APP_REDIS_PORT +ARG APP_REDIS_PASSWORD -ENV REDIS_URL=${REDIS_URL} -ENV DATABASE_URL=${DATABASE_URL} +ENV APP_DB_HOST=${APP_DB_HOST} +ENV APP_DB_PORT=${APP_DB_PORT} +ENV APP_DB_USER=${APP_DB_USER} +ENV APP_DB_PASSWORD=${APP_DB_PASSWORD} +ENV APP_DB_NAME=${APP_DB_NAME} +ENV APP_DB_SSL=${APP_DB_SSL} +ENV APP_REDIS_HOST=${APP_REDIS_HOST} +ENV APP_REDIS_PORT=${APP_REDIS_PORT} +ENV APP_REDIS_PASSWORD=${APP_REDIS_PASSWORD} # Expose the required port ENV PORT 3000