diff --git a/.changeset/env-vars-tracing-forceflush-typecheck.md b/.changeset/env-vars-tracing-forceflush-typecheck.md new file mode 100644 index 00000000000..9d90c0383d7 --- /dev/null +++ b/.changeset/env-vars-tracing-forceflush-typecheck.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Fix `@trigger.dev/core` build: cast the underlying log record exporter when calling `forceFlush` so it typechecks against the updated OpenTelemetry `LogRecordExporter` type (which no longer declares `forceFlush`). diff --git a/.server-changes/environment-variables-page-performance.md b/.server-changes/environment-variables-page-performance.md new file mode 100644 index 00000000000..2bd802cccba --- /dev/null +++ b/.server-changes/environment-variables-page-performance.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Make the Environment Variables page fast for projects with many variables across many environments (windowed SSR + table virtualization, decrypt only non-secret values, lightweight create-page loaders) diff --git a/apps/webapp/app/components/primitives/Table.tsx b/apps/webapp/app/components/primitives/Table.tsx index d69e1201035..bd3a27868aa 100644 --- a/apps/webapp/app/components/primitives/Table.tsx +++ b/apps/webapp/app/components/primitives/Table.tsx @@ -130,12 +130,13 @@ export const TableHeader = forwardRef type TableBodyProps = { className?: string; children?: ReactNode; + style?: React.CSSProperties; }; export const TableBody = forwardRef( - ({ className, children }, ref) => { + ({ className, children, style }, ref) => { return ( - + {children} ); diff --git a/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts b/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts index aff55263fec..2ab7f1f5a1b 100644 --- a/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts @@ -1,7 +1,6 @@ import { PrismaClient, prisma } from "~/db.server"; import { Project } from "~/models/project.server"; import { User } from "~/models/user.server"; -import { filterOrphanedEnvironments, sortEnvironments } from "~/utils/environmentSort"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; import type { EnvironmentVariableUpdater } from "~/v3/environmentVariables/repository"; import { @@ -9,6 +8,7 @@ import { EnvSlug, } from "~/v3/vercel/vercelProjectIntegrationSchema"; import { VercelIntegrationService } from "~/services/vercelIntegration.server"; +import { loadEnvironmentVariablesEnvironments } from "./environmentVariablesEnvironments.server"; type Result = Awaited>; export type EnvironmentVariableWithSetValues = Result["environmentVariables"][number]; @@ -62,16 +62,7 @@ export class EnvironmentVariablesPresenter { }, }, where: { - project: { - slug: projectSlug, - organization: { - members: { - some: { - userId, - }, - }, - }, - }, + projectId: project.id, }, }); @@ -111,32 +102,29 @@ export class EnvironmentVariablesPresenter { const usersRecord: Record = Object.fromEntries(users.map((u) => [u.id, u])); - const environments = await this.#prismaClient.runtimeEnvironment.findMany({ - select: { - id: true, - type: true, - isBranchableEnvironment: true, - branchName: true, - orgMember: { - select: { - userId: true, - }, - }, - }, - where: { - project: { - slug: projectSlug, - }, - archivedAt: null, - }, - }); - - const sortedEnvironments = sortEnvironments(filterOrphanedEnvironments(environments)).filter( - (e) => e.orgMember?.userId === userId || e.orgMember === null - ); + const { environments: sortedEnvironments, hasStaging } = + await loadEnvironmentVariablesEnvironments( + this.#prismaClient, + { userId, projectId: project.id }, + { skipProjectAccessCheck: true } + ); const repository = new EnvironmentVariablesRepository(this.#prismaClient); - const variables = await repository.getProject(project.id); + + const nonSecretItems: Array<{ environmentId: string; key: string }> = []; + for (const environmentVariable of environmentVariables) { + for (const env of sortedEnvironments) { + const valueRecord = environmentVariable.values.find((v) => v.environmentId === env.id); + if (valueRecord && !valueRecord.isSecret) { + nonSecretItems.push({ environmentId: env.id, key: environmentVariable.key }); + } + } + } + + const variableValuesByEnvAndKey = await repository.getVariableValuesForKeys( + project.id, + nonSecretItems + ); // Get Vercel integration data if it exists const vercelService = new VercelIntegrationService(this.#prismaClient); @@ -153,14 +141,19 @@ export class EnvironmentVariablesPresenter { return { environmentVariables: environmentVariables .flatMap((environmentVariable) => { - const variable = variables.find((v) => v.key === environmentVariable.key); - return sortedEnvironments.flatMap((env) => { - const val = variable?.values.find((v) => v.environment.id === env.id); const valueRecord = environmentVariable.values.find((v) => v.environmentId === env.id); const isSecret = valueRecord?.isSecret ?? false; - if (!val || !valueRecord) { + if (!valueRecord) { + return []; + } + + const val = isSecret + ? undefined + : variableValuesByEnvAndKey.get(`${env.id}:${environmentVariable.key}`); + + if (!isSecret && val === undefined) { return []; } @@ -185,7 +178,7 @@ export class EnvironmentVariablesPresenter { id: environmentVariable.id, key: environmentVariable.key, environment: { type: env.type, id: env.id, branchName: env.branchName }, - value: isSecret ? "" : val.value, + value: isSecret ? "" : val!, isSecret, version: valueRecord.version, lastUpdatedBy, @@ -196,13 +189,8 @@ export class EnvironmentVariablesPresenter { }); }) .sort((a, b) => a.key.localeCompare(b.key)), - environments: sortedEnvironments.map((environment) => ({ - id: environment.id, - type: environment.type, - isBranchableEnvironment: environment.isBranchableEnvironment, - branchName: environment.branchName, - })), - hasStaging: environments.some((environment) => environment.type === "STAGING"), + environments: sortedEnvironments, + hasStaging, // Vercel integration data vercelIntegration: vercelIntegration ? { diff --git a/apps/webapp/app/presenters/v3/environmentVariablesEnvironments.server.ts b/apps/webapp/app/presenters/v3/environmentVariablesEnvironments.server.ts new file mode 100644 index 00000000000..218d0be7eb6 --- /dev/null +++ b/apps/webapp/app/presenters/v3/environmentVariablesEnvironments.server.ts @@ -0,0 +1,75 @@ +import type { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { type PrismaClient } from "~/db.server"; +import { filterOrphanedEnvironments, sortEnvironments } from "~/utils/environmentSort"; + +export type EnvironmentVariablesEnvironment = { + id: string; + type: RuntimeEnvironmentType; + isBranchableEnvironment: boolean; + branchName: string | null; +}; + +export type EnvironmentVariablesEnvironmentsResult = { + environments: EnvironmentVariablesEnvironment[]; + hasStaging: boolean; +}; + +export async function loadEnvironmentVariablesEnvironments( + prismaClient: PrismaClient, + { userId, projectId }: { userId: string; projectId: string }, + options?: { skipProjectAccessCheck?: boolean } +): Promise { + if (!options?.skipProjectAccessCheck) { + const project = await prismaClient.project.findFirst({ + select: { + id: true, + }, + where: { + id: projectId, + organization: { + members: { + some: { + userId, + }, + }, + }, + }, + }); + + if (!project) { + throw new Error("Project not found"); + } + } + + const environments = await prismaClient.runtimeEnvironment.findMany({ + select: { + id: true, + type: true, + isBranchableEnvironment: true, + branchName: true, + orgMember: { + select: { + userId: true, + }, + }, + }, + where: { + projectId, + archivedAt: null, + }, + }); + + const sortedEnvironments = sortEnvironments(filterOrphanedEnvironments(environments)).filter( + (environment) => environment.orgMember?.userId === userId || environment.orgMember === null + ); + + return { + environments: sortedEnvironments.map((environment) => ({ + id: environment.id, + type: environment.type, + isBranchableEnvironment: environment.isBranchableEnvironment, + branchName: environment.branchName, + })), + hasStaging: environments.some((environment) => environment.type === "STAGING"), + }; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx index 86bd5bbc95d..4c318323b58 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx @@ -9,10 +9,11 @@ import { import { parse } from "@conform-to/zod"; import { LockClosedIcon, LockOpenIcon, PlusIcon, XMarkIcon } from "@heroicons/react/20/solid"; import { Form, useActionData, useNavigate, useNavigation } from "@remix-run/react"; -import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import dotenv from "dotenv"; -import { type RefObject, useCallback, useEffect, useRef, useState } from "react"; -import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson"; +import { type RefObject, useCallback, useRef, useState } from "react"; +import { redirect } from "remix-typedjson"; +import invariant from "tiny-invariant"; import { z } from "zod"; import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel"; import { Button, LinkButton } from "~/components/primitives/Buttons"; @@ -39,13 +40,15 @@ import { useEnvironment } from "~/hooks/useEnvironment"; import { useList } from "~/hooks/useList"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { EnvironmentVariablesPresenter } from "~/presenters/v3/EnvironmentVariablesPresenter.server"; -import { logger } from "~/services/logger.server"; +import { useTypedMatchesData } from "~/hooks/useTypedMatchData"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; +import { + environmentVariablesRouteId, + type loader as environmentVariablesLoader, +} from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route"; import { EnvironmentParamSchema, - ProjectParamSchema, v3BillingPath, v3EnvironmentVariablesPath, } from "~/utils/pathBuilder"; @@ -53,29 +56,6 @@ import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/enviro import { EnvironmentVariableKey } from "~/v3/environmentVariables/repository"; import { Select, SelectItem } from "~/components/primitives/Select"; -export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); - const { projectParam } = ProjectParamSchema.parse(params); - - try { - const presenter = new EnvironmentVariablesPresenter(); - const { environments, hasStaging } = await presenter.call({ - userId, - projectSlug: projectParam, - }); - - return typedjson({ - environments, - hasStaging, - }); - } catch (error) { - throw new Response(undefined, { - status: 400, - statusText: "Something went wrong, if this problem persists please contact support.", - }); - } -}; - const Variable = z.object({ key: EnvironmentVariableKey, value: z.string().nonempty("Value is required"), @@ -185,8 +165,15 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { }; export default function Page() { - const [isOpen, setIsOpen] = useState(false); - const { environments, hasStaging } = useTypedLoaderData(); + const [isOpen, setIsOpen] = useState(true); + const parentData = useTypedMatchesData({ + id: environmentVariablesRouteId, + }); + invariant( + parentData, + "Environment variables page loader data must be defined when rendering the create dialog" + ); + const { environments, hasStaging } = parentData; const lastSubmission = useActionData(); const navigation = useNavigation(); const navigate = useNavigate(); @@ -259,10 +246,6 @@ export default function Page() { const [revealAll, setRevealAll] = useState(true); - useEffect(() => { - setIsOpen(true); - }, []); - return ( { ]; }; +type PageVercelIntegration = NonNullable< + Awaited>["vercelIntegration"] +>; + +export type EnvironmentVariablesPageLoaderData = { + environmentVariables: EnvironmentVariableWithSetValues[]; + environments: EnvironmentVariablesEnvironment[]; + hasStaging: boolean; + vercelIntegration: PageVercelIntegration | null; +}; + +export const environmentVariablesRouteId = + "routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables"; + export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const { projectParam } = ProjectParamSchema.parse(params); @@ -258,27 +281,45 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { } }; +const SSR_ROW_WINDOW = 50; +const ROW_ESTIMATE_HEIGHT = 44; +const VIRTUAL_OVERSCAN = 10; + +type GroupedEnvironmentVariable = EnvironmentVariableWithSetValues & { + isFirstTime: boolean; + isLastTime: boolean; + occurences: number; +}; + export default function Page() { + const loaderData = useTypedLoaderData(); + + return ; +} + +function EnvironmentVariablesListPage({ + loaderData, +}: { + loaderData: EnvironmentVariablesPageLoaderData; +}) { const [revealAll, setRevealAll] = useState(false); - const { environmentVariables, environments, vercelIntegration } = - useTypedLoaderData(); + const { environmentVariables, vercelIntegration } = loaderData; const organization = useOrganization(); const project = useProject(); const environment = useEnvironment(); const { value } = useSearchParams(); const urlSearch = value("search") ?? ""; - const { setFilterText, filteredItems } = useFuzzyFilter({ + const { filteredItems } = useFuzzyFilter({ items: environmentVariables, keys: ["key", "value", "environment.type", "environment.branchName"], + filterText: urlSearch, }); - useEffect(() => { - setFilterText(urlSearch); - }, [urlSearch, setFilterText]); + const tableScrollRef = useRef(null); // Add isFirst and isLast to each environment variable // They're set based on if they're the first or last time that `key` has been seen in the list - const groupedEnvironmentVariables = useMemo(() => { + const groupedEnvironmentVariables = useMemo((): GroupedEnvironmentVariable[] => { // Create a map to track occurrences of each key const keyOccurrences = new Map(); @@ -313,6 +354,22 @@ export default function Page() { }); }, [filteredItems]); + const shouldVirtualize = groupedEnvironmentVariables.length > SSR_ROW_WINDOW; + const [isVirtualized, setIsVirtualized] = useState(false); + + useLayoutEffect(() => { + setIsVirtualized(shouldVirtualize); + }, [shouldVirtualize]); + + const staticRows = useMemo(() => { + if (shouldVirtualize) { + return groupedEnvironmentVariables.slice(0, SSR_ROW_WINDOW); + } + return groupedEnvironmentVariables; + }, [groupedEnvironmentVariables, shouldVirtualize]); + + const vercelColumnCount = vercelIntegration?.enabled ? 6 : 5; + return ( @@ -328,7 +385,7 @@ export default function Page() { -
+
{environmentVariables.length > 0 && (
@@ -350,196 +407,285 @@ export default function Page() {
)} - - - - - Key - - - Value - - - - Environment - - - } - content="Dev environment variables specified here will be overridden by ones in your .env file when running locally." - className="max-w-60" - /> - - {vercelIntegration?.enabled && ( - +
+
+ + + + Key + + + Value + + - Sync + Environment } - content="When enabled, this variable will be pulled from Vercel during builds. Requires 'Pull env vars before build' to be enabled in settings." + content="Dev environment variables specified here will be overridden by ones in your .env file when running locally." + className="max-w-60" /> - )} - - Updated - - - Actions - - - - - {groupedEnvironmentVariables.length > 0 ? ( - groupedEnvironmentVariables.map((variable) => { - const cellClassName = "py-2"; - let borderedCellClassName = ""; - - if (variable.occurences > 1) { - borderedCellClassName = - "relative after:absolute after:bottom-0 after:left-0 after:right-0 after:h-px after:bg-grid-bright group-hover/table-row:after:bg-grid-bright group-hover/table-row:before:bg-grid-bright"; - if (variable.isLastTime) { - borderedCellClassName = ""; - } else if (variable.isFirstTime) { - } - } else { - } - - return ( - - - {variable.isFirstTime ? ( - - ) : null} - - - {variable.isSecret ? ( - - - Secret - - } - content="This variable is secret and cannot be revealed." - /> - ) : ( - - )} - - - - - - {vercelIntegration?.enabled && ( - - {variable.environment.type !== "DEVELOPMENT" && ( - - )} - - )} - -
- {variable.updatedByUser ? ( -
- - {variable.updatedByUser.name} -
- ) : variable.lastUpdatedBy?.type === "integration" && - variable.lastUpdatedBy?.integration === "vercel" ? ( -
- - - {variable.lastUpdatedBy.integration} - -
- ) : null} - {variable.updatedAt ? ( - - - - ) : null} -
- - - - - + {vercelIntegration?.enabled && ( + + + Sync + + } + content="When enabled, this variable will be pulled from Vercel during builds. Requires 'Pull env vars before build' to be enabled in settings." /> - - ); - }) - ) : ( - - - {environmentVariables.length === 0 ? ( -
- You haven't set any environment variables yet. - - Add new - -
- ) : ( -
- No variables match your search. -
- )} - + + )} + + Updated + + + Actions + + + {groupedEnvironmentVariables.length > 0 ? ( + isVirtualized && shouldVirtualize ? ( + + ) : ( + + {staticRows.map((variable) => ( + + ))} + + ) + ) : ( + + + + {environmentVariables.length === 0 ? ( +
+ You haven't set any environment variables yet. + + Add new + +
+ ) : ( +
+ No variables match your search. +
+ )} + + + )} - -
+ +
- + ); } +function getBorderedCellClassName(variable: GroupedEnvironmentVariable) { + if (variable.occurences <= 1) { + return ""; + } + + if (variable.isLastTime) { + return ""; + } + + return "relative after:absolute after:bottom-0 after:left-0 after:right-0 after:h-px after:bg-grid-bright group-hover/table-row:after:bg-grid-bright group-hover/table-row:before:bg-grid-bright"; +} + +function EnvironmentVariableTableRow({ + variable, + revealAll, + vercelIntegration, +}: { + variable: GroupedEnvironmentVariable; + revealAll: boolean; + vercelIntegration: PageVercelIntegration | null; +}) { + const cellClassName = "py-2"; + const borderedCellClassName = getBorderedCellClassName(variable); + + return ( + + + {variable.isFirstTime ? ( + + ) : null} + + + {variable.isSecret ? ( + + + Secret + + } + content="This variable is secret and cannot be revealed." + /> + ) : ( + + )} + + + + + + {vercelIntegration?.enabled && ( + + {variable.environment.type !== "DEVELOPMENT" && ( + + )} + + )} + +
+ {variable.updatedAt ? ( + + + + ) : null} + {variable.updatedByUser ? ( +
+ + {variable.updatedByUser.name} +
+ ) : variable.lastUpdatedBy?.type === "integration" && + variable.lastUpdatedBy?.integration === "vercel" ? ( +
+ + + {variable.lastUpdatedBy.integration} + +
+ ) : null} +
+ + + + + + } + /> + + ); +} + +function EnvironmentVariablesVirtualTableBody({ + groupedEnvironmentVariables, + scrollRef, + revealAll, + vercelIntegration, + columnCount, +}: { + groupedEnvironmentVariables: GroupedEnvironmentVariable[]; + scrollRef: RefObject; + revealAll: boolean; + vercelIntegration: PageVercelIntegration | null; + columnCount: number; +}) { + const rowVirtualizer = useVirtualizer({ + count: groupedEnvironmentVariables.length, + getScrollElement: () => scrollRef.current, + estimateSize: () => ROW_ESTIMATE_HEIGHT, + overscan: VIRTUAL_OVERSCAN, + }); + + const virtualItems = rowVirtualizer.getVirtualItems(); + const topSpacerHeight = virtualItems[0]?.start ?? 0; + const bottomSpacerHeight = + rowVirtualizer.getTotalSize() - (virtualItems.at(-1)?.end ?? 0); + + return ( + + {topSpacerHeight > 0 && ( + + + + )} + {virtualItems.map((virtualRow) => { + const variable = groupedEnvironmentVariables[virtualRow.index]; + if (!variable) { + return null; + } + + return ( + + ); + })} + {bottomSpacerHeight > 0 && ( + + + + )} + + ); +} + function EditEnvironmentVariablePanel({ variable, revealAll, diff --git a/apps/webapp/app/services/secrets/secretStore.server.ts b/apps/webapp/app/services/secrets/secretStore.server.ts index 855463fcd13..58af8e86f76 100644 --- a/apps/webapp/app/services/secrets/secretStore.server.ts +++ b/apps/webapp/app/services/secrets/secretStore.server.ts @@ -18,6 +18,7 @@ type ProviderInitializationOptions = { export interface SecretStoreProvider { getSecret(schema: z.Schema, key: string): Promise; getSecrets(schema: z.Schema, keyPrefix: string): Promise<{ key: string; value: T }[]>; + getSecretsByKeys(schema: z.Schema, keys: string[]): Promise<{ key: string; value: T }[]>; setSecret(key: string, value: T): Promise; deleteSecret(key: string): Promise; } @@ -48,6 +49,10 @@ export class SecretStore { return this.provider.getSecrets(schema, keyPrefix); } + getSecretsByKeys(schema: z.Schema, keys: string[]): Promise<{ key: string; value: T }[]> { + return this.provider.getSecretsByKeys(schema, keys); + } + deleteSecret(key: string): Promise { return this.provider.deleteSecret(key); } @@ -83,29 +88,7 @@ class PrismaSecretStore implements SecretStoreProvider { return undefined; } - if (secret.version === "1") { - return schema.parse(secret.value); - } - - const encryptedData = EncryptedSecretValueSchema.safeParse(secret.value); - - if (!encryptedData.success) { - throw new Error(`Unable to parse encrypted secret ${key}: ${encryptedData.error.message}`); - } - - const decrypted = await this.#decrypt( - encryptedData.data.nonce, - encryptedData.data.ciphertext, - encryptedData.data.tag - ); - - const parsedDecrypted = safeJsonParse(decrypted); - - if (!parsedDecrypted) { - return; - } - - return schema.parse(parsedDecrypted); + return this.#parseStoredSecret(schema, secret); } async getSecrets( @@ -120,37 +103,74 @@ class PrismaSecretStore implements SecretStoreProvider { }, }); + return this.#parseStoredSecrets(schema, secrets); + } + + async getSecretsByKeys( + schema: z.Schema, + keys: string[] + ): Promise<{ key: string; value: T }[]> { + if (keys.length === 0) { + return []; + } + + const secrets = await this.#prismaClient.secretStore.findMany({ + where: { + key: { + in: keys, + }, + }, + }); + + return this.#parseStoredSecrets(schema, secrets); + } + + async #parseStoredSecrets( + schema: z.Schema, + secrets: Array<{ key: string; value: unknown; version: string }> + ): Promise<{ key: string; value: T }[]> { const results = [] as { key: string; value: T }[]; for (const secret of secrets) { - if (secret.version === "1") { - results.push({ key: secret.key, value: schema.parse(secret.value) }); + const value = await this.#parseStoredSecret(schema, secret); + if (value !== undefined) { + results.push({ key: secret.key, value }); } + } - const encryptedData = EncryptedSecretValueSchema.safeParse(secret.value); + return results; + } - if (!encryptedData.success) { - throw new Error( - `Unable to parse encrypted secret ${secret.key}: ${encryptedData.error.message}` - ); - } + async #parseStoredSecret( + schema: z.Schema, + secret: { key: string; value: unknown; version: string } + ): Promise { + if (secret.version === "1") { + return schema.parse(secret.value); + } - const decrypted = await this.#decrypt( - encryptedData.data.nonce, - encryptedData.data.ciphertext, - encryptedData.data.tag + const encryptedData = EncryptedSecretValueSchema.safeParse(secret.value); + + if (!encryptedData.success) { + throw new Error( + `Unable to parse encrypted secret ${secret.key}: ${encryptedData.error.message}` ); + } - const parsedDecrypted = safeJsonParse(decrypted); - if (!parsedDecrypted) { - logger.error(`Secret isn't JSON ${secret.key}`); - continue; - } + const decrypted = await this.#decrypt( + encryptedData.data.nonce, + encryptedData.data.ciphertext, + encryptedData.data.tag + ); + + const parsedDecrypted = safeJsonParse(decrypted); - results.push({ key: secret.key, value: schema.parse(parsedDecrypted) }); + if (!parsedDecrypted) { + logger.error(`Secret isn't JSON ${secret.key}`); + return undefined; } - return results; + return schema.parse(parsedDecrypted); } async setSecret(key: string, value: T): Promise { diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 9cc41995664..4a8e0e36c0a 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -577,6 +577,42 @@ export class EnvironmentVariablesRepository implements Repository { return results; } + async getVariableValuesForKeys( + projectId: string, + items: Array<{ environmentId: string; key: string }> + ): Promise> { + if (items.length === 0) { + return new Map(); + } + + const uniqueItems = new Map(); + for (const item of items) { + uniqueItems.set(`${item.environmentId}:${item.key}`, item); + } + + const secretStore = getSecretStore("DATABASE", { + prismaClient: this.prismaClient, + }); + + const storeKeys = Array.from(uniqueItems.values()).map((item) => + secretKey(projectId, item.environmentId, item.key) + ); + + const secrets = await secretStore.getSecretsByKeys(SecretValue, storeKeys); + const secretsByStoreKey = new Map(secrets.map((secret) => [secret.key, secret.value.secret])); + + const values = new Map(); + for (const item of uniqueItems.values()) { + const storeKey = secretKey(projectId, item.environmentId, item.key); + const value = secretsByStoreKey.get(storeKey); + if (value !== undefined) { + values.set(`${item.environmentId}:${item.key}`, value); + } + } + + return values; + } + async getEnvironmentWithRedactedSecrets( projectId: string, environmentId: string, diff --git a/apps/webapp/app/v3/environmentVariables/repository.ts b/apps/webapp/app/v3/environmentVariables/repository.ts index ea027bc2ca8..84dee8beaf3 100644 --- a/apps/webapp/app/v3/environmentVariables/repository.ts +++ b/apps/webapp/app/v3/environmentVariables/repository.ts @@ -106,6 +106,14 @@ export interface Repository { edit(projectId: string, options: EditEnvironmentVariable): Promise; editValue(projectId: string, options: EditEnvironmentVariableValue): Promise; getProject(projectId: string): Promise; + /** + * Fetch and decrypt only the given env var values (for dashboard display of non-secret rows). + * Map keys are `${environmentId}:${variableKey}`. + */ + getVariableValuesForKeys( + projectId: string, + items: Array<{ environmentId: string; key: string }> + ): Promise>; /** * Get the environment variables for a given environment, it does NOT return values for secret variables */ diff --git a/apps/webapp/test/EnvironmentVariablesPresenter.test.ts b/apps/webapp/test/EnvironmentVariablesPresenter.test.ts new file mode 100644 index 00000000000..1fd2c9e9619 --- /dev/null +++ b/apps/webapp/test/EnvironmentVariablesPresenter.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + $transaction: async ( + prismaClient: { + $transaction: (fn: (tx: unknown) => Promise) => Promise; + }, + nameOrFn: string | ((tx: unknown) => Promise), + fnOrOptions?: ((tx: unknown) => Promise) | unknown + ) => { + const fn = + typeof nameOrFn === "string" + ? (fnOrOptions as (tx: unknown) => Promise) + : nameOrFn; + + return prismaClient.$transaction(fn); + }, +})); + +import { postgresTest } from "@internal/testcontainers"; +import { EnvironmentVariablesPresenter } from "~/presenters/v3/EnvironmentVariablesPresenter.server"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { + createEnvironmentVariable, + createRuntimeEnvironment, + createTestOrgProjectWithMember, +} from "./fixtures/environmentVariablesFixtures"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("EnvironmentVariablesPresenter", () => { + postgresTest("keeps secret values redacted while returning non-secret values", async ({ prisma }) => { + const { user, organization, project, projectSlug } = await createTestOrgProjectWithMember(prisma); + const production = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + await createEnvironmentVariable(repository, project.id, { + environmentId: production.id, + key: "SECRET_VAR", + value: "super-secret", + isSecret: true, + userId: user.id, + }); + await createEnvironmentVariable(repository, project.id, { + environmentId: production.id, + key: "PLAIN_VAR", + value: "plain-value", + isSecret: false, + userId: user.id, + }); + + const result = await new EnvironmentVariablesPresenter(prisma).call({ + userId: user.id, + projectSlug, + }); + + const secretVariable = result.environmentVariables.find((variable) => variable.key === "SECRET_VAR"); + const nonSecretVariable = result.environmentVariables.find((variable) => variable.key === "PLAIN_VAR"); + + expect(secretVariable).toBeDefined(); + expect(nonSecretVariable).toBeDefined(); + expect(secretVariable!.value).toBe(""); + expect(nonSecretVariable!.value).toBe("plain-value"); + }); +}); diff --git a/apps/webapp/test/environmentVariablesEnvironments.test.ts b/apps/webapp/test/environmentVariablesEnvironments.test.ts new file mode 100644 index 00000000000..2555bcae0f7 --- /dev/null +++ b/apps/webapp/test/environmentVariablesEnvironments.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + $transaction: async ( + prismaClient: { + $transaction: (fn: (tx: unknown) => Promise) => Promise; + }, + nameOrFn: string | ((tx: unknown) => Promise), + fnOrOptions?: ((tx: unknown) => Promise) | unknown + ) => { + const fn = + typeof nameOrFn === "string" + ? (fnOrOptions as (tx: unknown) => Promise) + : nameOrFn; + + return prismaClient.$transaction(fn); + }, +})); + +import { postgresTest } from "@internal/testcontainers"; +import { loadEnvironmentVariablesEnvironments } from "~/presenters/v3/environmentVariablesEnvironments.server"; +import { + createRuntimeEnvironment, + createTestOrgProjectWithMember, + createTestUser, +} from "./fixtures/environmentVariablesFixtures"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("loadEnvironmentVariablesEnvironments", () => { + postgresTest("returns environments for a project member", async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + + const production = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const result = await loadEnvironmentVariablesEnvironments(prisma, { + userId: user.id, + projectId: project.id, + }); + + expect(result.environments.map((environment) => environment.id)).toContain(production.id); + expect(result.environments.every((environment) => typeof environment.id === "string")).toBe(true); + }); + + postgresTest("rejects users who are not project members", async ({ prisma }) => { + const { project } = await createTestOrgProjectWithMember(prisma); + const outsider = await createTestUser(prisma); + + await expect( + loadEnvironmentVariablesEnvironments(prisma, { + userId: outsider.id, + projectId: project.id, + }) + ).rejects.toThrow("Project not found"); + }); + + postgresTest("filters shared, personal, and inaccessible environments", async ({ prisma }) => { + const { user, organization, project, orgMember } = await createTestOrgProjectWithMember(prisma); + const otherUser = await createTestUser(prisma); + const otherOrgMember = await prisma.orgMember.create({ + data: { + organizationId: organization.id, + userId: otherUser.id, + role: "MEMBER", + }, + }); + + const sharedProduction = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + const currentUserDev = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "DEVELOPMENT", + orgMemberId: orgMember.id, + }); + const otherUserDev = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "DEVELOPMENT", + orgMemberId: otherOrgMember.id, + }); + const orphanedDev = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "DEVELOPMENT", + orgMemberId: null, + }); + + const result = await loadEnvironmentVariablesEnvironments(prisma, { + userId: user.id, + projectId: project.id, + }); + + const environmentIds = result.environments.map((environment) => environment.id); + + expect(environmentIds).toContain(sharedProduction.id); + expect(environmentIds).toContain(currentUserDev.id); + expect(environmentIds).not.toContain(otherUserDev.id); + expect(environmentIds).not.toContain(orphanedDev.id); + }); + + postgresTest("returns hasStaging true when a staging environment exists", async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + + await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "STAGING", + }); + + const result = await loadEnvironmentVariablesEnvironments(prisma, { + userId: user.id, + projectId: project.id, + }); + + expect(result.hasStaging).toBe(true); + }); + + postgresTest("returns hasStaging false when no staging environment exists", async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + + await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const result = await loadEnvironmentVariablesEnvironments(prisma, { + userId: user.id, + projectId: project.id, + }); + + expect(result.hasStaging).toBe(false); + }); +}); diff --git a/apps/webapp/test/environmentVariablesRepository.test.ts b/apps/webapp/test/environmentVariablesRepository.test.ts new file mode 100644 index 00000000000..ead865eb5da --- /dev/null +++ b/apps/webapp/test/environmentVariablesRepository.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + $transaction: async ( + prismaClient: { + $transaction: (fn: (tx: unknown) => Promise) => Promise; + }, + nameOrFn: string | ((tx: unknown) => Promise), + fnOrOptions?: ((tx: unknown) => Promise) | unknown + ) => { + const fn = + typeof nameOrFn === "string" + ? (fnOrOptions as (tx: unknown) => Promise) + : nameOrFn; + + return prismaClient.$transaction(fn); + }, +})); + +import { postgresTest } from "@internal/testcontainers"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { + createEnvironmentVariable, + createRuntimeEnvironment, + createTestOrgProjectWithMember, +} from "./fixtures/environmentVariablesFixtures"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("EnvironmentVariablesRepository.getVariableValuesForKeys", () => { + postgresTest("returns an empty map for an empty items array", async ({ prisma }) => { + const { project } = await createTestOrgProjectWithMember(prisma); + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + const result = await repository.getVariableValuesForKeys(project.id, []); + + expect(result).toBeInstanceOf(Map); + expect(result.size).toBe(0); + }); + + postgresTest("omits missing keys from the result without throwing", async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + const result = await repository.getVariableValuesForKeys(project.id, [ + { environmentId: environment.id, key: "DOES_NOT_EXIST" }, + ]); + + expect(result.size).toBe(0); + expect(result.has(`${environment.id}:DOES_NOT_EXIST`)).toBe(false); + }); + + postgresTest("returns requested values with correct map keys and decrypted values", async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + await createEnvironmentVariable(repository, project.id, { + environmentId: environment.id, + key: "VAR_A", + value: "value-a", + userId: user.id, + }); + await createEnvironmentVariable(repository, project.id, { + environmentId: environment.id, + key: "VAR_B", + value: "value-b", + userId: user.id, + }); + await createEnvironmentVariable(repository, project.id, { + environmentId: environment.id, + key: "VAR_C", + value: "value-c", + userId: user.id, + }); + + const result = await repository.getVariableValuesForKeys(project.id, [ + { environmentId: environment.id, key: "VAR_A" }, + { environmentId: environment.id, key: "VAR_C" }, + ]); + + expect(result).toBeInstanceOf(Map); + expect(result.size).toBe(2); + expect(result.get(`${environment.id}:VAR_A`)).toBe("value-a"); + expect(result.get(`${environment.id}:VAR_C`)).toBe("value-c"); + expect(result.has(`${environment.id}:VAR_B`)).toBe(false); + }); + + postgresTest("deduplicates duplicate environmentId and key requests", async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + await createEnvironmentVariable(repository, project.id, { + environmentId: environment.id, + key: "DEDUP_KEY", + value: "dedup-value", + userId: user.id, + }); + + const request = { environmentId: environment.id, key: "DEDUP_KEY" }; + const result = await repository.getVariableValuesForKeys(project.id, [request, request, request]); + + expect(result.size).toBe(1); + expect(result.get(`${environment.id}:DEDUP_KEY`)).toBe("dedup-value"); + }); + + postgresTest("isolates values by project", async ({ prisma }) => { + const { user, organization, project: projectA } = await createTestOrgProjectWithMember(prisma); + + const projectB = await prisma.project.create({ + data: { + name: "Project B", + slug: `proj-b-${Date.now()}`, + organizationId: organization.id, + externalRef: `ext-b-${Date.now()}`, + }, + }); + + const envA = await createRuntimeEnvironment(prisma, { + projectId: projectA.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + const envB = await createRuntimeEnvironment(prisma, { + projectId: projectB.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + await createEnvironmentVariable(repository, projectA.id, { + environmentId: envA.id, + key: "SHARED_KEY", + value: "project-a-value", + userId: user.id, + }); + await createEnvironmentVariable(repository, projectB.id, { + environmentId: envB.id, + key: "SHARED_KEY", + value: "project-b-value", + userId: user.id, + }); + + const resultForProjectA = await repository.getVariableValuesForKeys(projectA.id, [ + { environmentId: envA.id, key: "SHARED_KEY" }, + ]); + + expect(resultForProjectA.size).toBe(1); + expect(resultForProjectA.get(`${envA.id}:SHARED_KEY`)).toBe("project-a-value"); + expect(resultForProjectA.get(`${envB.id}:SHARED_KEY`)).toBeUndefined(); + + const crossProjectRequest = await repository.getVariableValuesForKeys(projectA.id, [ + { environmentId: envB.id, key: "SHARED_KEY" }, + ]); + + expect(crossProjectRequest.size).toBe(0); + }); +}); diff --git a/apps/webapp/test/fixtures/environmentVariablesFixtures.ts b/apps/webapp/test/fixtures/environmentVariablesFixtures.ts new file mode 100644 index 00000000000..8e358e1295b --- /dev/null +++ b/apps/webapp/test/fixtures/environmentVariablesFixtures.ts @@ -0,0 +1,104 @@ +import type { PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; + +let idCounter = 0; + +export function uniqueId(prefix: string) { + idCounter += 1; + return `${prefix}-${idCounter}-${Date.now()}`; +} + +export async function createTestUser(prisma: PrismaClient, email?: string) { + return prisma.user.create({ + data: { + email: email ?? `${uniqueId("user")}@test.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); +} + +export async function createTestOrgProjectWithMember( + prisma: PrismaClient, + options?: { userId?: string } +) { + const user = options?.userId + ? await prisma.user.findUniqueOrThrow({ where: { id: options.userId } }) + : await createTestUser(prisma); + + const orgSlug = uniqueId("org"); + const organization = await prisma.organization.create({ + data: { + title: "Test Org", + slug: orgSlug, + members: { create: { userId: user.id, role: "ADMIN" } }, + }, + include: { members: true }, + }); + + const projectSlug = uniqueId("proj"); + const project = await prisma.project.create({ + data: { + name: "Test Project", + slug: projectSlug, + organizationId: organization.id, + externalRef: uniqueId("ext"), + }, + }); + + return { + user, + organization, + project, + orgMember: organization.members[0]!, + projectSlug, + }; +} + +export async function createRuntimeEnvironment( + prisma: PrismaClient, + options: { + projectId: string; + organizationId: string; + type: RuntimeEnvironmentType; + orgMemberId?: string | null; + slug?: string; + } +) { + const slug = options.slug ?? uniqueId("env"); + return prisma.runtimeEnvironment.create({ + data: { + slug, + type: options.type, + projectId: options.projectId, + organizationId: options.organizationId, + orgMemberId: options.orgMemberId ?? null, + apiKey: uniqueId("api"), + pkApiKey: uniqueId("pk"), + shortcode: uniqueId("sc"), + }, + }); +} + +export async function createEnvironmentVariable( + repository: EnvironmentVariablesRepository, + projectId: string, + options: { + environmentId: string; + key: string; + value: string; + isSecret?: boolean; + userId: string; + } +) { + const result = await repository.create(projectId, { + override: true, + environmentIds: [options.environmentId], + variables: [{ key: options.key, value: options.value }], + isSecret: options.isSecret ?? false, + lastUpdatedBy: { type: "user", userId: options.userId }, + }); + + if (!result.success) { + throw new Error(result.error); + } +} diff --git a/packages/core/src/v3/otel/tracingSDK.ts b/packages/core/src/v3/otel/tracingSDK.ts index 435921d50f6..f03bd2fc3a6 100644 --- a/packages/core/src/v3/otel/tracingSDK.ts +++ b/packages/core/src/v3/otel/tracingSDK.ts @@ -523,7 +523,13 @@ class ExternalLogRecordExporterWrapper { } forceFlush(): Promise { - return this.underlyingExporter.forceFlush?.() ?? Promise.resolve(); + const underlyingExporter = this.underlyingExporter as LogRecordExporter & { + forceFlush?: () => Promise; + }; + + return underlyingExporter.forceFlush + ? underlyingExporter.forceFlush() + : Promise.resolve(); } transformLogRecord(