diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index a801779549..e10578db2e 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -101,7 +101,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22.18.0 + node-version: 22.22.3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ffc6c2187d..98d75c1990 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [22.18.0] + node-version: [22.22.3] runtime: - mode: v1 force-v2-all: '' diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 1460568160..dd040178f0 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: - node-version: [22.18.0] + node-version: [22.22.3] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/manual-preview.yml b/.github/workflows/manual-preview.yml index d0e7d7adc4..5ff9de2a2d 100644 --- a/.github/workflows/manual-preview.yml +++ b/.github/workflows/manual-preview.yml @@ -66,7 +66,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22.18.0 + node-version: 22.22.3 - name: ⚙️ Install zx run: npm install -g zx diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7f8b57bcc4..1823c90995 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -22,7 +22,7 @@ jobs: strategy: matrix: - node-version: [22.18.0] + node-version: [22.22.3] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/v2-benchmark-tests.yml b/.github/workflows/v2-benchmark-tests.yml index 54eaf768eb..24f878814b 100644 --- a/.github/workflows/v2-benchmark-tests.yml +++ b/.github/workflows/v2-benchmark-tests.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: - node-version: [22.18.0] + node-version: [22.22.3] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/v2-core-tests.yml b/.github/workflows/v2-core-tests.yml index 81a9382a02..47de660f71 100644 --- a/.github/workflows/v2-core-tests.yml +++ b/.github/workflows/v2-core-tests.yml @@ -33,10 +33,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Use Node.js 22.18.0 + - name: Use Node.js 22.22.3 uses: actions/setup-node@v4 with: - node-version: 22.18.0 + node-version: 22.22.3 - name: 📥 Monorepo install uses: ./.github/actions/pnpm-install @@ -63,10 +63,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Use Node.js 22.18.0 + - name: Use Node.js 22.22.3 uses: actions/setup-node@v4 with: - node-version: 22.18.0 + node-version: 22.22.3 - name: 📥 Monorepo install uses: ./.github/actions/pnpm-install @@ -102,10 +102,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Use Node.js 22.18.0 + - name: Use Node.js 22.22.3 uses: actions/setup-node@v4 with: - node-version: 22.18.0 + node-version: 22.22.3 - name: 📥 Download all coverage artifacts uses: actions/download-artifact@v4 diff --git a/.npmrc b/.npmrc index 756b5df5d1..2f3a15b724 100644 --- a/.npmrc +++ b/.npmrc @@ -4,5 +4,5 @@ auto-install-peers=true lockfile=true # force use npmjs.org registry registry=https://registry.npmjs.org/ -use-node-version=22.18.0 +use-node-version=22.22.3 save-prefix='' diff --git a/apps/nestjs-backend/package.json b/apps/nestjs-backend/package.json index 9d69112059..5836d0e493 100644 --- a/apps/nestjs-backend/package.json +++ b/apps/nestjs-backend/package.json @@ -104,6 +104,7 @@ "eslint-config-next": "15.5.9", "get-tsconfig": "4.7.3", "istanbul-merge": "2.0.0", + "neverthrow": "8.2.0", "npm-run-all2": "6.1.2", "nyc": "15.1.0", "pg-mem": "3.0.5", @@ -167,7 +168,7 @@ "@opentelemetry/instrumentation-ioredis": "0.49.0", "@opentelemetry/instrumentation-nestjs-core": "0.49.0", "@opentelemetry/instrumentation-pg": "0.49.0", - "@opentelemetry/instrumentation-pino": "0.49.0", + "@opentelemetry/instrumentation-pino": "0.54.0", "@opentelemetry/instrumentation-runtime-node": "0.24.0", "@opentelemetry/resources": "2.0.1", "@opentelemetry/sdk-node": "0.201.1", @@ -228,6 +229,7 @@ "keyv": "4.5.4", "knex": "3.1.0", "lodash": "4.17.21", + "markdown-it": "14.1.0", "mime-types": "2.1.35", "minio": "7.1.3", "ms": "2.1.3", diff --git a/apps/nestjs-backend/src/cache/types.ts b/apps/nestjs-backend/src/cache/types.ts index 942cc59623..dce4141198 100644 --- a/apps/nestjs-backend/src/cache/types.ts +++ b/apps/nestjs-backend/src/cache/types.ts @@ -34,7 +34,7 @@ export interface ICacheStore { [key: `waitlist:invite-code:${string}`]: number; [key: `send-mail-rate-limit:${string}`]: boolean; [key: `oauth:token-rate:${string}:${string}`]: number; - [key: `automation:email:rate:${string}:${number}`]: number; + [key: `email:send:rate:${string}:${number}`]: number; [key: `automation:email-att:${string}`]: string[]; [key: `automation:fail-notify-count:${string}`]: number; // Distributed lock keys diff --git a/apps/nestjs-backend/src/configs/threshold.config.ts b/apps/nestjs-backend/src/configs/threshold.config.ts index 83a550b690..215b481a9b 100644 --- a/apps/nestjs-backend/src/configs/threshold.config.ts +++ b/apps/nestjs-backend/src/configs/threshold.config.ts @@ -20,6 +20,9 @@ export const thresholdConfig = registerAs('threshold', () => ({ bigTransactionTimeout: Number( process.env.BIG_TRANSACTION_TIMEOUT ?? 10 * 60 * 1000 /* 10 mins */ ), + // DB statement_timeout (ms) for the search query, so a slow / full-scan search is canceled + // and its connection released instead of being held for minutes. Tune via SEARCH_TIMEOUT. + searchTimeout: Number(process.env.SEARCH_TIMEOUT ?? 15_000 /* 15s */), automationGap: Number(process.env.AUTOMATION_GAP ?? 200), maxAttachmentUploadSize: Number(process.env.MAX_ATTACHMENT_UPLOAD_SIZE ?? Infinity), maxOpenapiAttachmentUploadSize: Number( diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts index 069a3cf712..c932ed8176 100644 --- a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts +++ b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts @@ -22,6 +22,17 @@ const buildDateField = (): IFieldInstance => }, }) as IFieldInstance; +const buildMultipleSelectField = (): IFieldInstance => + ({ + id: 'fldMultiSelect0001', + dbFieldName: 'Tags', + cellValueType: CellValueType.String, + isMultipleCellValue: true, + isStructuredCellValue: false, + type: FieldType.MultipleSelect, + options: {}, + }) as IFieldInstance; + describe('SearchQueryPostgres', () => { const db = knex({ client: 'pg' }); @@ -52,4 +63,19 @@ describe('SearchQueryPostgres', () => { const compiled = builder.getQuery()?.toSQL(); expect(compiled?.sql).toBe('FALSE'); }); + + it('matches multipleSelect as a text cast so the gin_trgm index can be used', () => { + const field = buildMultipleSelectField(); + const builder = new SearchQueryPostgres( + db.queryBuilder(), + field, + ['Beta', 'fldMultiSelect0001'], + [] + ); + + const compiled = builder.getQuery()?.toSQL(); + expect(compiled?.sql).toContain('("Tags")::text ILIKE'); + expect(compiled?.sql).not.toContain('jsonb_array_elements'); + expect(compiled?.bindings).toEqual(['%Beta%']); + }); }); diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts index 53283be572..20d514ae9c 100644 --- a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts @@ -97,9 +97,11 @@ export class SearchQueryPostgres extends SearchQueryAbstract { case CellValueType.String: { if (isStructuredCellValue) { return this.multipleJson(); - } else { - return this.multipleText(); } + if (field.type === FieldType.MultipleSelect) { + return this.multipleSelectText(); + } + return this.multipleText(); } case CellValueType.DateTime: { return this.multipleDate(); @@ -180,6 +182,16 @@ export class SearchQueryPostgres extends SearchQueryAbstract { ); } + // multipleSelect stores a plain string[] of option names. Match the whole cell as text so the + // predicate is sargable against the gin_trgm index (built on the same ""::text expression) + // instead of a jsonb_array_elements + regex subquery that cannot use the index. Trades negligible + // precision (JSON brackets/quotes become matchable) for index usage. + protected multipleSelectText() { + const { search, knex } = this; + const escapedSearchValue = escapeLikeWildcards(search[0]); + return knex.raw(`(${this.fieldName})::text ILIKE ? ESCAPE '\\'`, [`%${escapedSearchValue}%`]); + } + protected multipleNumber() { const { search, knex } = this; const searchValue = search[0]; diff --git a/apps/nestjs-backend/src/event-emitter/events/event.enum.ts b/apps/nestjs-backend/src/event-emitter/events/event.enum.ts index 6588e55e74..688450e277 100644 --- a/apps/nestjs-backend/src/event-emitter/events/event.enum.ts +++ b/apps/nestjs-backend/src/event-emitter/events/event.enum.ts @@ -51,6 +51,7 @@ export enum Events { SHARED_VIEW_CREATE = 'shared.view.create', SHARED_VIEW_DELETE = 'shared.view.delete', SHARED_VIEW_UPDATE = 'shared.view.update', + SHARED_VIEW_REFRESH = 'shared.view.refresh', USER_SIGNIN = 'user.signin', USER_SIGNUP = 'user.signup', @@ -67,6 +68,20 @@ export enum Events { COLLABORATOR_DELETE = 'collaborator.delete', COLLABORATOR_UPDATE = 'collaborator.update', + // Base-scope collaborator audit actions (parallel to the generic COLLABORATOR_* + // business events above, which are kept for internal pub/sub). Future space-level + // audit can mirror this with SPACE_COLLABORATOR_*. + BASE_COLLABORATOR_CREATE = 'base.collaborator.create', + BASE_COLLABORATOR_DELETE = 'base.collaborator.delete', + BASE_COLLABORATOR_UPDATE = 'base.collaborator.update', + + // Base/Node share lifecycle (covers both node-scoped and base-wide shares; + // payload.type distinguishes 'node' | 'base'). + BASE_SHARE_CREATE = 'base.share.create', + BASE_SHARE_UPDATE = 'base.share.update', + BASE_SHARE_DELETE = 'base.share.delete', + BASE_SHARE_REFRESH = 'base.share.refresh', + BASE_FOLDER_CREATE = 'base.folder.create', BASE_FOLDER_DELETE = 'base.folder.delete', BASE_FOLDER_UPDATE = 'base.folder.update', @@ -98,6 +113,7 @@ export enum Events { // completion point — past `lastChunk`, error-file write, and presence cleanup — // instead of racing against per-chunk audit emits. TABLE_IMPORT_FINISH = 'table.import.finish', + V2_TABLE_IMPORT_FINISH = 'v2.table.import.finish', // Composite-operation terminal signals. Currently only e2e tests subscribe; reserved // for future use by notifications / webhooks / billing. Each fires after the synchronous diff --git a/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts index 8030d2b54d..36aa9569ef 100644 --- a/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts +++ b/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts @@ -9,6 +9,7 @@ import { getV2CreateTableLegacyEventsFlag } from '../../features/v2/v2-create-ta import { ShareDbService } from '../../share-db/share-db.service'; import type { IClsStore } from '../../types/cls'; import type { + IChangeRecord, RecordCreateEvent, RecordDeleteEvent, RecordUpdateEvent, @@ -19,6 +20,17 @@ import type { } from '../events'; import { Events } from '../events'; +const collectChangedRecordFieldIds = (record: IChangeRecord | IChangeRecord[]): string[] => { + const records = Array.isArray(record) ? record : [record]; + const fieldIds = new Set(); + for (const changeRecord of records) { + for (const fieldId of Object.keys(changeRecord?.fields ?? {})) { + fieldIds.add(fieldId); + } + } + return [...fieldIds]; +}; + type IViewEvent = ViewUpdateEvent; type IRecordEvent = RecordCreateEvent | RecordDeleteEvent | RecordUpdateEvent; type IListenerEvent = @@ -141,17 +153,21 @@ export class ActionTriggerListener { const { tableId } = event.payload; const buffer = match(event) - .returnType() - .with({ name: Events.TABLE_RECORD_CREATE }, () => ['addRecord']) - .with({ name: Events.TABLE_RECORD_UPDATE }, () => ['setRecord']) - .with({ name: Events.TABLE_RECORD_DELETE }, () => ['deleteRecord']) + .returnType() + .with({ name: Events.TABLE_RECORD_CREATE }, () => [{ actionKey: 'addRecord' as const }]) + .with({ name: Events.TABLE_RECORD_UPDATE }, (updateEvent) => [ + { + actionKey: 'setRecord' as const, + // changed cell field ids, letting field-aware listeners skip + // refreshes for irrelevant edits (same contract as the v2 emitter) + payload: { fieldIds: collectChangedRecordFieldIds(updateEvent.payload.record) }, + }, + ]) + .with({ name: Events.TABLE_RECORD_DELETE }, () => [{ actionKey: 'deleteRecord' as const }]) .otherwise(() => []); if (!isEmpty(buffer)) { - this.emitActionTrigger( - tableId, - buffer.map((actionKey) => ({ actionKey })) - ); + this.emitActionTrigger(tableId, buffer); } } diff --git a/apps/nestjs-backend/src/features/access-token/access-token.service.ts b/apps/nestjs-backend/src/features/access-token/access-token.service.ts index 2e3460af0b..70c1e24b44 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.service.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.service.ts @@ -142,6 +142,18 @@ export class AccessTokenService { @Audit({ action: Events.ACCESS_TOKEN_CREATE, resourceId: (input: { userId?: string }, ctx) => input.userId ?? ctx.cls.get('user.id')!, + userId: (input: { userId?: string }, ctx) => input.userId ?? ctx.cls.get('user.id'), + // Record the token's settings so the audit row shows what access was granted. NEVER the secret: + // the token `sign` is generated server-side and is not part of the input, so this is safe. + params: (input: CreateAccessTokenRo & { clientId?: string }) => ({ + name: input.name, + description: input.description, + scopes: input.scopes, + spaceIds: input.spaceIds, + baseIds: input.baseIds, + expiredTime: input.expiredTime, + hasFullAccess: input.hasFullAccess, + }), emit: true, }) async createAccessToken( diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts index a016d80940..5c10c744cd 100644 --- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts @@ -1,7 +1,7 @@ import type { IFilter, IGroup, ISortItem, StatisticsFunc } from '@teable/core'; import type { IAggregationField, - IQueryBaseRo, + IRowCountRo, IRawAggregationValue, IRawAggregations, IRawRowCountValue, @@ -60,7 +60,7 @@ export interface IAggregationService { * @param queryRo - Query parameters for filtering * @returns Promise - The row count result */ - performRowCount(tableId: string, queryRo: IQueryBaseRo): Promise; + performRowCount(tableId: string, queryRo: IRowCountRo): Promise; /** * Get field data for a table diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts index c6f1d8ec10..02709a4727 100644 --- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts @@ -1,5 +1,4 @@ import { Injectable, Logger } from '@nestjs/common'; -import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; import { CellValueType, HttpErrorCode, @@ -17,7 +16,7 @@ import { PrismaService } from '@teable/db-main-prisma'; import { StatisticsFunc } from '@teable/openapi'; import type { IAggregationField, - IQueryBaseRo, + IRowCountRo, IRawAggregationValue, IRawAggregations, IRawRowCountValue, @@ -501,7 +500,7 @@ export class AggregationService implements IAggregationService { * @returns Promise - The row count result * @throws NotImplementedException - This method is not yet implemented */ - async performRowCount(tableId: string, queryRo: IQueryBaseRo): Promise { + async performRowCount(tableId: string, queryRo: IRowCountRo): Promise { const { viewId, ignoreViewQuery, @@ -509,6 +508,7 @@ export class AggregationService implements IAggregationService { filterLinkCellSelected, selectedRecordIds, search, + projection, } = queryRo; // Retrieve the current user's ID to build user-related query conditions const currentUserId = this.cls.get('user.id'); @@ -534,6 +534,7 @@ export class AggregationService implements IAggregationService { filterLinkCellSelected, selectedRecordIds, search, + projection, withUserId: currentUserId, viewId: queryRo?.viewId, }); @@ -560,6 +561,7 @@ export class AggregationService implements IAggregationService { filterLinkCellSelected?: IGetRecordsRo['filterLinkCellSelected']; selectedRecordIds?: IGetRecordsRo['selectedRecordIds']; search?: [string, string?, boolean?]; + projection?: string[]; withUserId?: string; viewId?: string; }) { @@ -572,6 +574,7 @@ export class AggregationService implements IAggregationService { filterLinkCellSelected, selectedRecordIds, search, + projection, withUserId, viewId, } = params; @@ -605,10 +608,17 @@ export class AggregationService implements IAggregationService { ); if (search && search[2]) { + const enabledFieldIds = wrap.enabledFieldIds; + const searchProjection = projection + ? enabledFieldIds + ? projection.filter((fieldId) => enabledFieldIds.includes(fieldId)) + : projection + : enabledFieldIds; const searchFields = await this.recordService.getSearchFields( fieldInstanceMap, search, - viewId + viewId, + searchProjection ); const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); qb.where((builder) => { @@ -1014,8 +1024,15 @@ export class AggregationService implements IAggregationService { this.logger.debug('getRecordIndexBySearchOrder sql: %s', sql); + const searchTimeout = this.thresholdConfig.searchTimeout; + try { return await this.withDataPrismaTransaction(tableId, async (prisma) => { + // Bound the search at the DB level: a short / CJK term can defeat the pg_trgm index and + // degrade to a full-table scan. SET LOCAL statement_timeout makes Postgres cancel the + // statement and release the pooled connection, instead of the client abandoning the + // request while the query keeps running and starves the connection pool. + await prisma.$executeRawUnsafe(`SET LOCAL statement_timeout = ${searchTimeout}`); const result = await prisma.$queryRawUnsafe<{ __id: string; fieldId: string }[]>(sql); // no result found @@ -1080,18 +1097,52 @@ export class AggregationService implements IAggregationService { recordId: item.__id, }; }); + // statement_timeout (above) is the real cap; give the JS-side tx timer headroom so + // Postgres cancels the statement first, instead of the data-tx default timeout firing early. }); } catch (error) { - if (error instanceof PrismaClientKnownRequestError && error.code === 'P2028') { - throw new CustomHttpException(`${error.message}`, HttpErrorCode.REQUEST_TIMEOUT, { - localization: { - i18nKey: 'httpErrors.aggregation.searchTimeOut', - }, - }); + if (this.isSearchTimeoutError(error)) { + throw new CustomHttpException( + `${(error as Error).message}`, + HttpErrorCode.REQUEST_TIMEOUT, + { + localization: { + i18nKey: 'httpErrors.aggregation.searchTimeOut', + }, + } + ); } throw error; } } + + /** + * Detects a search timeout from any of the layers involved. The search runs on the data-db + * Prisma client, whose error classes come from a *separate* generated runtime, so cross-package + * `instanceof` (PrismaClientKnownRequestError / TimeoutHttpException) is unreliable here — we + * duck-type on the error shape instead: + * - Postgres cancels the statement via `SET LOCAL statement_timeout` → SQLSTATE 57014, + * - Prisma's interactive-transaction timeout → code P2028, + * - the data-prisma proxy converts P2028 into a TimeoutHttpException → code 'request_timeout'. + * The DB-level cancellation (57014) is what actually releases the pooled connection. + */ + private isSearchTimeoutError(error: unknown): boolean { + if (typeof error !== 'object' || error === null) { + return false; + } + const err = error as { code?: unknown; meta?: unknown; message?: unknown }; + const pgErrorCode = + typeof err.meta === 'object' && err.meta !== null + ? (err.meta as { code?: unknown }).code + : undefined; + const message = typeof err.message === 'string' ? err.message : ''; + return ( + err.code === 'P2028' || + err.code === 'request_timeout' || + pgErrorCode === '57014' || + /canceling statement due to statement timeout|Transaction already closed/i.test(message) + ); + } async getRecordIndex(tableId: string, queryRo: IRecordIndexRo): Promise { const { recordId } = queryRo; diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts index 7a152b2a86..86beba965b 100644 --- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts @@ -18,10 +18,10 @@ import { groupPointsRoSchema, IAggregationRo, IGroupPointsRo, - IQueryBaseRo, + IRowCountRo, searchCountRoSchema, ISearchCountRo, - queryBaseSchema, + rowCountRoSchema, ICalendarDailyCollectionRo, ISearchIndexByQueryRo, searchIndexByQueryRoSchema, @@ -115,7 +115,7 @@ export class AggregationOpenApiController { @Permissions('table|read') async getRowCount( @Param('tableId') tableId: string, - @Query(new ZodValidationPipe(queryBaseSchema), TqlPipe) query?: IQueryBaseRo + @Query(new ZodValidationPipe(rowCountRoSchema), TqlPipe) query?: IRowCountRo ): Promise { return await this.getAggregationWithCache('row_count', tableId, query, () => this.aggregationOpenApiService.getRowCount(tableId, query) diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts index aa28ad35a8..24145164f1 100644 --- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts +++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts @@ -9,7 +9,7 @@ import type { ICalendarDailyCollectionVo, IGroupPointsRo, IGroupPointsVo, - IQueryBaseRo, + IRowCountRo, IRowCountVo, ISearchCountRo, IRecordIndexRo, @@ -69,7 +69,7 @@ export class AggregationOpenApiService { return { aggregations: result?.aggregations }; } - async getRowCount(tableId: string, query: IQueryBaseRo = {}): Promise { + async getRowCount(tableId: string, query: IRowCountRo = {}): Promise { const result = await this.aggregationService.performRowCount(tableId, query); return { rowCount: result.rowCount, diff --git a/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts index 8ade9c4780..14743f3633 100644 --- a/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts +++ b/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts @@ -2,6 +2,11 @@ import { SetMetadata } from '@nestjs/common'; import type { Action } from '@teable/core'; export const PERMISSIONS_KEY = 'permissions'; +export const ANY_PERMISSIONS_KEY = 'anyPermissions'; // eslint-disable-next-line @typescript-eslint/naming-convention export const Permissions = (...permissions: Action[]) => SetMetadata(PERMISSIONS_KEY, permissions); + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const AnyPermissions = (...permissionGroups: Action[][]) => + SetMetadata(ANY_PERMISSIONS_KEY, permissionGroups); diff --git a/apps/nestjs-backend/src/features/auth/guard/permission.guard.spec.ts b/apps/nestjs-backend/src/features/auth/guard/permission.guard.spec.ts new file mode 100644 index 0000000000..9c13202fb8 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/guard/permission.guard.spec.ts @@ -0,0 +1,125 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { ExecutionContext } from '@nestjs/common'; +import type { Reflector } from '@nestjs/core'; +import { HttpErrorCode, type Action } from '@teable/core'; +import type { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import type { PermissionService } from '../permission.service'; +import { PermissionGuard } from './permission.guard'; + +vi.mock('../permission.service', () => ({ + PermissionService: class PermissionService {}, +})); + +const tableId = 'tblxxxxxxxxxxxx'; +const tableUpdatePermissions: Action[] = ['table|update']; +const instanceUpdatePermissions: Action[] = ['instance|update']; + +const createContext = (): ExecutionContext => + ({ + getHandler: vi.fn(), + getClass: vi.fn(), + switchToHttp: () => ({ + getRequest: () => ({ + params: { tableId }, + headers: {}, + }), + }), + }) as unknown as ExecutionContext; + +const createForbiddenError = (permissions: Action[]) => + new CustomHttpException( + `not allowed to operate ${permissions.join(', ')} on ${tableId}`, + HttpErrorCode.RESTRICTED_RESOURCE + ); + +describe('PermissionGuard', () => { + const createGuard = ({ + primaryPermissions, + anyPermissions, + isAdmin = false, + validPermissions, + }: { + primaryPermissions: Action[]; + anyPermissions?: Action[][]; + isAdmin?: boolean; + validPermissions: PermissionService['validPermissions']; + }) => { + const reflector = { + getAllAndOverride: vi.fn((key: string) => { + if (key === 'permissions') { + return primaryPermissions; + } + if (key === 'anyPermissions') { + return anyPermissions; + } + return undefined; + }), + } as unknown as Reflector; + const cls = { + get: vi.fn((key: string) => { + if (key === 'user.id') { + return 'usrxxxxxxxxxxxx'; + } + if (key === 'user.isAdmin') { + return isAdmin; + } + return undefined; + }), + set: vi.fn(), + } as unknown as ClsService; + const permissionService = { + validPermissions, + } as unknown as PermissionService; + + return { + guard: new PermissionGuard(reflector, cls, permissionService), + cls, + permissionService, + }; + }; + + it('keeps table update as the primary permission for alternative permission routes', async () => { + const validPermissions = vi.fn().mockResolvedValue(tableUpdatePermissions); + const { guard, cls } = createGuard({ + primaryPermissions: tableUpdatePermissions, + anyPermissions: [instanceUpdatePermissions], + validPermissions, + }); + + await expect(guard.canActivate(createContext())).resolves.toBe(true); + + expect(validPermissions).toHaveBeenCalledWith(tableId, tableUpdatePermissions, undefined); + expect(cls.get).not.toHaveBeenCalledWith('user.isAdmin'); + }); + + it('allows instance admins through alternative permissions when table update is unavailable', async () => { + const validPermissions = vi + .fn() + .mockRejectedValue(createForbiddenError(tableUpdatePermissions)); + const { guard } = createGuard({ + primaryPermissions: tableUpdatePermissions, + anyPermissions: [instanceUpdatePermissions], + isAdmin: true, + validPermissions, + }); + + await expect(guard.canActivate(createContext())).resolves.toBe(true); + }); + + it('still rejects users without table update or instance update', async () => { + const validPermissions = vi + .fn() + .mockRejectedValue(createForbiddenError(tableUpdatePermissions)); + const { guard } = createGuard({ + primaryPermissions: tableUpdatePermissions, + anyPermissions: [instanceUpdatePermissions], + validPermissions, + }); + + await expect(guard.canActivate(createContext())).rejects.toThrow( + `not allowed to operate table|update on ${tableId}` + ); + }); +}); diff --git a/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts b/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts index c5f52475f8..42cb61f1ab 100644 --- a/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts +++ b/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts @@ -8,7 +8,7 @@ import { CustomHttpException } from '../../../custom.exception'; import type { IClsStore } from '../../../types/cls'; import { AllowAnonymousType, IS_ALLOW_ANONYMOUS } from '../decorators/allow-anonymous.decorator'; import { IS_DISABLED_PERMISSION } from '../decorators/disabled-permission.decorator'; -import { PERMISSIONS_KEY } from '../decorators/permissions.decorator'; +import { ANY_PERMISSIONS_KEY, PERMISSIONS_KEY } from '../decorators/permissions.decorator'; import { IS_PUBLIC_KEY } from '../decorators/public.decorator'; import type { IResourceMeta } from '../decorators/resource_meta.decorator'; import { RESOURCE_META } from '../decorators/resource_meta.decorator'; @@ -323,6 +323,10 @@ export class PermissionGuard { context.getHandler(), context.getClass(), ]); + const anyPermissions = this.reflector.getAllAndOverride( + ANY_PERMISSIONS_KEY, + [context.getHandler(), context.getClass()] + ); const resourceId = this.getResourceId(context) || this.defaultResourceId(context); const accessTokenId = this.cls.get('accessTokenId'); if (accessTokenId && !permissions?.length) { @@ -337,6 +341,24 @@ export class PermissionGuard { if (!permissions?.length) { return true; } + if (anyPermissions?.length) { + try { + return await this.checkPermissions(resourceId, permissions); + } catch (error) { + for (const permissionGroup of anyPermissions) { + try { + return await this.checkPermissions(resourceId, permissionGroup); + } catch { + // Try the next alternative and preserve the primary error if all alternatives fail. + } + } + throw error; + } + } + return await this.checkPermissions(resourceId, permissions); + } + + private async checkPermissions(resourceId: string | undefined, permissions: Action[]) { // instance permission check if (permissions?.includes('instance|update')) { return this.instancePermissionChecker('instance|update'); diff --git a/apps/nestjs-backend/src/features/base-share/base-share.service.ts b/apps/nestjs-backend/src/features/base-share/base-share.service.ts index 2d49402469..bb623580b8 100644 --- a/apps/nestjs-backend/src/features/base-share/base-share.service.ts +++ b/apps/nestjs-backend/src/features/base-share/base-share.service.ts @@ -5,9 +5,12 @@ import type { ICreateBaseShareRo, IUpdateBaseShareRo, IBaseShareVo } from '@teab import { BaseNodeResourceType } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../custom.exception'; +import { Events } from '../../event-emitter/events'; import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; import { generateBaseShareListCacheKey } from '../../performance-cache/generate-keys'; import type { IClsStore } from '../../types/cls'; +import { AuditScope } from '../audit/audit-scope'; +import { Audit } from '../audit/audit.decorator'; const baseShareNotFoundMessage = 'Base share not found'; const baseShareNotFoundKey = 'httpErrors.baseShare.notFound'; @@ -19,7 +22,8 @@ export class BaseShareService { constructor( private readonly prismaService: PrismaService, private readonly cls: ClsService, - private readonly performanceCacheService: PerformanceCacheService + private readonly performanceCacheService: PerformanceCacheService, + private readonly audit: AuditScope ) {} private async invalidateBaseShareListCache(baseId: string): Promise { @@ -75,6 +79,16 @@ export class BaseShareService { }; } + @Audit({ + action: Events.BASE_SHARE_CREATE, + resourceId: (baseId: string) => baseId, + params: (baseId: string, data: ICreateBaseShareRo) => ({ + baseId, + nodeId: data.nodeId ?? null, + type: data.nodeId ? 'node' : 'base', + }), + emit: (result: IBaseShareVo) => ({ shareId: result.shareId, enabled: result.enabled }), + }) async createBaseShare(baseId: string, data: ICreateBaseShareRo): Promise { const nodeId = data.nodeId ?? null; @@ -154,6 +168,22 @@ export class BaseShareService { return this.formatBaseShareVo(share); } + @Audit({ + action: Events.BASE_SHARE_UPDATE, + resourceId: (_baseId: string, shareId: string) => shareId, + params: (baseId: string, shareId: string, data: IUpdateBaseShareRo) => ({ + baseId, + shareId, + // First-pass: store raw request body so any field change is preserved. + // Password value itself is never logged — controller-layer secrets stay opaque. + changes: { ...data, password: data.password !== undefined ? '[set]' : undefined }, + }), + emit: (result: IBaseShareVo) => ({ + nodeId: result.nodeId, + type: result.nodeId ? 'node' : 'base', + enabled: result.enabled, + }), + }) async updateBaseShare( baseId: string, shareId: string, @@ -199,6 +229,12 @@ export class BaseShareService { return this.formatBaseShareVo(updated); } + @Audit({ + action: Events.BASE_SHARE_DELETE, + resourceId: (_baseId: string, shareId: string) => shareId, + params: (baseId: string, shareId: string) => ({ baseId, shareId }), + emit: true, + }) async deleteBaseShare(baseId: string, shareId: string): Promise { const share = await this.prismaService.baseShare.findFirst({ where: { baseId, shareId, enabled: true }, @@ -222,6 +258,12 @@ export class BaseShareService { await this.invalidateBaseShareListCache(baseId); } + @Audit({ + action: Events.BASE_SHARE_REFRESH, + resourceId: (_baseId: string, shareId: string) => shareId, + params: (baseId: string, shareId: string) => ({ baseId, oldShareId: shareId }), + emit: (result: IBaseShareVo) => ({ newShareId: result.shareId }), + }) async refreshBaseShareId(baseId: string, shareId: string): Promise { const share = await this.prismaService.baseShare.findFirst({ where: { baseId, shareId, enabled: true }, diff --git a/apps/nestjs-backend/src/features/base-sql-executor/allowed-functions.ts b/apps/nestjs-backend/src/features/base-sql-executor/allowed-functions.ts new file mode 100644 index 0000000000..c965202d92 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-sql-executor/allowed-functions.ts @@ -0,0 +1,144 @@ +/** + * Whitelist of PostgreSQL functions allowed in user-facing sql-query API. + * Any function NOT in this set will be rejected at the AST validation layer. + * + * Criteria for inclusion: + * - IMMUTABLE or STABLE, or VOLATILE but side-effect-free (e.g. now(), timezone()) + * - Cannot execute embedded SQL (excludes query_to_xml, ts_stat, ts_rewrite, etc.) + * - Cannot modify database state (excludes setval, lo_create, set_config, etc.) + * - Cannot acquire locks or send notifications (excludes pg_advisory_lock, pg_notify, etc.) + * - Cannot generate unbounded rows from nothing (excludes generate_series) + */ +export const allowedFunctions = new Set([ + // ── Aggregation ── + 'avg', + 'bool_and', + 'bool_or', + 'count', + 'every', + 'json_agg', + 'jsonb_agg', + 'max', + 'min', + 'string_agg', + 'sum', + 'array_agg', + + // ── Window ── + 'cume_dist', + 'dense_rank', + 'first_value', + 'lag', + 'last_value', + 'lead', + 'nth_value', + 'ntile', + 'percent_rank', + 'rank', + 'row_number', + + // ── Math ── + 'abs', + 'ceil', + 'ceiling', + 'div', + 'floor', + 'greatest', + 'least', + 'mod', + 'power', + 'pow', + 'round', + 'sign', + 'sqrt', + 'trunc', + + // ── String ── + 'ascii', + 'char_length', + 'chr', + 'concat', + 'concat_ws', + 'initcap', + 'left', + 'length', + 'lower', + 'lpad', + 'ltrim', + 'md5', + 'position', + 'regexp_match', + 'regexp_matches', + 'regexp_replace', + 'repeat', + 'replace', + 'reverse', + 'right', + 'rpad', + 'rtrim', + 'split_part', + 'starts_with', + 'strpos', + 'substr', + 'substring', + 'translate', + 'trim', + 'upper', + + // ── Date / Time ── + 'age', + 'date_part', + 'date_trunc', + 'extract', + 'make_date', + 'make_timestamp', + 'now', + 'to_char', + 'to_date', + 'to_number', + 'to_timestamp', + 'timezone', + + // ── JSON / JSONB ── + 'json_array_elements', + 'json_array_elements_text', + 'json_array_length', + 'json_build_array', + 'json_build_object', + 'json_extract_path', + 'json_extract_path_text', + 'json_typeof', + 'jsonb_array_elements', + 'jsonb_array_elements_text', + 'jsonb_array_length', + 'jsonb_build_array', + 'jsonb_build_object', + 'jsonb_each', + 'jsonb_each_text', + 'jsonb_extract_path', + 'jsonb_extract_path_text', + 'jsonb_object_keys', + 'jsonb_pretty', + 'jsonb_set', + 'jsonb_strip_nulls', + 'jsonb_typeof', + 'to_json', + 'to_jsonb', + + // ── Array ── + 'array_append', + 'array_cat', + 'array_length', + 'array_position', + 'array_remove', + 'array_to_string', + 'string_to_array', + 'unnest', + + // ── Conditional ── + 'coalesce', + 'nullif', + + // ── Type conversion ── + 'cast', +]); diff --git a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts index d8f21106e4..39765390e9 100644 --- a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts +++ b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts @@ -5,6 +5,7 @@ import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core'; import { Prisma, PrismaService, getDatabaseUrl } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; import { CustomHttpException } from '../../custom.exception'; import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex'; @@ -21,7 +22,8 @@ export class BaseSqlExecutorService { private readonly prismaService: PrismaService, private readonly databaseRouter: DatabaseRouter, private readonly configService: ConfigService, - @InjectModel(DATA_KNEX) private readonly knex: Knex + @InjectModel(DATA_KNEX) private readonly knex: Knex, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) { this.dsn = parseDsn(this.getDatabaseUrl()); this.driver = this.dsn.driver as DriverClient; @@ -144,23 +146,27 @@ export class BaseSqlExecutorService { } } - private async setRole( + private async setLocalRole( prisma: { $executeRawUnsafe(query: string): Promise }, baseId: string ) { const roleName = this.getReadOnlyRoleName(baseId); - await prisma.$executeRawUnsafe(this.knex.raw(`SET ROLE ??`, [roleName]).toQuery()); + await prisma.$executeRawUnsafe(this.knex.raw(`SET LOCAL ROLE ??`, [roleName]).toQuery()); } - private async resetRole(prisma: { $executeRawUnsafe(query: string): Promise }) { - await prisma.$executeRawUnsafe(this.knex.raw(`RESET ROLE`).toQuery()); + private async setTransactionReadOnly(prisma: { + $executeRawUnsafe(query: string): Promise; + }) { + await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY'); } - private async readonlyExecuteSql(baseId: string, sql: string) { - return this.databaseRouter.dataPrismaTransactionForBase(baseId, async (prisma) => { - await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY'); - return await prisma.$queryRawUnsafe(sql); - }); + private async setLocalStatementTimeout(prisma: { + $executeRawUnsafe(query: string): Promise; + }) { + const timeoutMs = this.thresholdConfig.searchTimeout; + await prisma.$executeRawUnsafe( + this.knex.raw(`SET LOCAL statement_timeout = ?`, [timeoutMs]).toQuery() + ); } /** @@ -197,23 +203,7 @@ export class BaseSqlExecutorService { database: this.driver, }); // 3. read only role check table access, only pg and pg version > 14 support - try { - await this.readonlyExecuteSql(baseId, sql); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - throw new CustomHttpException( - `read only check failed: ${error?.meta?.message || error?.message}`, - HttpErrorCode.VALIDATION_ERROR, - { - localization: { - i18nKey: 'httpErrors.baseSqlExecutor.readOnlyCheckFailed', - context: { - message: error?.meta?.message || error?.message, - }, - }, - } - ); - } + // TODO: need read only db connection for better security } async executeQuerySql( @@ -228,7 +218,9 @@ export class BaseSqlExecutorService { await this.roleCheckAndCreate(baseId); return this.databaseRouter.dataPrismaTransactionForBase(baseId, async (prisma) => { try { - await this.setRole(prisma, baseId); + await this.setLocalStatementTimeout(prisma); + await this.setTransactionReadOnly(prisma); + await this.setLocalRole(prisma, baseId); return await prisma.$queryRawUnsafe(sql); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { @@ -244,10 +236,6 @@ export class BaseSqlExecutorService { }, } ); - } finally { - await this.resetRole(prisma).catch((error) => { - console.log('resetRole error', error); - }); } }); } diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts index dc37594fc4..e618634237 100644 --- a/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts +++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts @@ -13,6 +13,14 @@ describe('base sql executor utils', () => { expect(() => validateRoleOperations('set role xxx;')).toThrow(); }); + it('should throw an error if the sql contains set local role', () => { + expect(() => validateRoleOperations('set local role xxx')).toThrow(); + }); + + it('should throw an error if the sql contains set session role', () => { + expect(() => validateRoleOperations('set session role xxx')).toThrow(); + }); + it('should throw an error if the sql contains set role with line break', () => { expect(() => validateRoleOperations(`set @@ -136,5 +144,74 @@ describe('base sql executor utils', () => { }) ).not.toThrow(); }); + + it.each([ + 'SELECT count(*) FROM "bseXXX"."tblYYY"', + 'SELECT sum("amount") FROM "bseXXX"."tblYYY"', + 'SELECT abs("amount") FROM "bseXXX"."tblYYY"', + 'SELECT round("amount", 2) FROM "bseXXX"."tblYYY"', + 'SELECT floor("amount") FROM "bseXXX"."tblYYY"', + 'SELECT ceil("amount") FROM "bseXXX"."tblYYY"', + 'SELECT ceiling("amount") FROM "bseXXX"."tblYYY"', + 'SELECT lower("name") FROM "bseXXX"."tblYYY"', + 'SELECT replace("name", \'a\', \'b\') FROM "bseXXX"."tblYYY"', + 'SELECT regexp_replace("name", \'a\', \'b\') FROM "bseXXX"."tblYYY"', + 'SELECT substring("name" from 1 for 3) FROM "bseXXX"."tblYYY"', + 'SELECT substr("name", 1, 3) FROM "bseXXX"."tblYYY"', + 'SELECT concat("first_name", "last_name") FROM "bseXXX"."tblYYY"', + 'SELECT concat_ws(\' \', "first_name", "last_name") FROM "bseXXX"."tblYYY"', + 'SELECT split_part("name", \'/\', 1) FROM "bseXXX"."tblYYY"', + 'SELECT coalesce("name", \'unknown\') FROM "bseXXX"."tblYYY"', + 'SELECT to_char("created_time", \'YYYY-MM-DD\') FROM "bseXXX"."tblYYY"', + 'SELECT extract(year from "created_time") FROM "bseXXX"."tblYYY"', + 'SELECT json_extract_path_text("payload"::json, \'name\') FROM "bseXXX"."tblYYY"', + ])('allows explicitly permitted function in %s', (sql) => { + expect(() => + checkTableAccess(sql, { + tableNames: ['bseXXX.tblYYY'], + database: DriverClient.Pg, + }) + ).not.toThrow(); + }); + + it.each([ + [ + "SELECT query_to_xml('SELECT rolname FROM pg_catalog.pg_roles', true, false, '')", + 'query_to_xml', + ], + [ + "SELECT query_to_xmlschema('SELECT relname FROM pg_catalog.pg_class', true, false, '')", + 'query_to_xmlschema', + ], + [ + "SELECT query_to_xml_and_xmlschema('SELECT table_name FROM information_schema.tables', true, false, '')", + 'query_to_xml_and_xmlschema', + ], + ["SELECT ts_stat('SELECT to_tsvector(''simple'', ''abc'')')", 'ts_stat'], + ["SELECT set_config('statement_timeout','0',true)", 'set_config'], + ['SELECT lo_create(0)', 'lo_create'], + ["SELECT lo_from_bytea(0, decode('414243','hex'))", 'lo_from_bytea'], + ['SELECT pg_try_advisory_lock(123456789)', 'pg_try_advisory_lock'], + ['SELECT pg_advisory_unlock_all()', 'pg_advisory_unlock_all'], + [ + "WITH n AS (SELECT pg_notify('cuppy_probe', 'blackbox')) SELECT 'ok' AS result FROM n", + 'pg_notify', + ], + ["SELECT pg_logical_emit_message(false, 'cuppy_probe', 'hello')", 'pg_logical_emit_message'], + ['SELECT pg_export_snapshot()', 'pg_export_snapshot'], + ['SELECT pg_lock_status()', 'pg_lock_status'], + ['SELECT pg_control_system()', 'pg_control_system'], + ['SELECT pg_current_wal_lsn()', 'pg_current_wal_lsn'], + ['SELECT pg_database_size(current_database())', 'pg_database_size'], + ['SELECT version()', 'version'], + ["SELECT pg_catalog.pg_notify('cuppy_probe', 'blackbox')", 'pg_notify'], + ])('blocks unapproved function %s', (sql, functionName) => { + expect(() => + checkTableAccess(sql, { + tableNames: [], + database: DriverClient.Pg, + }) + ).toThrow(new RegExp(`function ${functionName}`)); + }); }); }); diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts index a56e1922e1..0474f82dc9 100644 --- a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts +++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts @@ -2,6 +2,9 @@ import { DriverClient, HttpErrorCode } from '@teable/core'; import type { AST } from 'node-sql-parser'; import { Parser } from 'node-sql-parser'; import { CustomHttpException } from '../../custom.exception'; +import { allowedFunctions } from './allowed-functions'; + +const whiteListCheckErrorKey = 'httpErrors.baseSqlExecutor.whiteListCheckError'; export const validateRoleOperations = (sql: string) => { const removeQuotedContent = (sql: string) => { @@ -11,7 +14,11 @@ export const validateRoleOperations = (sql: string) => { const normalizedSql = sql.toLowerCase().replace(/\s+/g, ' '); const sqlWithoutQuotes = removeQuotedContent(normalizedSql); - const roleOperationPatterns = [/set\s+role/, /reset\s+role/, /set\s+session/]; + const roleOperationPatterns = [ + /set\s+(?:local\s+|session\s+)?role/, + /reset\s+role/, + /set\s+session/, + ]; for (const pattern of roleOperationPatterns) { if (pattern.test(sqlWithoutQuotes)) { @@ -35,6 +42,79 @@ const databaseTypeMap = { [DriverClient.Pg]: 'postgresql', }; +const getFunctionName = (node: unknown): string | null => { + const functionNode = node as { + type?: unknown; + name?: { name?: Array<{ value?: unknown }> }; + }; + if (functionNode.type !== 'function') { + return null; + } + + const nameParts = functionNode.name?.name; + const lastNamePart = nameParts?.[nameParts.length - 1]?.value; + return typeof lastNamePart === 'string' ? lastNamePart.toLowerCase() : null; +}; + +const findUnallowedFunctionInArray = (values: unknown[]): string | null => { + for (const value of values) { + const unallowed = findUnallowedFunction(value); + if (unallowed) { + return unallowed; + } + } + + return null; +}; + +const findUnallowedFunctionInValue = (value: unknown): string | null => { + if (Array.isArray(value)) { + return findUnallowedFunctionInArray(value); + } + + return findUnallowedFunction(value); +}; + +function findUnallowedFunction(node: unknown): string | null { + if (!node || typeof node !== 'object') { + return null; + } + + const functionName = getFunctionName(node); + if (functionName && !allowedFunctions.has(functionName)) { + return functionName; + } + + for (const value of Object.values(node)) { + const unallowed = findUnallowedFunctionInValue(value); + if (unallowed) { + return unallowed; + } + } + + return null; +} + +const validateFunctionCalls = (ast: AST | AST[]) => { + const unallowed = findUnallowedFunction(ast); + if (!unallowed) { + return; + } + + throw new CustomHttpException( + `not allowed to execute sql with function ${unallowed}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: whiteListCheckErrorKey, + context: { + function: unallowed, + }, + }, + } + ); +}; + const collectWithNames = (ast?: AST) => { if (!ast) { return []; @@ -79,6 +159,7 @@ export const checkTableAccess = ( ); } })(); + validateFunctionCalls(ast); const withNames = Array.isArray(ast) ? ast.flatMap(collectWithNames) : collectWithNames(ast); const allowedTables = new Set([...withNames, ...tableNames]); const whiteColumnList = Array.from(allowedTables).map((table) => { @@ -95,6 +176,18 @@ export const checkTableAccess = ( } const sqlTableList = parser.tableList(sql, opt); + + if (!sqlTableList.length) { + throw new CustomHttpException( + 'SQL syntax error or no table accessed, please check your query', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseSqlExecutor.sqlSyntaxError', + }, + } + ); + } const invalidTableNames = sqlTableList .filter((t: string) => !whiteColumnList.includes(t)) .map((t: string) => t.split('::').pop()!); @@ -111,7 +204,7 @@ export const checkTableAccess = ( HttpErrorCode.VALIDATION_ERROR, { localization: { - i18nKey: 'httpErrors.baseSqlExecutor.whiteListCheckError', + i18nKey: whiteListCheckErrorKey, context: { message, }, diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts index fc0f545ebb..cbe60640fd 100644 --- a/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts @@ -2,11 +2,10 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; import { FieldType, Relationship } from '@teable/core'; import { BaseDuplicateMode } from '@teable/openapi'; -import { v2RecordRepositoryPostgresTokens } from '@teable/v2-adapter-table-repository-postgres'; -import { TableByIdSpec, v2CoreTokens } from '@teable/v2-core'; +import { v2CoreTokens, type DuplicateBaseRecordReadOptions } from '@teable/v2-core'; import { GlobalModule } from '../../global/global.module'; import { BaseDuplicateService } from './base-duplicate.service'; -import type { BaseImportProgressCallback, IBaseImportProgress } from './base-import.service'; +import type { IBaseImportProgress } from './base-import.service'; import { BaseModule } from './base.module'; import type { ILinkFieldTableMap } from './utils'; @@ -150,7 +149,6 @@ describe('BaseDuplicateService normalizeDuplicateStructureForV2', () => { }); describe('BaseDuplicateService duplicateBaseV2', () => { - const okResult = (value: T) => ({ isErr: () => false, value }); type IServiceArgs = ConstructorParameters; const duplicateBaseName = 'Duplicated base'; const sourceTableName = 'Source table'; @@ -160,15 +158,19 @@ describe('BaseDuplicateService duplicateBaseV2', () => { buildDuplicateStructureConfig: (...args: unknown[]) => Promise; getCrossBaseLinkFieldTableMap: (...args: unknown[]) => Promise; getDisconnectedLinkFieldTableMap: (...args: unknown[]) => Promise; - getDisconnectedLinkFieldIds: (...args: unknown[]) => Promise; + getV2CrossBaseLinkFieldTableMap: (...args: unknown[]) => Promise; + getV2DisconnectedLinkFieldTableMap: (...args: unknown[]) => Promise; + getV2InternalLinkRelationTableMap: (...args: unknown[]) => Promise>; normalizeDuplicateStructureForV2: (structure: unknown) => unknown; createDuplicateBaseSource: (...args: unknown[]) => { - records(tableId: string): AsyncIterable<{ fields: Record }>; + records( + tableId: string, + options?: DuplicateBaseRecordReadOptions + ): AsyncIterable<{ fields: Record }>; }; duplicateTableData: (...args: unknown[]) => Promise; duplicateAttachments: (...args: unknown[]) => Promise; duplicateLinkJunction: (...args: unknown[]) => Promise; - backfillDuplicatedBaseComputedFields: (...args: unknown[]) => Promise; }; it('should create the v2 execution context from the space data container', async () => { @@ -231,9 +233,9 @@ describe('BaseDuplicateService duplicateBaseV2', () => { {} as IServiceArgs[6], { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7], {} as IServiceArgs[8], - {} as IServiceArgs[9], - v2ContainerService as unknown as IServiceArgs[10], - v2ContextFactory as unknown as IServiceArgs[11] + v2ContainerService as unknown as IServiceArgs[9], + v2ContextFactory as unknown as IServiceArgs[10], + {} as IServiceArgs[11] ); const internals = service as unknown as IDuplicateServiceInternals; @@ -243,9 +245,9 @@ describe('BaseDuplicateService duplicateBaseV2', () => { structure, sourceDbTableNameByTableId, }); - vi.spyOn(internals, 'getCrossBaseLinkFieldTableMap').mockResolvedValue({}); - vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue({}); - vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]); + vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({}); vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure); vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source); @@ -262,12 +264,13 @@ describe('BaseDuplicateService duplicateBaseV2', () => { 'bseSource', structure, {}, - sourceDbTableNameByTableId + sourceDbTableNameByTableId, + {} ); expect(commandBus.execute).toHaveBeenCalledWith(context, expect.any(Object)); }); - it('should create v2 structure first and copy records with raw table duplication', async () => { + it('should copy records through the v2 duplicate command stream', async () => { const spaceId = 'spcTarget'; const targetBaseId = 'bseTarget'; const tableIdMap = { tblSource: 'tblTarget' }; @@ -296,7 +299,7 @@ describe('BaseDuplicateService duplicateBaseV2', () => { tableIdMap, fieldIdMap, viewIdMap, - recordsLength: 0, + recordsLength: 12, }; })(), }), @@ -335,10 +338,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => { {} as IServiceArgs[5], persistedComputedBackfillService as unknown as IServiceArgs[6], { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7], - {} as IServiceArgs[8], - dataDbClientManager as unknown as IServiceArgs[9], - v2ContainerService as unknown as IServiceArgs[10], - v2ContextFactory as unknown as IServiceArgs[11] + dataDbClientManager as unknown as IServiceArgs[8], + v2ContainerService as unknown as IServiceArgs[9], + v2ContextFactory as unknown as IServiceArgs[10], + {} as IServiceArgs[11] ); const internals = service as unknown as IDuplicateServiceInternals; const mergedLinkFieldTableMap = { @@ -349,17 +352,16 @@ describe('BaseDuplicateService duplicateBaseV2', () => { structure, sourceDbTableNameByTableId: { tblSource: sourceDbTableName }, }); - vi.spyOn(internals, 'getCrossBaseLinkFieldTableMap').mockResolvedValue({}); - vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue( + vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue( mergedLinkFieldTableMap ); - vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue(['fldDisconnected']); + vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({}); vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure); vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source); vi.spyOn(internals, 'duplicateTableData').mockResolvedValue(12); vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined); vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined); - vi.spyOn(internals, 'backfillDuplicatedBaseComputedFields').mockResolvedValue(undefined); const result = await service.duplicateBaseV2({ fromBaseId: 'bseSource', @@ -368,88 +370,23 @@ describe('BaseDuplicateService duplicateBaseV2', () => { withRecords: true, }); - const executedCommand = commandBus.execute.mock.calls[0]?.[1] as { withRecords: boolean }; - expect(executedCommand.withRecords).toBe(false); - expect(internals.duplicateTableData).toHaveBeenCalledWith( - targetBaseId, - tableIdMap, - fieldIdMap, - viewIdMap, - mergedLinkFieldTableMap, - undefined - ); - expect(internals.duplicateLinkJunction).toHaveBeenCalledWith( + const executedCommand = commandBus.execute.mock.calls[0]?.[1] as { + withRecords: boolean; + batchSize: number; + }; + expect(executedCommand.withRecords).toBe(true); + expect(executedCommand.batchSize).toBe(500); + expect(internals.duplicateTableData).not.toHaveBeenCalled(); + expect(internals.duplicateLinkJunction).not.toHaveBeenCalled(); + expect(persistedComputedBackfillService.recomputeForTables).not.toHaveBeenCalled(); + expect(internals.duplicateAttachments).toHaveBeenCalledWith( targetBaseId, tableIdMap, - fieldIdMap, - true, - ['fldDisconnected'] - ); - expect(persistedComputedBackfillService.recomputeForTables).toHaveBeenCalledWith(['tblTarget']); - expect(internals.backfillDuplicatedBaseComputedFields).toHaveBeenCalledWith( - container, - context, - ['tblTarget'] + fieldIdMap ); expect(result.recordsLength).toBe(12); }); - it('should synchronously backfill v2 computed and link fields after raw record copy', async () => { - const targetTableId = 'tblaaaaaaaaaaaaaaaa'; - const context = { requestId: 'ctx' }; - const fields = [{ name: 'Owner' }]; - const table = { - getFields: vi.fn().mockReturnValue(fields), - }; - const tableRepository = { - findOne: vi.fn().mockResolvedValue(okResult(table)), - }; - const backfillService = { - executeSyncMany: vi.fn().mockResolvedValue(okResult(undefined)), - }; - const container = { - resolve: vi.fn((token: symbol) => { - if (token === v2CoreTokens.tableRepository) { - return tableRepository; - } - if (token === v2RecordRepositoryPostgresTokens.computedFieldBackfillService) { - return backfillService; - } - throw new Error('Unexpected token'); - }), - }; - - const service = new BaseDuplicateService( - {} as IServiceArgs[0], - {} as IServiceArgs[1], - {} as IServiceArgs[2], - {} as IServiceArgs[3], - {} as IServiceArgs[4], - {} as IServiceArgs[5], - {} as IServiceArgs[6], - {} as IServiceArgs[7], - {} as IServiceArgs[8], - {} as IServiceArgs[9], - {} as IServiceArgs[10], - {} as IServiceArgs[11] - ); - const internals = service as unknown as IDuplicateServiceInternals; - - await internals.backfillDuplicatedBaseComputedFields(container, context, [targetTableId]); - - expect(container.resolve).toHaveBeenCalledWith(v2CoreTokens.tableRepository); - expect(container.resolve).toHaveBeenCalledWith( - v2RecordRepositoryPostgresTokens.computedFieldBackfillService - ); - expect(tableRepository.findOne).toHaveBeenCalledWith(context, expect.any(TableByIdSpec)); - expect(backfillService.executeSyncMany).toHaveBeenCalledWith(context, { - table, - fields, - skipDistinctFilter: true, - includeOneManyTwoWay: true, - }); - }); - it('should forward real row totals through duplicateBaseV2 progress events', async () => { const spaceId = 'spcTarget'; const targetBaseId = 'bseTarget'; @@ -473,13 +410,35 @@ describe('BaseDuplicateService duplicateBaseV2', () => { execute: vi.fn().mockResolvedValue({ isErr: () => false, value: (async function* () { + yield { + id: 'progress', + phase: 'table_data_start', + processedRows: 0, + totalRows: 12, + }; + yield { + id: 'progress', + phase: 'table_data_progress', + tableId: 'tblTarget', + tableName: sourceTableName, + processedRows: 5, + batchProcessedRows: 5, + currentBatch: 1, + totalRows: 12, + }; + yield { + id: 'progress', + phase: 'table_data_done', + processedRows: 12, + totalRows: 12, + }; yield { id: 'done', baseId: targetBaseId, tableIdMap, fieldIdMap, viewIdMap, - recordsLength: 0, + recordsLength: 12, }; })(), }), @@ -517,10 +476,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => { {} as IServiceArgs[5], persistedComputedBackfillService as unknown as IServiceArgs[6], { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7], - {} as IServiceArgs[8], - dataDbClientManager as unknown as IServiceArgs[9], - v2ContainerService as unknown as IServiceArgs[10], - v2ContextFactory as unknown as IServiceArgs[11] + dataDbClientManager as unknown as IServiceArgs[8], + v2ContainerService as unknown as IServiceArgs[9], + v2ContextFactory as unknown as IServiceArgs[10], + {} as IServiceArgs[11] ); const internals = service as unknown as IDuplicateServiceInternals; const progressEvents: unknown[] = []; @@ -529,29 +488,14 @@ describe('BaseDuplicateService duplicateBaseV2', () => { structure, sourceDbTableNameByTableId: { tblSource: sourceDbTableName }, }); - vi.spyOn(internals, 'getCrossBaseLinkFieldTableMap').mockResolvedValue({}); - vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue({}); - vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]); + vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({}); vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure); vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source); - vi.spyOn(internals, 'duplicateTableData').mockImplementation(async (...args: unknown[]) => { - const onProgress = args[5] as BaseImportProgressCallback | undefined; - onProgress?.({ phase: 'table_data_start', processedRows: 0, totalRows: 12 }); - onProgress?.({ - phase: 'table_data_progress', - tableId: 'tblTarget', - tableName: sourceTableName, - processedRows: 5, - batchProcessedRows: 5, - currentBatch: 1, - totalRows: 12, - }); - onProgress?.({ phase: 'table_data_done', processedRows: 12, totalRows: 12 }); - return 12; - }); + vi.spyOn(internals, 'duplicateTableData').mockResolvedValue(12); vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined); vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined); - vi.spyOn(internals, 'backfillDuplicatedBaseComputedFields').mockResolvedValue(undefined); const result = await service.duplicateBaseV2( { @@ -565,14 +509,7 @@ describe('BaseDuplicateService duplicateBaseV2', () => { (event: string | IBaseImportProgress) => progressEvents.push(event) ); - expect(internals.duplicateTableData).toHaveBeenCalledWith( - targetBaseId, - tableIdMap, - fieldIdMap, - viewIdMap, - {}, - expect.any(Function) - ); + expect(internals.duplicateTableData).not.toHaveBeenCalled(); expect(progressEvents).toEqual( expect.arrayContaining([ expect.objectContaining({ phase: 'table_data_start', processedRows: 0, totalRows: 12 }), @@ -661,10 +598,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => { knex as unknown as IServiceArgs[5], {} as IServiceArgs[6], {} as IServiceArgs[7], - {} as IServiceArgs[8], { dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma), - } as unknown as IServiceArgs[9], + } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], {} as IServiceArgs[10], {} as IServiceArgs[11] ); @@ -724,10 +661,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => { {} as IServiceArgs[5], {} as IServiceArgs[6], {} as IServiceArgs[7], - {} as IServiceArgs[8], { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex), - } as unknown as IServiceArgs[9], + } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], {} as IServiceArgs[10], {} as IServiceArgs[11] ); @@ -768,7 +705,503 @@ describe('BaseDuplicateService duplicateBaseV2', () => { recordId: 'recSource', fields: { fldText: 'A' }, autoNumber: 1, + lastModifiedTime: null, + lastModifiedBy: null, }, ]); }); + + it('should skip internal v2 link relation reads during insert phase', async () => { + const sourceTableQuery = { + select: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockReturnThis(), + limit: vi + .fn() + .mockResolvedValueOnce([ + { + __id: 'recSource', + __auto_number: 1, + fldStory: [{ id: 'recStale', title: 'Deleted story' }], + }, + ]) + .mockResolvedValueOnce([]), + }; + const dataKnex = vi.fn((tableName: string) => { + if (tableName === 'bseSource.tblSource') return sourceTableQuery; + throw new Error(`unexpected table ${tableName}`); + }); + const service = new BaseDuplicateService( + {} as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const source = internals.createDuplicateBaseSource( + 'bseSource', + { + name: duplicateBaseName, + tables: [ + { + id: 'tblSource', + name: sourceTableName, + dbTableName: 'tblShortName', + fields: [ + { + id: 'fldStory', + name: 'Story', + dbFieldName: 'fldStory', + type: FieldType.Link, + }, + ], + views: [], + }, + ], + }, + {}, + { tblSource: 'bseSource.tblSource' }, + { + tblSource: [ + { + fieldId: 'fldStory', + dbFieldName: 'fldStory', + foreignTableId: 'tblStory', + lookupFieldId: 'fldStoryName', + relationship: 'manyMany', + fkHostTableName: 'bseSource.junction_fldStory', + selfKeyName: '__fk_self', + foreignKeyName: '__fk_foreign', + isOneWay: false, + isMultipleCellValue: true, + orderColumnName: '__order', + }, + ], + } + ); + + const records = []; + for await (const record of source.records('tblSource', { phase: 'insert' })) { + records.push(record); + } + + expect(dataKnex).toHaveBeenCalledWith('bseSource.tblSource'); + expect(dataKnex).not.toHaveBeenCalledWith('bseSource.junction_fldStory'); + expect(records[0].fields.fldStory).toEqual([{ id: 'recStale', title: 'Deleted story' }]); + }); + + it('should rebuild internal v2 link values from relation storage and ignore stale cache ids', async () => { + const sourceTableQuery = { + select: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockReturnThis(), + limit: vi + .fn() + .mockResolvedValueOnce([ + { + __id: 'recSource', + __auto_number: 1, + fldStory: [ + { id: 'recExisting', title: 'Existing story' }, + { id: 'recStale', title: 'Deleted story' }, + ], + }, + ]) + .mockResolvedValueOnce([]), + }; + const junctionQuery = { + select: vi.fn().mockReturnThis(), + whereIn: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockResolvedValue([ + { + sourceRecordId: 'recSource', + foreignRecordId: 'recExisting', + }, + ]), + }; + const dataKnex = vi.fn((tableName: string) => { + if (tableName === 'bseSource.tblSource') return sourceTableQuery; + if (tableName === 'bseSource.junction_fldStory') return junctionQuery; + throw new Error(`unexpected table ${tableName}`); + }); + const service = new BaseDuplicateService( + {} as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const source = internals.createDuplicateBaseSource( + 'bseSource', + { + name: duplicateBaseName, + tables: [ + { + id: 'tblSource', + name: sourceTableName, + dbTableName: 'tblShortName', + fields: [ + { + id: 'fldStory', + name: 'Story', + dbFieldName: 'fldStory', + type: FieldType.Link, + }, + ], + views: [], + }, + ], + }, + {}, + { tblSource: 'bseSource.tblSource' }, + { + tblSource: [ + { + fieldId: 'fldStory', + dbFieldName: 'fldStory', + foreignTableId: 'tblStory', + lookupFieldId: 'fldStoryName', + relationship: 'manyMany', + fkHostTableName: 'bseSource.junction_fldStory', + selfKeyName: '__fk_self', + foreignKeyName: '__fk_foreign', + isOneWay: false, + isMultipleCellValue: true, + orderColumnName: '__order', + }, + ], + } + ); + + const records = []; + for await (const record of source.records('tblSource', { phase: 'linkRestore' })) { + records.push(record); + } + + expect(dataKnex).toHaveBeenCalledWith('bseSource.tblSource'); + expect(dataKnex).toHaveBeenCalledWith('bseSource.junction_fldStory'); + expect(junctionQuery.whereIn).toHaveBeenCalledWith('__fk_self', ['recSource']); + expect(records[0].fields.fldStory).toEqual([{ id: 'recExisting' }]); + }); + + it('should rebuild many-one internal v2 link values from current table FK storage', async () => { + const sourceTableQuery = { + select: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockReturnThis(), + limit: vi + .fn() + .mockResolvedValueOnce([ + { + __id: 'recChild', + __auto_number: 1, + fldParent: { id: 'recStale', title: 'Deleted parent' }, + }, + ]) + .mockResolvedValueOnce([]), + whereIn: vi.fn().mockReturnThis(), + whereNotNull: vi.fn().mockResolvedValue([ + { + sourceRecordId: 'recChild', + foreignRecordId: 'recParent', + }, + ]), + }; + const dataKnex = vi.fn((tableName: string) => { + if (tableName === 'bseSource.childTable') return sourceTableQuery; + throw new Error(`unexpected table ${tableName}`); + }); + const service = new BaseDuplicateService( + {} as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const source = internals.createDuplicateBaseSource( + 'bseSource', + { + name: duplicateBaseName, + tables: [ + { + id: 'tblChild', + name: sourceTableName, + dbTableName: 'childTable', + fields: [ + { + id: 'fldParent', + name: 'Parent', + dbFieldName: 'fldParent', + type: FieldType.Link, + }, + ], + views: [], + }, + ], + }, + {}, + { tblChild: 'bseSource.childTable' }, + { + tblChild: [ + { + fieldId: 'fldParent', + dbFieldName: 'fldParent', + foreignTableId: 'tblParent', + lookupFieldId: 'fldParentName', + relationship: 'manyOne', + fkHostTableName: 'bseSource.childTable', + selfKeyName: '__id', + foreignKeyName: '__fk_parent', + isOneWay: false, + isMultipleCellValue: false, + orderColumnName: '__fk_parent_order', + }, + ], + } + ); + + const records = []; + for await (const record of source.records('tblChild', { phase: 'linkRestore' })) { + records.push(record); + } + + expect(dataKnex).toHaveBeenCalledWith('bseSource.childTable'); + expect(sourceTableQuery.whereIn).toHaveBeenCalledWith('__id', ['recChild']); + expect(sourceTableQuery.whereNotNull).toHaveBeenCalledWith('__fk_parent'); + expect(records[0].fields.fldParent).toEqual({ id: 'recParent' }); + }); + + it('should rebuild two-way one-many internal v2 link values from foreign table FK storage', async () => { + const sourceTableQuery = { + select: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockReturnThis(), + limit: vi + .fn() + .mockResolvedValueOnce([ + { + __id: 'recParent', + __auto_number: 1, + fldChildren: [ + { id: 'recChildExisting', title: 'Existing child' }, + { id: 'recChildStale', title: 'Deleted child' }, + ], + }, + ]) + .mockResolvedValueOnce([]), + }; + const foreignTableQuery = { + select: vi.fn().mockReturnThis(), + whereIn: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockResolvedValue([ + { + sourceRecordId: 'recParent', + foreignRecordId: 'recChildExisting', + }, + ]), + }; + const dataKnex = vi.fn((tableName: string) => { + if (tableName === 'bseSource.parentTable') return sourceTableQuery; + if (tableName === 'bseSource.childTable') return foreignTableQuery; + throw new Error(`unexpected table ${tableName}`); + }); + const service = new BaseDuplicateService( + {} as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const source = internals.createDuplicateBaseSource( + 'bseSource', + { + name: duplicateBaseName, + tables: [ + { + id: 'tblParent', + name: sourceTableName, + dbTableName: 'parentTable', + fields: [ + { + id: 'fldChildren', + name: 'Children', + dbFieldName: 'fldChildren', + type: FieldType.Link, + }, + ], + views: [], + }, + ], + }, + {}, + { tblParent: 'bseSource.parentTable' }, + { + tblParent: [ + { + fieldId: 'fldChildren', + dbFieldName: 'fldChildren', + foreignTableId: 'tblChild', + lookupFieldId: 'fldChildName', + relationship: 'oneMany', + fkHostTableName: 'bseSource.childTable', + selfKeyName: '__fk_parent', + foreignKeyName: '__id', + isOneWay: false, + isMultipleCellValue: true, + orderColumnName: '__fk_parent_order', + }, + ], + } + ); + + const records = []; + for await (const record of source.records('tblParent', { phase: 'linkRestore' })) { + records.push(record); + } + + expect(dataKnex).toHaveBeenCalledWith('bseSource.childTable'); + expect(foreignTableQuery.whereIn).toHaveBeenCalledWith('__fk_parent', ['recParent']); + expect(foreignTableQuery.orderBy).toHaveBeenCalledWith('__fk_parent_order', 'asc'); + expect(records[0].fields.fldChildren).toEqual([{ id: 'recChildExisting' }]); + }); + + it('should normalize postgres array literal link values when downgrading v2 cross-base links', async () => { + const query = { + select: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockReturnThis(), + limit: vi + .fn() + .mockResolvedValueOnce([ + { + __id: 'recSource', + __auto_number: 1, + fldVendor: '{"{\\"id\\":\\"recVendor\\",\\"title\\":\\"Vendor A\\"}"}', + }, + ]) + .mockResolvedValueOnce([]), + }; + const dataKnex = vi.fn().mockReturnValue(query); + const service = new BaseDuplicateService( + {} as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + { + dataKnexForBase: vi.fn().mockResolvedValue(dataKnex), + } as unknown as IServiceArgs[8], + {} as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const source = internals.createDuplicateBaseSource( + 'bseSource', + { + name: duplicateBaseName, + tables: [ + { + id: 'tblSource', + name: sourceTableName, + dbTableName: 'tblSource', + fields: [ + { + id: 'fldVendor', + name: 'Vendor', + dbFieldName: 'fldVendor', + type: FieldType.SingleLineText, + }, + ], + views: [], + }, + ], + }, + { + tblSource: [ + { + dbFieldName: 'fldVendor', + selfKeyName: 'fk_fld_vendor', + isMultipleCellValue: true, + }, + ], + }, + { tblSource: 'bseSource.tblSource' } + ); + + const records = []; + for await (const record of source.records('tblSource')) { + records.push(record); + } + + expect(records[0].fields.fldVendor).toBe('Vendor A'); + }); + + it('should include nullable isLookup host link fields when building v2 cross-base maps', async () => { + const fieldFindMany = vi.fn().mockResolvedValue([]); + const service = new BaseDuplicateService( + { + txClient: vi.fn().mockReturnValue({ + field: { findMany: fieldFindMany }, + }), + } as unknown as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + {} as IServiceArgs[8], + {} as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + + await internals.getV2CrossBaseLinkFieldTableMap({ tblSource: 'tblSource' }); + + expect(fieldFindMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + OR: [{ isLookup: false }, { isLookup: null }], + }), + }) + ); + }); }); diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts index 8a5f0202a8..f021ef8f36 100644 --- a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts +++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts @@ -10,26 +10,18 @@ import { type IDuplicateBaseRo, } from '@teable/openapi'; import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; -import { - v2RecordRepositoryPostgresTokens, - type ComputedFieldBackfillService, -} from '@teable/v2-adapter-table-repository-postgres'; import { DuplicateBaseCommand, - TableByIdSpec, - TableId, v2CoreTokens, + type DuplicateBaseRecordReadOptions, type DotTeaFieldInput, type DuplicateBaseResult, type DuplicateBaseSource, type ICommandBus, - type IExecutionContext, - type ITableRepository, type NormalizedDotTeaField, type NormalizedDotTeaStructure, } from '@teable/v2-core'; -import type { DependencyContainer } from '@teable/v2-di'; -import { normalizeField } from '@teable/v2-dottea'; +import { normalizeFields } from '@teable/v2-dottea'; import { Knex } from 'knex'; import type { Kysely } from 'kysely'; import { groupBy, omit } from 'lodash'; @@ -60,6 +52,8 @@ import type { ILinkFieldTableInfo, ILinkFieldTableMap } from './utils'; type DuplicatedBase = Awaited>['base']; const v2DuplicateReadBatchSize = 500; const v2DuplicateCopyBatchSize = 500; +const v2DuplicateLinkFieldBatchSize = 500; +const v2DuplicateTableIdQueryChunkSize = 100; type DuplicateStructureConfig = Awaited>; type DuplicateV2FieldConfig = Omit< DuplicateStructureConfig['tables'][number]['fields'][number], @@ -77,6 +71,33 @@ type DuplicateStructureConfigResult = { structure: DuplicateStructureConfig; sourceDbTableNameByTableId: Record; }; +type DuplicateBaseStructConfigInput = Parameters< + BaseExportService['generateBaseStructConfig'] +>[0] & { + includeWorkflowRuntimeState?: boolean; +}; +type DuplicateLinkFieldRaw = { + id: string; + tableId: string; + dbFieldName: string; + isMultipleCellValue: boolean | null; + meta: string | null; + options: string | null; +}; +type V2InternalLinkFieldTableInfo = { + fieldId: string; + dbFieldName: string; + foreignTableId: string; + lookupFieldId: string; + relationship: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + isOneWay: boolean; + isMultipleCellValue: boolean; + orderColumnName?: string; +}; +type V2LinkCellItem = { id: string; title?: string }; type IDataPrismaExecutor = { $executeRawUnsafe(query: string, ...values: unknown[]): Promise; @@ -269,9 +290,9 @@ export class BaseDuplicateService { // allowCrossBase=true keeps same-space cross-base links intact. Mirrors // the v1 branch in duplicateBase above. const crossBaseLinkFieldTableMap: ILinkFieldTableMap = allowCrossBase - ? await this.getCrossBaseLinkFieldTableMap(sourceTableIdMap, spaceId) - : await this.getCrossBaseLinkFieldTableMap(sourceTableIdMap); - const disconnectedLinkFieldTableMap = await this.getDisconnectedLinkFieldTableMap( + ? await this.getV2CrossBaseLinkFieldTableMap(sourceTableIdMap, spaceId) + : await this.getV2CrossBaseLinkFieldTableMap(sourceTableIdMap); + const disconnectedLinkFieldTableMap = await this.getV2DisconnectedLinkFieldTableMap( sourceTableIdMap, fromBaseId, nodes, @@ -281,12 +302,8 @@ export class BaseDuplicateService { crossBaseLinkFieldTableMap, disconnectedLinkFieldTableMap ); - const disconnectedLinkFieldIds = await this.getDisconnectedLinkFieldIds( - sourceTableIdMap, - fromBaseId, - nodes, - skipParentNodes - ); + const internalLinkRelationTableMap = + await this.getV2InternalLinkRelationTableMap(sourceTableIdMap); const container = await this.v2ContainerService.getContainerForSpace(spaceId); const commandBus = container.resolve(v2CoreTokens.commandBus); const db = container.resolve>(v2PostgresDbTokens.db); @@ -315,12 +332,14 @@ export class BaseDuplicateService { fromBaseId, normalizedStructure, mergedLinkFieldTableMap, - sourceDbTableNameByTableId + sourceDbTableNameByTableId, + internalLinkRelationTableMap ); const commandResult = DuplicateBaseCommand.createFromSource({ baseId: base.id, source, - withRecords: false, + withRecords: Boolean(withRecords), + batchSize: v2DuplicateCopyBatchSize, }); if (commandResult.isErr()) { throw new Error(commandResult.error.message); @@ -360,37 +379,15 @@ export class BaseDuplicateService { structure, { tableIdMap, fieldIdMap, viewIdMap }, duplicateMode, - onProgress, - { restoreEeResources: true } + onProgress ); if (withRecords) { - recordsLength = await this.duplicateTableData( - base.id, - tableIdMap, - fieldIdMap, - viewIdMap, - mergedLinkFieldTableMap, - onProgress - ); onProgress?.({ phase: 'attachments_copying', processedRows: recordsLength, totalRows: recordsLength, }); await this.duplicateAttachments(base.id, tableIdMap, fieldIdMap); - await this.duplicateLinkJunction( - base.id, - tableIdMap, - fieldIdMap, - allowCrossBase, - disconnectedLinkFieldIds - ); - await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap)); - await this.backfillDuplicatedBaseComputedFields( - container, - context, - Object.values(tableIdMap) - ); await prisma.base.update({ where: { id: base.id }, data: { @@ -509,23 +506,10 @@ export class BaseDuplicateService { }, }); const tableIds = tableRaws.map(({ id }) => id); - const fieldRaws = await prisma.field.findMany({ - where: { - tableId: { in: tableIds }, - deletedTime: null, - }, - }); - const viewRaws = await prisma.view.findMany({ - where: { - tableId: { in: tableIds }, - deletedTime: null, - }, - orderBy: { - order: 'asc', - }, - }); + const fieldRaws = await this.baseExportService.findFieldsByTableIds(tableIds); + const viewRaws = await this.baseExportService.findViewsByTableIds(tableIds); - const structure = await this.baseExportService.generateBaseStructConfig({ + const structureInput: DuplicateBaseStructConfigInput = { baseRaw, tableRaws, fieldRaws, @@ -539,7 +523,9 @@ export class BaseDuplicateService { excludedTableIds, rootNodeIds, destSpaceId, - }); + includeWorkflowRuntimeState: false, + }; + const structure = await this.baseExportService.generateBaseStructConfig(structureInput); return { structure, @@ -555,26 +541,38 @@ export class BaseDuplicateService { sourceBaseId: string, structure: DuplicateV2StructureConfig, disconnectedLinkFieldTableMap: ILinkFieldTableMap, - sourceDbTableNameByTableId: Record = {} + sourceDbTableNameByTableId: Record = {}, + internalLinkRelationTableMap: Record = {} ): DuplicateBaseSource { const tableById = new Map(structure.tables.map((table) => [table.id, table])); - const readRows = (sourceDbTableName: string, crossBaseLinkInfo: ILinkFieldTableInfo[]) => - this.createSourceTableRecordRows(sourceBaseId, sourceDbTableName, crossBaseLinkInfo); + const readRows = ( + sourceDbTableName: string, + crossBaseLinkInfo: ILinkFieldTableInfo[], + internalLinkInfo: V2InternalLinkFieldTableInfo[] + ) => + this.createSourceTableRecordRows( + sourceBaseId, + sourceDbTableName, + crossBaseLinkInfo, + internalLinkInfo + ); const toRecordInput = (table: DuplicateV2TableConfig, row: Record) => this.toDuplicateBaseRecordInput(table, row); return { structure, - async *records(tableId: string) { + async *records(tableId: string, options?: DuplicateBaseRecordReadOptions) { const table = tableById.get(tableId); const sourceDbTableName = sourceDbTableNameByTableId[tableId] ?? table?.dbTableName; if (!table || !sourceDbTableName) { return; } + const shouldReadInternalLinks = options?.phase !== 'insert'; for await (const row of readRows( sourceDbTableName, - disconnectedLinkFieldTableMap[tableId] || [] + disconnectedLinkFieldTableMap[tableId] || [], + shouldReadInternalLinks ? internalLinkRelationTableMap[tableId] || [] : [] )) { yield toRecordInput(table, row); } @@ -616,29 +614,13 @@ export class BaseDuplicateService { return { ...structure, tables: structure.tables.map((table) => { - const tableFieldTypesById = new Map( - table.fields - .filter((field) => field.id) - .map( - (field) => - [ - field.id!, - field.isConditionalLookup - ? 'conditionalLookup' - : field.isLookup - ? 'lookup' - : field.type, - ] as const - ) - ); - return { ...table, - fields: table.fields.map((field) => { - const normalized = normalizeField(field as DotTeaFieldInput, tableFieldTypesById, { - availableTableIds, - fieldIdsByTableId, - }); + fields: normalizeFields(table.fields as ReadonlyArray, { + availableTableIds, + fieldIdsByTableId, + }).map((normalized, index) => { + const field = table.fields[index]!; const normalizedField = { ...field, ...normalized }; if ( @@ -723,12 +705,8 @@ export class BaseDuplicateService { ...(row.__auto_number ? { autoNumber: Number(row.__auto_number) } : {}), ...(row.__created_time ? { createdTime: this.toRestoreString(row.__created_time) } : {}), ...(row.__created_by ? { createdBy: this.toRestoreString(row.__created_by) } : {}), - ...(row.__last_modified_time - ? { lastModifiedTime: this.toRestoreString(row.__last_modified_time) } - : {}), - ...(row.__last_modified_by - ? { lastModifiedBy: this.toRestoreString(row.__last_modified_by) } - : {}), + lastModifiedTime: null, + lastModifiedBy: null, }; } @@ -1013,6 +991,211 @@ export class BaseDuplicateService { return tableId2DbFieldNameMap; } + private async findV2DuplicateLinkFields(tableIds: string[]) { + const prisma = this.prismaService.txClient(); + const fields: DuplicateLinkFieldRaw[] = []; + + for (let index = 0; index < tableIds.length; index += v2DuplicateTableIdQueryChunkSize) { + const tableIdChunk = tableIds.slice(index, index + v2DuplicateTableIdQueryChunkSize); + let cursor: string | undefined; + let hasMore = true; + + while (hasMore) { + const page = await prisma.field.findMany({ + where: { + tableId: { in: tableIdChunk }, + type: FieldType.Link, + OR: [{ isLookup: false }, { isLookup: null }], + deletedTime: null, + }, + ...(cursor + ? { + cursor: { + id: cursor, + }, + skip: 1, + } + : {}), + select: { + id: true, + tableId: true, + dbFieldName: true, + isMultipleCellValue: true, + meta: true, + options: true, + }, + orderBy: [{ tableId: 'asc' }, { id: 'asc' }], + take: v2DuplicateLinkFieldBatchSize, + }); + + fields.push(...page); + + hasMore = page.length === v2DuplicateLinkFieldBatchSize; + if (hasMore) { + cursor = page[page.length - 1].id; + } + } + } + + return fields; + } + + private parseLinkFieldOptions(options: string | null): ILinkFieldOptions | undefined { + if (!options) { + return undefined; + } + + try { + return JSON.parse(options) as ILinkFieldOptions; + } catch { + return undefined; + } + } + + private resolveV2LinkOrderColumnName(options: ILinkFieldOptions): string | undefined { + switch (options.relationship) { + case 'manyMany': + return '__order'; + case 'oneMany': + return options.isOneWay ? '__order' : `${options.selfKeyName}_order`; + case 'manyOne': + case 'oneOne': + return `${options.foreignKeyName}_order`; + default: + return undefined; + } + } + + private hasV2LinkOrderColumn(meta: string | null): boolean { + if (!meta) { + return false; + } + + try { + return Boolean((JSON.parse(meta) as { hasOrderColumn?: unknown }).hasOrderColumn); + } catch { + return false; + } + } + + private buildLinkFieldTableMap( + tableIdMap: Record, + fields: DuplicateLinkFieldRaw[] + ): ILinkFieldTableMap { + const tableId2DbFieldNameMap: ILinkFieldTableMap = {}; + + Object.entries(groupBy(fields, 'tableId')).forEach(([tableId, fields]) => { + const info = fields.flatMap(({ dbFieldName, isMultipleCellValue, options }) => { + const parsedOptions = this.parseLinkFieldOptions(options); + if (!parsedOptions?.selfKeyName) { + return []; + } + return [ + { + dbFieldName, + selfKeyName: parsedOptions.selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }, + ]; + }); + + if (info.length) { + tableId2DbFieldNameMap[tableId] = info; + tableId2DbFieldNameMap[tableIdMap[tableId]] = info; + } + }); + + return tableId2DbFieldNameMap; + } + + private async getV2InternalLinkRelationTableMap( + tableIdMap: Record + ): Promise> { + const tableIds = Object.keys(tableIdMap); + const fields = (await this.findV2DuplicateLinkFields(tableIds)).filter((field) => { + const options = this.parseLinkFieldOptions(field.options); + return ( + options?.foreignTableId && + !options.baseId && + tableIdMap[options.foreignTableId] && + options.fkHostTableName && + options.selfKeyName && + options.foreignKeyName + ); + }); + + if (!fields.length) { + return {}; + } + + const tableId2Info: Record = {}; + Object.entries(groupBy(fields, 'tableId')).forEach(([tableId, tableFields]) => { + const info = tableFields.flatMap((field) => { + const options = this.parseLinkFieldOptions(field.options); + const foreignTableId = options?.foreignTableId; + const lookupFieldId = options?.lookupFieldId; + if ( + !foreignTableId || + !lookupFieldId || + !options?.relationship || + !options.fkHostTableName || + !options.selfKeyName || + !options.foreignKeyName + ) { + return []; + } + + return [ + { + fieldId: field.id, + dbFieldName: field.dbFieldName, + foreignTableId, + lookupFieldId, + relationship: options.relationship, + fkHostTableName: options.fkHostTableName, + selfKeyName: options.selfKeyName, + foreignKeyName: options.foreignKeyName, + isOneWay: Boolean(options.isOneWay), + isMultipleCellValue: !!field.isMultipleCellValue, + orderColumnName: this.hasV2LinkOrderColumn(field.meta) + ? this.resolveV2LinkOrderColumnName(options) + : undefined, + }, + ]; + }); + + if (info.length) { + tableId2Info[tableId] = info; + tableId2Info[tableIdMap[tableId]] = info; + } + }); + + return tableId2Info; + } + + private async getV2DisconnectedLinkFieldTableMap( + tableIdMap: Record, + fromBaseId: string, + nodes?: string[], + skipParentNodes: boolean = false + ): Promise { + const { excludedTableIds } = await this.collectNodesAndResourceIds( + fromBaseId, + nodes, + skipParentNodes + ); + + if (!nodes?.length || !excludedTableIds?.length) { + return {}; + } + + const fields = (await this.findV2DuplicateLinkFields(Object.keys(tableIdMap))).filter((field) => + excludedTableIds.includes(this.parseLinkFieldOptions(field.options)?.foreignTableId ?? '') + ); + + return this.buildLinkFieldTableMap(tableIdMap, fields); + } + async previewCrossSpaceAffectedFields( fromBaseId: string, destSpaceId: string @@ -1127,6 +1310,42 @@ export class BaseDuplicateService { return tableId2DbFieldNameMap; } + private async getV2CrossBaseLinkFieldTableMap( + tableIdMap: Record, + destSpaceId?: string + ): Promise { + const linkFields = (await this.findV2DuplicateLinkFields(Object.keys(tableIdMap))).filter( + (field) => this.parseLinkFieldOptions(field.options)?.baseId + ); + + let crossBaseLinkFields = linkFields; + if (destSpaceId) { + const prisma = this.prismaService.txClient(); + const foreignBaseIds = Array.from( + new Set( + linkFields + .map((field) => this.parseLinkFieldOptions(field.options)?.baseId) + .filter((baseId): baseId is string => Boolean(baseId)) + ) + ); + const bases = foreignBaseIds.length + ? await prisma.base.findMany({ + where: { id: { in: foreignBaseIds }, deletedTime: null }, + select: { id: true, spaceId: true }, + }) + : []; + const crossSpaceBaseIds = new Set( + bases.filter((base) => base.spaceId !== destSpaceId).map((base) => base.id) + ); + crossBaseLinkFields = linkFields.filter((field) => { + const baseId = this.parseLinkFieldOptions(field.options)?.baseId; + return baseId && crossSpaceBaseIds.has(baseId); + }); + } + + return this.buildLinkFieldTableMap(tableIdMap, crossBaseLinkFields); + } + private async duplicateTableData( targetBaseId: string, tableIdMap: Record, @@ -1396,10 +1615,64 @@ export class BaseDuplicateService { return String(value); } + private parsePostgresTextArrayLiteral(value: string): string[] | undefined { + if (!value.startsWith('{') || !value.endsWith('}')) { + return undefined; + } + + const inner = value.slice(1, -1); + if (!inner) { + return []; + } + + const items: string[] = []; + let item = ''; + let inQuotes = false; + + for (let index = 0; index < inner.length; index += 1) { + const char = inner[index]; + + if (inQuotes) { + if (char === '\\') { + index += 1; + item += inner[index] ?? ''; + continue; + } + if (char === '"') { + inQuotes = false; + continue; + } + item += char; + continue; + } + + if (char === '"') { + inQuotes = true; + continue; + } + + if (char === ',') { + items.push(item); + item = ''; + continue; + } + + item += char; + } + + if (inQuotes) { + return undefined; + } + + items.push(item); + return items; + } + private async *createSourceTableRecordRows( sourceBaseId: string, sourceDbTableName: string, - crossBaseLinkInfo: ILinkFieldTableInfo[] + crossBaseLinkInfo: ILinkFieldTableInfo[], + internalLinkInfo: V2InternalLinkFieldTableInfo[] = [] ): AsyncGenerator> { const dataKnex = await this.dataDbClientManager.dataKnexForBase(sourceBaseId, { useTransaction: true, @@ -1416,8 +1689,18 @@ export class BaseDuplicateService { return; } - for (const row of rows) { - yield this.normalizeCrossBaseLinkColumns(row, crossBaseLinkInfo); + const sourceRecordIds = rows.flatMap((row) => + typeof row.__id === 'string' ? [row.__id] : [] + ); + const normalizedRows = await this.normalizeInternalLinkColumns( + dataKnex, + rows.map((row) => this.normalizeCrossBaseLinkColumns(row, crossBaseLinkInfo)), + sourceRecordIds, + internalLinkInfo + ); + + for (const row of normalizedRows) { + yield row; } lastAutoNumber = Number(rows[rows.length - 1]?.__auto_number ?? lastAutoNumber); @@ -1444,6 +1727,138 @@ export class BaseDuplicateService { return nextRow; } + private async normalizeInternalLinkColumns( + dataKnex: Knex, + rows: Record[], + sourceRecordIds: string[], + internalLinkInfo: V2InternalLinkFieldTableInfo[] + ) { + if (!rows.length || !internalLinkInfo.length) { + return rows; + } + + const rowsById = new Map( + rows.flatMap((row) => (typeof row.__id === 'string' ? ([[row.__id, row]] as const) : [])) + ); + for (const row of rows) { + for (const { dbFieldName } of internalLinkInfo) { + delete row[dbFieldName]; + } + } + + if (!sourceRecordIds.length) { + return rows; + } + + for (const info of internalLinkInfo) { + const relations = await this.findV2InternalLinkRelations(dataKnex, sourceRecordIds, info); + for (const [sourceRecordId, linkItems] of relations) { + const row = rowsById.get(sourceRecordId); + if (!row) continue; + if (info.isMultipleCellValue) { + row[info.dbFieldName] = linkItems; + } else if (linkItems[0]) { + row[info.dbFieldName] = linkItems[0]; + } + } + } + + return rows; + } + + private async findV2InternalLinkRelations( + dataKnex: Knex, + sourceRecordIds: string[], + info: V2InternalLinkFieldTableInfo + ): Promise> { + if (info.relationship === 'manyMany' || (info.relationship === 'oneMany' && info.isOneWay)) { + return this.findV2JunctionLinkRelations(dataKnex, sourceRecordIds, info); + } + + if ( + (info.relationship === 'manyOne' || info.relationship === 'oneOne') && + info.foreignKeyName !== '__id' + ) { + return this.findV2CurrentTableFkLinkRelations(dataKnex, sourceRecordIds, info); + } + + if ( + info.relationship === 'oneMany' || + ((info.relationship === 'manyOne' || info.relationship === 'oneOne') && + info.foreignKeyName === '__id') + ) { + return this.findV2ForeignTableFkLinkRelations(dataKnex, sourceRecordIds, info); + } + + return new Map(); + } + + private async findV2JunctionLinkRelations( + dataKnex: Knex, + sourceRecordIds: string[], + info: V2InternalLinkFieldTableInfo + ): Promise> { + const relationRows = await dataKnex(info.fkHostTableName) + .select({ + sourceRecordId: info.selfKeyName, + foreignRecordId: info.foreignKeyName, + }) + .whereIn(info.selfKeyName, sourceRecordIds) + .orderBy(info.orderColumnName || info.foreignKeyName, 'asc'); + + return this.groupLinkRelationRows(relationRows); + } + + private async findV2CurrentTableFkLinkRelations( + dataKnex: Knex, + sourceRecordIds: string[], + info: V2InternalLinkFieldTableInfo + ): Promise> { + const relationRows = await dataKnex(info.fkHostTableName) + .select({ + sourceRecordId: info.selfKeyName, + foreignRecordId: info.foreignKeyName, + }) + .whereIn(info.selfKeyName, sourceRecordIds) + .whereNotNull(info.foreignKeyName); + + return this.groupLinkRelationRows(relationRows); + } + + private async findV2ForeignTableFkLinkRelations( + dataKnex: Knex, + sourceRecordIds: string[], + info: V2InternalLinkFieldTableInfo + ): Promise> { + const query = dataKnex(info.fkHostTableName) + .select({ + sourceRecordId: info.selfKeyName, + foreignRecordId: info.foreignKeyName, + }) + .whereIn(info.selfKeyName, sourceRecordIds); + const relationRows = info.orderColumnName + ? await query.orderBy(info.orderColumnName, 'asc') + : await query; + + return this.groupLinkRelationRows(relationRows); + } + + private groupLinkRelationRows( + relationRows: Array<{ sourceRecordId?: unknown; foreignRecordId?: unknown }> + ): Map { + const relations = new Map(); + for (const { sourceRecordId, foreignRecordId } of relationRows) { + if (typeof sourceRecordId !== 'string' || typeof foreignRecordId !== 'string') { + continue; + } + const items = relations.get(sourceRecordId) ?? []; + items.push({ id: foreignRecordId }); + relations.set(sourceRecordId, items); + } + + return relations; + } + private toCrossBaseLinkTitle(value: unknown, isMultipleCellValue: boolean): unknown { if (value == null) { return value; @@ -1453,21 +1868,19 @@ export class BaseDuplicateService { try { return this.toCrossBaseLinkTitle(JSON.parse(value), isMultipleCellValue); } catch { + const arrayValue = this.parsePostgresTextArrayLiteral(value); + if (arrayValue) { + return this.toCrossBaseLinkTitle(arrayValue, isMultipleCellValue); + } return value; } } - if (isMultipleCellValue) { - return Array.isArray(value) - ? value - .map((item) => - item && typeof item === 'object' && 'title' in item - ? String((item as { title?: unknown }).title ?? '') - : '' - ) - .filter(Boolean) - .join(', ') - : value; + if (Array.isArray(value)) { + const titles = value + .map((item) => this.toCrossBaseLinkTitle(item, false)) + .filter((item): item is string => typeof item === 'string' && Boolean(item)); + return isMultipleCellValue ? titles.join(', ') : titles[0]; } return value && typeof value === 'object' && 'title' in value @@ -1515,44 +1928,4 @@ export class BaseDuplicateService { disconnectedLinkFieldIds ); } - - private async backfillDuplicatedBaseComputedFields( - container: DependencyContainer, - context: IExecutionContext, - targetTableIds: string[] - ) { - if (!targetTableIds.length) { - return; - } - - const tableRepository = container.resolve(v2CoreTokens.tableRepository); - const backfillService = container.resolve( - v2RecordRepositoryPostgresTokens.computedFieldBackfillService - ); - - for (const rawTableId of targetTableIds) { - const tableIdResult = TableId.create(rawTableId); - if (tableIdResult.isErr()) { - throw new Error(tableIdResult.error.message); - } - - const tableResult = await tableRepository.findOne( - context, - TableByIdSpec.create(tableIdResult.value) - ); - if (tableResult.isErr()) { - throw new Error(tableResult.error.message); - } - - const backfillResult = await backfillService.executeSyncMany(context, { - table: tableResult.value, - fields: tableResult.value.getFields(), - skipDistinctFilter: true, - includeOneManyTwoWay: true, - }); - if (backfillResult.isErr()) { - throw new Error(backfillResult.error.message); - } - } - } } diff --git a/apps/nestjs-backend/src/features/base/base-export.service.ts b/apps/nestjs-backend/src/features/base/base-export.service.ts index 850368fdfc..e61dc63d0e 100644 --- a/apps/nestjs-backend/src/features/base/base-export.service.ts +++ b/apps/nestjs-backend/src/features/base/base-export.service.ts @@ -23,7 +23,7 @@ import type { import archiver from 'archiver'; import { stringify } from 'csv-stringify/sync'; import { Knex } from 'knex'; -import { omit, pick } from 'lodash'; +import { groupBy, omit, pick } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import { IStorageConfig, StorageConfig } from '../../configs/storage'; @@ -47,6 +47,9 @@ import { EXCLUDE_SYSTEM_FIELDS } from './constant'; @Injectable() export class BaseExportService { public static CSV_CHUNK = 500; + public static FIELD_EXPORT_BATCH_SIZE = 500; + public static VIEW_EXPORT_BATCH_SIZE = 500; + private static readonly TABLE_ID_QUERY_CHUNK_SIZE = 100; public static FILE_SUFFIX = 'tea'; public static EXPORT_FIELD_COLUMNS = [ 'id', @@ -336,25 +339,8 @@ export class BaseExportService { }, }); const tableIds = tableRaws.map(({ id }) => id); - const fieldRaws = await prisma.field.findMany({ - where: { - tableId: { - in: tableIds, - }, - deletedTime: null, - }, - }); - const viewRaws = await prisma.view.findMany({ - where: { - tableId: { - in: tableIds, - }, - deletedTime: null, - }, - orderBy: { - order: 'asc', - }, - }); + const fieldRaws = await this.findFieldsByTableIds(tableIds); + const viewRaws = await this.findViewsByTableIds(tableIds); // 2. generate base structure json onProgress?.('exporting_structure'); @@ -504,6 +490,8 @@ export class BaseExportService { fieldRaws, destSpaceId ); + const fieldsByTableId = groupBy(fieldRaws, 'tableId'); + const viewsByTableId = groupBy(viewRaws, 'tableId'); const tables = [] as IBaseJson['tables']; for (const table of tableRaws) { const { name, description, order, id, icon, dbTableName } = table; @@ -516,18 +504,18 @@ export class BaseExportService { icon, dbTableName: realDbTableName, } as IBaseJson['tables'][number]; - const currentTableFields = fieldRaws.filter(({ tableId }) => tableId === id); + const currentTableFields = fieldsByTableId[id] ?? []; tableObject.fields = this.generateFieldConfig( currentTableFields, allowCrossBase, excludedTableIds, crossSpaceForeignBaseIds ); - tableObject.views = this.generateViewConfig(viewRaws.filter(({ tableId }) => tableId === id)); + tableObject.views = this.generateViewConfig(viewsByTableId[id] ?? []); tables.push(tableObject); } - const plugins = await this.generatePluginConfig(baseId, includedDashboardIds); + const plugins = await this.generatePluginConfig(baseId, includedDashboardIds, viewRaws); const folders = await this.generateFolderConfig(baseId, includedFolderIds); const nodes = await this.generateNodeConfig(baseId, includeNodes, rootNodeIds); @@ -919,6 +907,98 @@ export class BaseExportService { return await prisma.$queryRawUnsafe[]>(recordsQuery); } + async findFieldsByTableIds(tableIds: string[]) { + const prisma = this.prismaService.txClient(); + const fieldRaws: Field[] = []; + + for ( + let index = 0; + index < tableIds.length; + index += BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE + ) { + const tableIdChunk = tableIds.slice( + index, + index + BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE + ); + let cursor: string | undefined; + let hasMore = true; + + while (hasMore) { + const page = await prisma.field.findMany({ + where: { + tableId: { in: tableIdChunk }, + deletedTime: null, + }, + ...(cursor + ? { + cursor: { + id: cursor, + }, + skip: 1, + } + : {}), + orderBy: [{ tableId: 'asc' }, { id: 'asc' }], + take: BaseExportService.FIELD_EXPORT_BATCH_SIZE, + }); + + fieldRaws.push(...page); + + hasMore = page.length === BaseExportService.FIELD_EXPORT_BATCH_SIZE; + if (hasMore) { + cursor = page[page.length - 1].id; + } + } + } + + return fieldRaws; + } + + async findViewsByTableIds(tableIds: string[]) { + const prisma = this.prismaService.txClient(); + const viewRaws: View[] = []; + + for ( + let index = 0; + index < tableIds.length; + index += BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE + ) { + const tableIdChunk = tableIds.slice( + index, + index + BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE + ); + let cursor: string | undefined; + let hasMore = true; + + while (hasMore) { + const page = await prisma.view.findMany({ + where: { + tableId: { in: tableIdChunk }, + deletedTime: null, + }, + ...(cursor + ? { + cursor: { + id: cursor, + }, + skip: 1, + } + : {}), + orderBy: [{ tableId: 'asc' }, { order: 'asc' }, { id: 'asc' }], + take: BaseExportService.VIEW_EXPORT_BATCH_SIZE, + }); + + viewRaws.push(...page); + + hasMore = page.length === BaseExportService.VIEW_EXPORT_BATCH_SIZE; + if (hasMore) { + cursor = page[page.length - 1].id; + } + } + } + + return viewRaws; + } + /** * @description convert the cell value to the csv value * @param value - the cell value @@ -1437,7 +1517,7 @@ export class BaseExportService { }); } - async generatePluginConfig(baseId: string, includedDashboardIds?: string[]) { + async generatePluginConfig(baseId: string, includedDashboardIds?: string[], viewRaws?: View[]) { const pluginJson = {} as IBaseJson['plugins']; pluginJson[PluginPosition.Dashboard] = await this.generateDashboard( @@ -1447,35 +1527,17 @@ export class BaseExportService { pluginJson[PluginPosition.Panel] = await this.generatePluginPanel(baseId); - pluginJson[PluginPosition.View] = await this.generatePluginView(baseId); + pluginJson[PluginPosition.View] = await this.generatePluginView(baseId, viewRaws); return pluginJson; } - private async generatePluginView(baseId: string) { - const tableIds = await this.prismaService.txClient().tableMeta.findMany({ - where: { - baseId, - deletedTime: null, - }, - }); - - const prisma = this.prismaService.txClient(); - - const viewPluginRaws = await prisma.view.findMany({ - where: { - tableId: { - in: tableIds.map(({ id }) => id), - }, - type: ViewType.Plugin, - deletedTime: null, - }, - orderBy: { - createdTime: 'asc', - }, - }); + private async generatePluginView(baseId: string, viewRaws?: View[]) { + const viewPluginRaws = viewRaws + ? viewRaws.filter(({ type }) => type === ViewType.Plugin) + : await this.findPluginViewsByBaseId(baseId); - const viewPluginInstallRaws = await prisma.pluginInstall.findMany({ + const viewPluginInstallRaws = await this.prismaService.txClient().pluginInstall.findMany({ where: { positionId: { in: viewPluginRaws.map(({ id }) => id), @@ -1503,6 +1565,22 @@ export class BaseExportService { }) as unknown as IBaseJson['plugins'][PluginPosition.View]; } + private async findPluginViewsByBaseId(baseId: string) { + const tableIds = await this.prismaService.txClient().tableMeta.findMany({ + where: { + baseId, + deletedTime: null, + }, + select: { + id: true, + }, + }); + + return (await this.findViewsByTableIds(tableIds.map(({ id }) => id))).filter( + ({ type }) => type === ViewType.Plugin + ); + } + private async generatePluginPanel(baseId: string) { const prisma = this.prismaService.txClient(); const tableIds = await prisma.tableMeta.findMany({ diff --git a/apps/nestjs-backend/src/features/base/base-freeze.service.spec.ts b/apps/nestjs-backend/src/features/base/base-freeze.service.spec.ts new file mode 100644 index 0000000000..fb737fbf58 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-freeze.service.spec.ts @@ -0,0 +1,67 @@ +import { HttpErrorCode } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CustomHttpException } from '../../custom.exception'; +import { BaseService } from './base.service'; + +describe('BaseService write freeze', () => { + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + const prismaService = { + base: { + create: vi.fn(), + update: vi.fn(), + }, + }; + const migrationGuard = { + assertSpaceWritable: vi.fn(), + assertBaseWritable: vi.fn(), + }; + + const service = () => + new BaseService( + prismaService as never, + {} as never, + { get: vi.fn().mockReturnValue('usrxxx') } as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + migrationGuard as never + ); + + beforeEach(() => { + vi.clearAllMocks(); + migrationGuard.assertSpaceWritable.mockRejectedValue(freezeError); + migrationGuard.assertBaseWritable.mockRejectedValue(freezeError); + }); + + it('rejects base creation before calculating order or creating metadata when the target space is migrating', async () => { + await expect( + service().createBase({ name: 'Blocked base', spaceId: 'spcxxx' } as never) + ).rejects.toBe(freezeError); + + expect(migrationGuard.assertSpaceWritable).toHaveBeenCalledWith('spcxxx'); + expect(prismaService.base.create).not.toHaveBeenCalled(); + }); + + it('rejects base update and delete before metadata writes when the base space is migrating', async () => { + await expect(service().updateBase('bsexxx', { name: 'Blocked' })).rejects.toBe(freezeError); + await expect(service().deleteBase('bsexxx')).rejects.toBe(freezeError); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledTimes(2); + expect(prismaService.base.update).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts index 8d41623670..a1b9ffa07c 100644 --- a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts @@ -4,7 +4,7 @@ import { vi } from 'vitest'; import { BaseImportCsvQueueProcessor } from './base-import-csv.processor'; describe('BaseImportCsvQueueProcessor', () => { - it('writes imported record history into the routed data DB internal schema', async () => { + it('does not write record history when importing base records', async () => { const executedSql: string[] = []; const dataKnex = Knex({ client: 'pg' }); const dataPrisma = { @@ -64,6 +64,10 @@ describe('BaseImportCsvQueueProcessor', () => { columnInfo: vi.fn().mockReturnValue('SELECT * FROM columns'), }; processor.dataDbClientManager = dataDbClientManager; + Object.assign(processor, { + audit: { emitAtomic: vi.fn().mockResolvedValue(undefined) }, + cls: { get: vi.fn() }, + }); await processor.handleChunk( [{ __id: 'recImported', fldText: 'Imported value' }], @@ -82,10 +86,89 @@ describe('BaseImportCsvQueueProcessor', () => { ); expect(executedSql.some((sql) => sql.includes('"bse_data"."tbl_imported"'))).toBe(true); - expect(executedSql.some((sql) => sql.includes('"teable_internal"."record_history"'))).toBe( - true + // Base import inserts records only — it must not generate per-cell record history + // (the v1 hot path that bloated each chunk transaction and timed out on large tables). + expect(executedSql.some((sql) => sql.toLowerCase().includes('record_history'))).toBe(false); + + await dataKnex.destroy(); + }); + + it('drops ghost columns missing from the target table instead of aborting the insert', async () => { + const executedSql: string[] = []; + const dataKnex = Knex({ client: 'pg' }); + const dataPrisma = { + $queryRawUnsafe: vi + .fn() + // foreign keys info + .mockResolvedValueOnce([]) + // columnInfo: the freshly-created table only has __id + fldText, NOT the ghost column + .mockResolvedValueOnce([{ name: '__id' }, { name: 'fldText' }]), + $executeRawUnsafe: vi.fn(async (sql: string) => { + executedSql.push(sql); + return 1; + }), + }; + const prismaService = { + tableMeta: { + findUniqueOrThrow: vi.fn().mockResolvedValue({ dbTableName: 'bse_data.tbl_imported' }), + }, + txClient: vi.fn().mockReturnValue({ + attachmentsTable: { createMany: vi.fn().mockResolvedValue(undefined) }, + }), + }; + const dataDbClientManager = { + dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma), + dataKnexForBase: vi.fn().mockResolvedValue(dataKnex), + getDataDatabaseForBase: vi.fn().mockResolvedValue({ + url: 'postgresql://user:pass@example.test:5432/data?schema=teable_internal', + }), + }; + const processor = Object.create(BaseImportCsvQueueProcessor.prototype) as { + handleChunk: ( + results: Record[], + config: Record, + excludeDbFieldNames: string[] + ) => Promise; + prismaService: typeof prismaService; + dbProvider: { + getForeignKeysInfo: ReturnType; + columnInfo: ReturnType; + }; + dataDbClientManager: typeof dataDbClientManager; + }; + + processor.prismaService = prismaService; + processor.dbProvider = { + getForeignKeysInfo: vi.fn().mockReturnValue('SELECT * FROM foreign_keys'), + columnInfo: vi.fn().mockReturnValue('SELECT * FROM columns'), + }; + processor.dataDbClientManager = dataDbClientManager; + Object.assign(processor, { + audit: { emitAtomic: vi.fn().mockResolvedValue(undefined) }, + cls: { get: vi.fn() }, + }); + + await processor.handleChunk( + // fldGhost is an orphan physical column dumped into the .tea CSV with no matching field + [{ __id: 'recImported', fldText: 'Imported value', fldGhost: 'orphan value' }], + { + baseId: 'bseImport', + tableId: 'tblImport', + userId: 'usrImport', + fieldIdMap: {}, + viewIdMap: {}, + fkMap: {}, + attachmentsFields: [], + notNullFieldMap: new Map(), + fieldDbNameMap: new Map([['fldText', 'fldMappedText']]), + }, + [] ); - expect(executedSql.some((sql) => sql.includes('insert into "record_history"'))).toBe(false); + + const dataInsert = executedSql.find((sql) => sql.includes('"bse_data"."tbl_imported"')); + expect(dataInsert).toBeDefined(); + expect(dataInsert).toContain('"fldText"'); + expect(dataInsert).not.toContain('fldGhost'); await dataKnex.destroy(); }); diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts index 84d8487977..07693ddb69 100644 --- a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts @@ -2,12 +2,7 @@ import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; import { Injectable, Logger } from '@nestjs/common'; import type { IAttachmentCellValue, ILinkFieldOptions } from '@teable/core'; -import { - DbFieldType, - FieldType, - generateAttachmentId, - generateRecordHistoryId, -} from '@teable/core'; +import { DbFieldType, FieldType, generateAttachmentId } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { IBaseJson, ImportBaseRo } from '@teable/openapi'; import { CreateRecordAction, UploadType } from '@teable/openapi'; @@ -200,9 +195,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { }); } }); - const fieldDbNameMap = new Map( - table?.fields?.map(({ dbFieldName, id }) => [dbFieldName, fieldIdMap[id] ?? id]) ?? [] - ); const batchProcessor = new BatchProcessor>(async (chunk) => { totalRecordsCount += chunk.length; @@ -219,7 +211,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { fkMap, attachmentsFields, notNullFieldMap, - fieldDbNameMap, }, excludeDbFieldNames ); @@ -305,10 +296,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { return await fn(dataPrisma); } - private getDataDbInternalSchema(dataDbUrl: string) { - return new URL(dataDbUrl).searchParams.get('schema') || 'public'; - } - // Raw SQL chunk insert (no v2 events). Active BaseImport operation is set by // `handleBaseImportCsv` above; the `@Audit` atomic emit mode writes one audit row per // chunk with atomic record-create action and rootAction=BaseImport. @@ -327,20 +314,11 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { fkMap: Record; attachmentsFields: { dbFieldName: string; id: string }[]; notNullFieldMap: Map; - fieldDbNameMap: Map; }, excludeDbFieldNames: string[] ) { - const { - baseId, - tableId, - userId, - fieldIdMap, - attachmentsFields, - fkMap, - notNullFieldMap, - fieldDbNameMap, - } = config; + const { baseId, tableId, userId, fieldIdMap, attachmentsFields, fkMap, notNullFieldMap } = + config; const { dbTableName } = await this.prismaService.tableMeta.findUniqueOrThrow({ where: { id: tableId }, select: { @@ -364,22 +342,11 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { recordId: string; fieldId: string; }[]; - const recordHistoryList: { - id: string; - table_id: string; - record_id: string; - field_id: string; - before: string; - after: string; - created_by: string; - }[] = []; const dataPrisma = (await this.dataDbClientManager.dataPrismaForBase( baseId )) as IDataPrismaScopedClient; const dataKnex = await this.dataDbClientManager.dataKnexForBase(baseId); - const dataDb = await this.dataDbClientManager.getDataDatabaseForBase(baseId); - const dataDbInternalSchema = this.getDataDbInternalSchema(dataDb.url); await this.dataTransaction(dataPrisma, async (prisma) => { // delete foreign keys if(exist) then duplicate table data @@ -432,9 +399,24 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { : fkMap[name] || name; }); + // Authoritative set of columns that actually exist on the freshly-created table. + // The .tea CSV is dumped from the source data table's physical columns, which can + // include "ghost" columns left behind by deleted/renamed fields. Those have no field + // in the exported structure, so the new table lacks them; inserting them raw aborts the + // whole table transaction (42703 -> 25P02) and drops every record. Mirror the v2 import, + // which only restores columns present in the field metadata. + const realColumns = new Set(columnInfo.map(({ name }) => name)); + const recordsToInsert = newResult.map((result) => { const res = { ...result }; Object.entries(res).forEach(([key, value]) => { + // drop ghost business columns absent from the target table (system / __fk_ / __row_ + // columns are left to the dedicated handling below and the lacking-column ALTER step) + if (!key.startsWith('__') && !realColumns.has(key)) { + delete res[key]; + return; + } + if (res[key] === '') { const notNullInfo = notNullFieldMap.get(key); if (notNullInfo) { @@ -468,24 +450,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { }); }); } - - if (key.startsWith('__') && !key.startsWith('__fk_')) { - return; - } - - const sourceFieldId = key.startsWith('__fk_') ? key.slice(5) : key; - const fieldId = fieldIdMap[sourceFieldId] ?? fieldDbNameMap.get(key) ?? sourceFieldId; - if (fieldId && value !== '' && value != null) { - recordHistoryList.push({ - id: generateRecordHistoryId(), - table_id: tableId, - record_id: res['__id'] as string, - field_id: fieldId, - before: JSON.stringify({ data: null }), - after: JSON.stringify({ data: value }), - created_by: userId, - }); - } }); // default value set @@ -513,15 +477,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { const sql = dataKnex.table(dbTableName).insert(recordsToInsert).toQuery(); await prisma.$executeRawUnsafe(sql); - - if (recordHistoryList.length) { - const historySql = dataKnex - .withSchema(dataDbInternalSchema) - .insert(recordHistoryList) - .into('record_history') - .toQuery(); - await prisma.$executeRawUnsafe(historySql); - } }); // restore foreign keys with NOT VALID diff --git a/apps/nestjs-backend/src/features/base/base-import.service.spec.ts b/apps/nestjs-backend/src/features/base/base-import.service.spec.ts index 45deeffa89..89e61a8691 100644 --- a/apps/nestjs-backend/src/features/base/base-import.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/base-import.service.spec.ts @@ -1,9 +1,10 @@ import type { Readable } from 'stream'; -import { DbFieldType, FieldType } from '@teable/core'; -import type { IBaseJson, ImportBaseRo } from '@teable/openapi'; +import { DbFieldType, FieldType, HttpErrorCode } from '@teable/core'; +import { BaseDuplicateMode, type IBaseJson, type ImportBaseRo } from '@teable/openapi'; import type { RestoreRecordInput } from '@teable/v2-core'; import archiver from 'archiver'; import { vi } from 'vitest'; +import { CustomHttpException } from '../../custom.exception'; import { BaseImportService, formatBaseImportError } from './base-import.service'; @@ -39,10 +40,29 @@ interface IProcessStructureService { createBaseStructure: ReturnType; } +interface IImportBaseV2Service { + importBaseV2( + importBaseRo: Pick, + onProgress?: (...args: unknown[]) => void + ): Promise; + storageAdapter: unknown; + readDotTeaStructure: ReturnType; + v2ContainerService: unknown; + v2ContextFactory: unknown; + createBaseV2: ReturnType; + restoreBaseExtrasV2: ReturnType; + importAttachmentsV2: ReturnType; + importTableDataV2: ReturnType; + importTableLinkFieldsV2: ReturnType; + audit: unknown; + cls: unknown; +} + const dbTableName = 'bse_test.tbl_test'; const jsonColumnName = 'json_col'; const textColumnName = 'text_col'; const textCellValue = 'plain text'; +const importedBaseName = 'Imported base'; const createService = () => Object.create(BaseImportService.prototype) as IRestoreRecordInputBuilder; @@ -55,6 +75,15 @@ const createZipStream = (structure: IBaseJson) => { }; describe('BaseImportService', () => { + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + describe('formatBaseImportError', () => { it('falls back when an Error has an empty message', () => { expect(formatBaseImportError(new Error(''), 'Unknown import error')).toBe( @@ -84,6 +113,43 @@ describe('BaseImportService', () => { }); describe('processStructure', () => { + it('rejects import before downloading files or opening a transaction when the space is migrating', async () => { + const service = Object.create(BaseImportService.prototype) as { + importBase: BaseImportService['importBase']; + importBaseV2: BaseImportService['importBaseV2']; + audit: { withOperation: ReturnType }; + cls: { get: ReturnType }; + spaceDataDbMigrationGuard: { assertSpaceWritable: ReturnType }; + storageAdapter: { downloadFile: ReturnType }; + prismaService: { $tx: ReturnType }; + }; + service.audit = { + withOperation: vi.fn((_, fn: () => Promise) => fn()), + }; + service.cls = { get: vi.fn() }; + service.spaceDataDbMigrationGuard = { + assertSpaceWritable: vi.fn().mockRejectedValue(freezeError), + }; + service.storageAdapter = { downloadFile: vi.fn() }; + service.prismaService = { $tx: vi.fn() }; + const importRo = { + spaceId: 'spcImport', + notify: { path: 'imports/base.tea' }, + } as ImportBaseRo; + const onProgress = vi.fn(); + + await expect(service.importBase(importRo, onProgress)).rejects.toBe(freezeError); + await expect(service.importBaseV2(importRo, onProgress)).rejects.toBe(freezeError); + + expect(service.spaceDataDbMigrationGuard.assertSpaceWritable).toHaveBeenCalledTimes(2); + expect(service.spaceDataDbMigrationGuard.assertSpaceWritable).toHaveBeenCalledWith( + 'spcImport' + ); + expect(onProgress).not.toHaveBeenCalled(); + expect(service.storageAdapter.downloadFile).not.toHaveBeenCalled(); + expect(service.prismaService.$tx).not.toHaveBeenCalled(); + }); + it('passes transaction-aware data DB routing into structure creation', async () => { const service = Object.create(BaseImportService.prototype) as IProcessStructureService; const structure = { @@ -123,14 +189,14 @@ describe('BaseImportService', () => { it('creates imported base schemas through the space routed data client', async () => { const createdBase = { id: 'bseImported', - name: 'Imported base', + name: importedBaseName, spaceId: 'spcImport', order: 1, }; const baseCreate = vi.fn().mockResolvedValue(createdBase); const baseUpdate = vi.fn().mockResolvedValue({ ...createdBase, - name: 'Imported base', + name: importedBaseName, }); const routedExecute = vi.fn().mockResolvedValue(0); const fallbackExecute = vi.fn().mockResolvedValue(0); @@ -145,7 +211,9 @@ describe('BaseImportService', () => { cls: unknown; prismaService: unknown; dbProvider: unknown; - dataDbClientManager: unknown; + dataDbClientManager: { + dataPrismaForSpace: ReturnType; + }; dataPrismaService: unknown; }; @@ -174,10 +242,10 @@ describe('BaseImportService', () => { }; await expect( - service.createBase('spcImport', 'Imported base', 'icon', { useTransaction: true }) + service.createBase('spcImport', importedBaseName, 'icon', { useTransaction: true }) ).resolves.toMatchObject({ id: 'bseImported', - name: 'Imported base', + name: importedBaseName, }); expect(service.dataDbClientManager.dataPrismaForSpace).toHaveBeenCalledWith('spcImport', { @@ -188,6 +256,101 @@ describe('BaseImportService', () => { }); }); + describe('importBaseV2', () => { + it('restores edition extras during dottea import and returns their id maps', async () => { + const tableIdMap = { tblSource: 'tblImported' }; + const fieldIdMap = { fldSource: 'fldImported' }; + const viewIdMap = { viwSource: 'viwImported' }; + const appIdMap = { appSource: 'appImported' }; + const workflowIdMap = { wflSource: 'wflImported' }; + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + tableIdMap, + fieldIdMap, + viewIdMap, + }, + }), + }; + const queryBus = {}; + const tableRecordRepository = {}; + const unitOfWork = {}; + const db = {}; + const context = {}; + const container = { + resolve: vi.fn((token: unknown) => { + const tokenText = String(token); + if (tokenText.includes('commandBus')) return commandBus; + if (tokenText.includes('queryBus')) return queryBus; + if (tokenText.includes('tableRecordRepository')) return tableRecordRepository; + if (tokenText.includes('unitOfWork')) return unitOfWork; + return db; + }), + }; + const structure = { + id: 'bseSource', + name: 'Source base', + icon: 'icon', + tables: [], + plugins: {}, + folders: [], + nodes: [], + } as unknown as IBaseJson; + const importedBase = { id: 'bseImported', name: 'Source base', spaceId: 'spcImport' }; + const service = Object.create(BaseImportService.prototype) as IImportBaseV2Service; + + service.storageAdapter = { + downloadFile: vi.fn().mockReturnValue({}), + }; + service.readDotTeaStructure = vi.fn().mockResolvedValue(structure); + service.v2ContainerService = { + getContainerForSpace: vi.fn().mockResolvedValue(container), + }; + service.v2ContextFactory = { + createContext: vi.fn().mockResolvedValue(context), + }; + service.createBaseV2 = vi.fn().mockResolvedValue(importedBase); + service.restoreBaseExtrasV2 = vi.fn().mockResolvedValue({ appIdMap, workflowIdMap }); + service.importAttachmentsV2 = vi.fn().mockResolvedValue(undefined); + service.importTableDataV2 = vi.fn().mockResolvedValue(undefined); + service.importTableLinkFieldsV2 = vi.fn().mockResolvedValue(undefined); + service.audit = { + withOperation: vi.fn((_resolved, run: () => Promise) => run()), + }; + service.cls = { + get: vi.fn().mockReturnValue('usrImport'), + }; + + await expect( + service.importBaseV2({ + spaceId: 'spcImport', + notify: { path: 'import.tea' } as ImportBaseRo['notify'], + }) + ).resolves.toMatchObject({ + base: importedBase, + tableIdMap, + fieldIdMap, + viewIdMap, + appIdMap, + workflowIdMap, + baseIdMap: { [structure.id]: importedBase.id }, + }); + + expect(service.restoreBaseExtrasV2).toHaveBeenCalledWith( + db, + importedBase.id, + structure, + { tableIdMap, fieldIdMap, viewIdMap }, + BaseDuplicateMode.Normal, + undefined + ); + expect(commandBus.execute).toHaveBeenCalledTimes(1); + expect(service.importTableDataV2).toHaveBeenCalledTimes(1); + expect(service.importTableLinkFieldsV2).toHaveBeenCalledTimes(1); + }); + }); + describe('toRestoreRecordInput', () => { it('serializes JSON extra column values for v2 dottea row restore', () => { const service = createService(); diff --git a/apps/nestjs-backend/src/features/base/base-import.service.ts b/apps/nestjs-backend/src/features/base/base-import.service.ts index 6c6efeec85..cce4eebc62 100644 --- a/apps/nestjs-backend/src/features/base/base-import.service.ts +++ b/apps/nestjs-backend/src/features/base/base-import.service.ts @@ -1,5 +1,5 @@ import type { Readable } from 'stream'; -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger, Optional } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { DbFieldType, @@ -76,6 +76,7 @@ import { InjectStorageAdapter } from '../attachments/plugins/storage'; import { AuditScope } from '../audit/audit-scope'; import { Audit } from '../audit/audit.decorator'; import { FieldDuplicateService } from '../field/field-duplicate/field-duplicate.service'; +import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service'; import { TableService } from '../table/table.service'; import { V2ContainerService } from '../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; @@ -112,6 +113,7 @@ type IDataPrismaScopedClient = IDataPrismaExecutor & { const tableDataImportBatchSize = 100; const linkFieldImportBatchSize = 25; +const attachmentsDirPrefix = 'attachments/'; const stringifyErrorDetails = (details: unknown): string | undefined => { if (details === undefined || details === null) { @@ -181,9 +183,20 @@ export class BaseImportService { private readonly v2ContainerService: V2ContainerService, private readonly v2ContextFactory: V2ExecutionContextFactory, private readonly dataDbClientManager: DataDbClientManager, - private readonly audit: AuditScope + private readonly audit: AuditScope, + @Optional() + @Inject(SpaceDataDbMigrationGuardService) + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertSpaceWritable(spaceId: string) { + await this.spaceDataDbMigrationGuard?.assertSpaceWritable(spaceId); + } + + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + private async getMaxOrder(spaceId: string) { const spaceAggregate = await this.prismaService.txClient().base.aggregate({ where: { spaceId, deletedTime: null }, @@ -198,6 +211,7 @@ export class BaseImportService { icon?: string, routingOptions?: IDataDbRoutingOptions ) { + await this.assertSpaceWritable(spaceId); const userId = this.cls.get('user.id'); const order = (await this.getMaxOrder(spaceId)) + 1; @@ -259,7 +273,9 @@ export class BaseImportService { updateExistingBase: boolean = true ): Promise { const userId = this.cls.get('user.id'); + await this.assertSpaceWritable(spaceId); if (baseId) { + await this.assertBaseWritable(baseId); const existingResult = await sql<{ id: string; name: string; @@ -351,8 +367,10 @@ export class BaseImportService { }) async importBase(importBaseRo: ImportBaseRo, onProgress?: BaseImportProgressCallback) { const { + spaceId, notify: { path }, } = importBaseRo; + await this.assertSpaceWritable(spaceId); const logId = this.audit.current()!.operationId; onProgress?.('parsing_structure'); @@ -418,6 +436,7 @@ export class BaseImportService { spaceId, notify: { path }, } = importBaseRo; + await this.assertSpaceWritable(spaceId); onProgress?.('importing_v2'); onProgress?.('parsing_structure'); @@ -473,7 +492,7 @@ export class BaseImportService { const { tableIdMap, fieldIdMap, viewIdMap } = result.value; onProgress?.('structure_created', base.id); - await this.restoreBaseExtrasV2( + const { appIdMap, workflowIdMap } = await this.restoreBaseExtrasV2( db, base.id, structure, @@ -512,6 +531,15 @@ export class BaseImportService { tableIdMap, fieldIdMap, viewIdMap, + appIdMap, + workflowIdMap, + // Source->new base id, so EE import can rewrite base-id references baked into app + // build artifacts (matches the v1 import/duplicate idMap; v2 dropped it). + baseIdMap: { [structure.id]: base.id }, + } as IImportBaseVo & { + appIdMap: Record; + workflowIdMap: Record; + baseIdMap: Record; }; } @@ -525,8 +553,7 @@ export class BaseImportService { viewIdMap: Record; }, duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal, - onProgress?: BaseImportProgressCallback, - options?: { restoreEeResources?: boolean } + onProgress?: BaseImportProgressCallback ): Promise<{ appIdMap: Record; workflowIdMap: Record }> { const { tableIdMap, fieldIdMap, viewIdMap } = idMaps; let dashboardIdMap: Record = {}; @@ -545,20 +572,17 @@ export class BaseImportService { )); } - // Restore edition-specific resources (apps / workflows / authority matrix) and collect their - // id maps so the matching base_node rows can be remapped below. Community has none, so the - // hook is a no-op; the EE subclass overrides it. Gated to the duplicate/copy path - // (restoreEeResources) so the .tea import path keeps its current behavior untouched. - const { workflowIdMap = {}, appIdMap = {} } = options?.restoreEeResources - ? await this.restoreExtraBaseResourcesV2( - db, - baseId, - structure, - { tableIdMap, fieldIdMap, viewIdMap }, - duplicateMode, - onProgress - ) - : {}; + // Restore edition-specific resources (apps / workflows / authority matrix) through the v2 + // extension hook and collect their id maps so matching base_node rows can be remapped below. + // Community has none, so the hook is a no-op; EE overrides it for imported and duplicated bases. + const { workflowIdMap = {}, appIdMap = {} } = await this.restoreExtraBaseResourcesV2( + db, + baseId, + structure, + { tableIdMap, fieldIdMap, viewIdMap }, + duplicateMode, + onProgress + ); const hasFolders = Array.isArray(structure.folders) && structure.folders.length > 0; const hasNodes = Array.isArray(structure.nodes) && structure.nodes.length > 0; @@ -589,10 +613,10 @@ export class BaseImportService { } /** - * Hook for edition-specific (EE) base resources restored during a v2 duplicate/copy: - * apps, workflows, and the authority matrix. Community has none, so this is a no-op. - * The EE subclass overrides it to create those rows and returns their id maps so the - * caller can remap the corresponding base_node entries. + * Hook for edition-specific base resources restored during v2 `.tea` import and v2 duplicate/copy: + * apps, workflows, and the authority matrix. Community has none, so this is a no-op. The EE + * subclass overrides it to create those rows and returns their id maps so the caller can remap the + * corresponding base_node entries. */ protected async restoreExtraBaseResourcesV2( _db: Kysely, @@ -1085,35 +1109,92 @@ export class BaseImportService { } } + /** + * Stream the uploaded .tea straight from storage and hand only the entries selected by + * `match` to `consume`, one at a time. + * + * Uses the event-based `unzipper.Parse()` (NOT `{ forceStream: true }`) and, crucially, + * decides skip-vs-consume and calls `entry.autodrain()` for skipped entries + * SYNCHRONOUSLY inside the 'entry' event. That matters: unzipper begins inflating an + * entry (`zlib.createInflateRaw`) the moment it is emitted unless `autodrain()` is + * called before the next tick — a late/deferred autodrain still decompresses the entry. + * With `{ forceStream: true }` + `for await`, `autodrain()` always runs a tick too late, + * so a single entry whose deflate stream doesn't decode cleanly (e.g. an odd app-version + * zip, a truncated upload, or a data-descriptor boundary false-match) throws + * "unexpected end of file (Z_BUF_ERROR)" even though we only meant to skip it. Draining + * inline routes skipped entries through a PassThrough (no inflate), so they can never + * fail — the same approach the v1 CSV/junction importers use. + * + * Memory stays bounded to a single in-flight entry: the parser back-pressures on the + * current entry instead of buffering the archive (no temp file, no full-file buffer). + * `consume` may `await` before reading the entry; the parser simply waits. + */ + private async forEachDotTeaEntry( + path: string, + match: (entryPath: string, entry: unzipper.Entry) => boolean, + consume: (entry: unzipper.Entry) => Promise + ): Promise { + const zipStream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + const parser = unzipper.Parse(); + + await new Promise((resolve, reject) => { + let settled = false; + let chain: Promise = Promise.resolve(); + + const fail = (error: unknown) => { + if (settled) return; + settled = true; + zipStream.destroy(); + reject(error instanceof Error ? error : new Error(String(error))); + }; + + parser.on('entry', (entry: unzipper.Entry) => { + // Synchronous skip: drain (PassThrough, no inflate) entries we don't need. + if (settled || !match(entry.path, entry)) { + entry.autodrain(); + return; + } + // Matched entries are consumed sequentially; the parser back-pressures until done. + chain = chain.then(() => consume(entry)).catch(fail); + }); + parser.on('error', fail); + zipStream.on('error', fail); + parser.on('close', () => { + chain + .then(() => { + if (!settled) { + settled = true; + resolve(); + } + }) + .catch(fail); + }); + + zipStream.pipe(parser); + }); + } + private async importAttachmentsV2(db: Kysely, path: string) { await this.importAttachmentFilesV2(db, path); await this.importAttachmentMetadataV2(db, path); } private async importAttachmentFilesV2(db: Kysely, path: string) { - const zipStream = await this.storageAdapter.downloadFile( - StorageAdapter.getBucket(UploadType.Import), - path - ); - const parser = unzipper.Parse({ forceStream: true }); - zipStream.pipe(parser); const bucket = StorageAdapter.getBucket(UploadType.Table); - try { - for await (const entry of parser as AsyncIterable) { + await this.forEachDotTeaEntry( + path, + (entryPath, entry) => + entryPath.startsWith(attachmentsDirPrefix) && + entry.type !== 'Directory' && + (entryPath.split('.').pop() ?? '') !== 'csv', + async (entry) => { const filePath = entry.path; const fileSuffix = filePath.split('.').pop() ?? ''; - - if ( - !filePath.startsWith('attachments/') || - entry.type === 'Directory' || - fileSuffix === 'csv' - ) { - entry.autodrain(); - continue; - } - - const token = filePath.replace('attachments/', '').split('.')[0]; + const token = filePath.replace(attachmentsDirPrefix, '').split('.')[0]; const isThumbnail = token.includes('thumbnail__'); const finalPath = isThumbnail ? `table/${token.split('__')[1].split('.')[0]}` @@ -1128,7 +1209,7 @@ export class BaseImportService { if (existing.rows[0]) { entry.autodrain(); - continue; + return; } await this.storageAdapter.uploadFileStream(bucket, finalPath, entry, { @@ -1136,32 +1217,19 @@ export class BaseImportService { 'Content-Type': this.getAttachmentMimeType(fileSuffix), }); } - } finally { - zipStream.destroy(); - } + ); } private async importAttachmentMetadataV2(db: Kysely, path: string) { - const zipStream = await this.storageAdapter.downloadFile( - StorageAdapter.getBucket(UploadType.Import), - path - ); - const parser = unzipper.Parse({ forceStream: true }); - zipStream.pipe(parser); const userId = this.cls.get('user.id'); - try { - for await (const entry of parser as AsyncIterable) { - const filePath = entry.path; - if ( - !filePath.startsWith('attachments/') || - entry.type === 'Directory' || - !filePath.endsWith('.csv') - ) { - entry.autodrain(); - continue; - } - + await this.forEachDotTeaEntry( + path, + (entryPath, entry) => + entryPath.startsWith(attachmentsDirPrefix) && + entry.type !== 'Directory' && + entryPath.endsWith('.csv'), + async (entry) => { const csvStream = entry.pipe( csvParser.default({ mapHeaders: ({ header }) => header.replace(/^\uFEFF/, ''), @@ -1216,9 +1284,7 @@ export class BaseImportService { `.execute(db); } } - } finally { - zipStream.destroy(); - } + ); } private getAttachmentMimeType(extension: string): string { @@ -1263,6 +1329,23 @@ export class BaseImportService { return extensionToMimeType[ext] || 'application/octet-stream'; } + /** A `tables/.csv` entry (excluding junction tables) whose table exists in the structure. */ + private isImportableTableCsv( + entryPath: string, + entry: unzipper.Entry, + tablesById: Map + ): boolean { + if ( + !entryPath.startsWith('tables/') || + entry.type === 'Directory' || + !entryPath.endsWith('.csv') || + entryPath.includes('junction_') + ) { + return false; + } + return tablesById.has(entryPath.replace('tables/', '').replace(/\.csv$/, '')); + } + private async importTableDataV2( path: string, baseId: string, @@ -1274,49 +1357,29 @@ export class BaseImportService { context: IExecutionContext, onProgress?: BaseImportProgressCallback ) { - const dotTeaDataStream = await this.storageAdapter.downloadFile( - StorageAdapter.getBucket(UploadType.Import), - path - ); - const parser = unzipper.Parse({ forceStream: true }); - dotTeaDataStream.pipe(parser); - const tablesById = new Map(structure.tables.map((table) => [table.id, table])); let importedTables = 0; - for await (const entry of parser as AsyncIterable) { - const filePath = entry.path; - const isTableCsv = - filePath.startsWith('tables/') && - entry.type !== 'Directory' && - filePath.endsWith('.csv') && - !filePath.includes('junction_'); - - if (!isTableCsv) { - entry.autodrain(); - continue; - } - - const tableId = filePath.replace('tables/', '').replace(/\.csv$/, ''); - const table = tablesById.get(tableId); - if (!table) { - entry.autodrain(); - continue; + await this.forEachDotTeaEntry( + path, + (entryPath, entry) => this.isImportableTableCsv(entryPath, entry, tablesById), + async (entry) => { + const tableId = entry.path.replace('tables/', '').replace(/\.csv$/, ''); + const table = tablesById.get(tableId)!; + importedTables++; + await this.importTableDataEntryV2( + entry, + table, + baseId, + tableIdMap[table.id] ?? table.id, + viewIdMap, + commandBus, + queryBus, + context, + onProgress + ); } - - importedTables++; - await this.importTableDataEntryV2( - entry, - table, - baseId, - tableIdMap[table.id] ?? table.id, - viewIdMap, - commandBus, - queryBus, - context, - onProgress - ); - } + ); if (importedTables === 0) { onProgress?.('table_data_empty'); @@ -1476,13 +1539,6 @@ export class BaseImportService { return; } - const dotTeaDataStream = await this.storageAdapter.downloadFile( - StorageAdapter.getBucket(UploadType.Import), - path - ); - const parser = unzipper.Parse({ forceStream: true }); - dotTeaDataStream.pipe(parser); - const tablesById = new Map(structure.tables.map((table) => [table.id, table])); let processedRows = 0; let currentBatch = 0; @@ -1513,37 +1569,24 @@ export class BaseImportService { }); }; - for await (const entry of parser as AsyncIterable) { - const filePath = entry.path; - const isTableCsv = - filePath.startsWith('tables/') && - entry.type !== 'Directory' && - filePath.endsWith('.csv') && - !filePath.includes('junction_'); - - if (!isTableCsv) { - entry.autodrain(); - continue; - } - - const tableId = filePath.replace('tables/', '').replace(/\.csv$/, ''); - const table = tablesById.get(tableId); - if (!table) { - entry.autodrain(); - continue; + await this.forEachDotTeaEntry( + path, + (entryPath, entry) => this.isImportableTableCsv(entryPath, entry, tablesById), + async (entry) => { + const tableId = entry.path.replace('tables/', '').replace(/\.csv$/, ''); + const table = tablesById.get(tableId)!; + await this.importTableLinkFieldEntryV2( + entry, + baseId, + tableIdMap[table.id] ?? table.id, + queryBus, + tableRecordRepository, + unitOfWork, + context, + onLinkBatchUpdated + ); } - - await this.importTableLinkFieldEntryV2( - entry, - baseId, - tableIdMap[table.id] ?? table.id, - queryBus, - tableRecordRepository, - unitOfWork, - context, - onLinkBatchUpdated - ); - } + ); if (processedRows > 0) { onProgress?.({ @@ -1563,55 +1606,35 @@ export class BaseImportService { queryBus: IQueryBus, context: IExecutionContext ) { - const dotTeaDataStream = await this.storageAdapter.downloadFile( - StorageAdapter.getBucket(UploadType.Import), - path - ); - const parser = unzipper.Parse({ forceStream: true }); - dotTeaDataStream.pipe(parser); - const tablesById = new Map(structure.tables.map((table) => [table.id, table])); let totalRows = 0; - for await (const entry of parser as AsyncIterable) { - const filePath = entry.path; - const isTableCsv = - filePath.startsWith('tables/') && - entry.type !== 'Directory' && - filePath.endsWith('.csv') && - !filePath.includes('junction_'); - - if (!isTableCsv) { - entry.autodrain(); - continue; - } - - const tableId = filePath.replace('tables/', '').replace(/\.csv$/, ''); - const table = tablesById.get(tableId); - if (!table) { - entry.autodrain(); - continue; - } - - const config = await this.buildTableDataImportConfig( - baseId, - tableIdMap[table.id] ?? table.id, - queryBus, - context - ); - const hasLinkFields = [...config.fieldsByDbFieldName.values()].some( - (field) => field.type === FieldType.Link && !this.isRestoreComputedField(field) - ); + await this.forEachDotTeaEntry( + path, + (entryPath, entry) => this.isImportableTableCsv(entryPath, entry, tablesById), + async (entry) => { + const tableId = entry.path.replace('tables/', '').replace(/\.csv$/, ''); + const table = tablesById.get(tableId)!; + const config = await this.buildTableDataImportConfig( + baseId, + tableIdMap[table.id] ?? table.id, + queryBus, + context + ); + const hasLinkFields = [...config.fieldsByDbFieldName.values()].some( + (field) => field.type === FieldType.Link && !this.isRestoreComputedField(field) + ); - if (!hasLinkFields) { - entry.autodrain(); - continue; - } + if (!hasLinkFields) { + entry.autodrain(); + return; + } - for await (const _record of this.createTableLinkFieldUpdateStream(entry, config)) { - totalRows += 1; + for await (const _record of this.createTableLinkFieldUpdateStream(entry, config)) { + totalRows += 1; + } } - } + ); return totalRows; } diff --git a/apps/nestjs-backend/src/features/base/base.service.spec.ts b/apps/nestjs-backend/src/features/base/base.service.spec.ts index cb66471f20..0d5bb5f945 100644 --- a/apps/nestjs-backend/src/features/base/base.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/base.service.spec.ts @@ -21,6 +21,148 @@ describe('BaseService', () => { expect(service).toBeDefined(); }); + describe('enrichBaseListV2Status', () => { + const createService = () => { + const canaryService = { + shouldUseV2WithReason: vi.fn().mockResolvedValue({ useV2: true, reason: 'space_feature' }), + isSpaceInCanary: vi.fn().mockResolvedValue(true), + }; + + return { + service: new BaseService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + canaryService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ), + canaryService, + }; + }; + + it('adds canary-space v2 status for list items that are not new-base v2', async () => { + const { service, canaryService } = createService(); + + const result = await service.enrichBaseListV2Status([ + { id: 'bse1', spaceId: 'spc1', v2Enabled: false }, + ]); + + expect(canaryService.shouldUseV2WithReason).toHaveBeenCalledTimes(1); + expect(canaryService.shouldUseV2WithReason).toHaveBeenCalledWith('spc1', 'getRecords'); + expect(canaryService.isSpaceInCanary).toHaveBeenCalledWith('spc1'); + expect(result[0]).toMatchObject({ + id: 'bse1', + isCanary: true, + v2Status: { useV2: true, reason: 'space_feature' }, + }); + }); + + it('keeps new-base v2 status while batching space-level canary checks', async () => { + const { service, canaryService } = createService(); + + const result = await service.enrichBaseListV2Status([ + { id: 'bse1', spaceId: 'spc1', v2Enabled: true }, + { id: 'bse2', spaceId: 'spc1', v2Enabled: false }, + ]); + + expect(canaryService.shouldUseV2WithReason).toHaveBeenCalledTimes(1); + expect(result).toMatchObject([ + { + id: 'bse1', + v2Status: { useV2: true, reason: 'new_base' }, + }, + { + id: 'bse2', + v2Status: { useV2: true, reason: 'space_feature' }, + }, + ]); + }); + }); + + describe('getAllBaseList', () => { + it('includes v2 status for canary-space bases from the all-base list endpoint', async () => { + const spaceId = 'spc1'; + const createdTime = new Date('2026-06-13T00:00:00.000Z'); + const base = { + id: 'bse1', + name: 'Base', + order: 1, + spaceId, + icon: null, + createdBy: 'usr1', + createdTime, + lastModifiedTime: createdTime, + v2Enabled: false, + }; + const prismaService = { + base: { + findMany: vi.fn().mockResolvedValue([base]), + }, + user: { + findMany: vi.fn().mockResolvedValue([{ id: 'usr1', name: 'Nee', avatar: null }]), + }, + baseShare: { + findMany: vi.fn().mockResolvedValue([]), + }, + }; + const collaboratorService = { + getCurrentUserCollaboratorsBaseAndSpaceArray: vi.fn().mockResolvedValue({ + spaceIds: [spaceId], + baseIds: [], + roleMap: { [spaceId]: Role.Owner }, + }), + buildSpaceOwnerContext: vi.fn().mockResolvedValue({ + validCreatorSet: new Set([`${spaceId}:usr1`]), + spaceOwnerMap: new Map([[spaceId, 'usr1']]), + }), + }; + const canaryService = { + shouldUseV2WithReason: vi.fn().mockResolvedValue({ useV2: true, reason: 'space_feature' }), + isSpaceInCanary: vi.fn().mockResolvedValue(true), + }; + const testService = new BaseService( + prismaService as never, + {} as never, + {} as never, + collaboratorService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + canaryService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ); + + const result = await testService.getAllBaseList(); + + expect(canaryService.shouldUseV2WithReason).toHaveBeenCalledWith(spaceId, 'getRecords'); + expect(result[0]).toMatchObject({ + id: base.id, + role: Role.Owner, + isCanary: true, + v2Status: { useV2: true, reason: 'space_feature' }, + }); + expect(result[0]).not.toHaveProperty('v2Enabled'); + }); + }); + describe('getBaseById', () => { const createService = (params: { base: { @@ -58,10 +200,10 @@ describe('BaseService', () => { service: new BaseService( prismaService as never, {} as never, - {} as never, cls as never, {} as never, {} as never, + {} as never, permissionService as never, {} as never, {} as never, @@ -173,16 +315,17 @@ describe('BaseService', () => { const { service } = { service: new BaseService( {} as never, - defaultDataPrisma as never, dataDbClientManager as never, {} as never, {} as never, {} as never, {} as never, + {} as never, tableOpenApiService as never, {} as never, {} as never, {} as never, + {} as never, dbProvider as never, {} as never, {} as never, @@ -218,6 +361,7 @@ describe('BaseService', () => { {} as never, {} as never, {} as never, + {} as never, { dropSchema: vi.fn().mockReturnValue('') } as never, {} as never, {} as never, diff --git a/apps/nestjs-backend/src/features/base/base.service.ts b/apps/nestjs-backend/src/features/base/base.service.ts index 3231bb598c..2f8b334349 100644 --- a/apps/nestjs-backend/src/features/base/base.service.ts +++ b/apps/nestjs-backend/src/features/base/base.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger, Optional } from '@nestjs/common'; import { ActionPrefix, actionPrefixMap, @@ -52,9 +52,11 @@ import { AuditScope } from '../audit/audit-scope'; import { Audit } from '../audit/audit.decorator'; import { PermissionService } from '../auth/permission.service'; import { CanaryService } from '../canary'; +import type { IV2Decision } from '../canary'; import { CollaboratorService } from '../collaborator/collaborator.service'; import { FieldOpenApiService } from '../field/open-api/field-open-api.service'; import { GraphService } from '../graph/graph.service'; +import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service'; import { TableOpenApiService } from '../table/open-api/table-open-api.service'; import { BaseDuplicateV2Service } from './base-duplicate-v2.service'; import { BaseDuplicateService } from './base-duplicate.service'; @@ -70,6 +72,21 @@ type IDataPrismaExecutor = { $executeRawUnsafe(query: string, ...values: unknown[]): PromiseLike; }; +type IBaseListV2Source = { + spaceId: string; + v2Enabled?: boolean | null; +}; + +type IBaseListV2Context = { + spaceDecisionMap: Map; + canarySpaceMap: Map; +}; + +type IBaseListV2Info = { + isCanary?: boolean; + v2Status: IV2Decision; +}; + /** * Stable key for deduplicating orphan link-storage drops across both sides of * a symmetric pair. Both sides reference the same underlying junction (M:N) or @@ -126,13 +143,74 @@ export class BaseService { // Explicit @Inject after consecutive token-based @Inject decorators (SWC fails // to emit design:paramtypes metadata for plain class types in this position). @Inject(AuditScope) private readonly audit: AuditScope, - @Inject(EventEmitterService) private readonly eventEmitterService: EventEmitterService + @Inject(EventEmitterService) private readonly eventEmitterService: EventEmitterService, + @Optional() + @Inject(SpaceDataDbMigrationGuardService) + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} private getDataPrismaExecutor(prisma: IDataPrismaScopedClient): IDataPrismaExecutor { return prisma.txClient?.() ?? prisma; } + private async assertSpaceWritable(spaceId: string) { + await this.spaceDataDbMigrationGuard?.assertSpaceWritable(spaceId); + } + + private async buildBaseListV2Context(baseList: IBaseListV2Source[]) { + const spaceIds = uniq(baseList.map((base) => base.spaceId)); + const [spaceDecisionEntries, canarySpaceEntries] = await Promise.all([ + Promise.all( + spaceIds.map(async (spaceId) => { + return [ + spaceId, + await this.canaryService.shouldUseV2WithReason(spaceId, 'getRecords'), + ] as const; + }) + ), + Promise.all( + spaceIds.map(async (spaceId) => { + return [spaceId, await this.canaryService.isSpaceInCanary(spaceId)] as const; + }) + ), + ]); + + return { + spaceDecisionMap: new Map(spaceDecisionEntries), + canarySpaceMap: new Map(canarySpaceEntries), + }; + } + + private getBaseListV2Info(base: IBaseListV2Source, context: IBaseListV2Context): IBaseListV2Info { + return { + isCanary: context.canarySpaceMap.get(base.spaceId) || undefined, + v2Status: base.v2Enabled + ? { useV2: true, reason: 'new_base' } + : context.spaceDecisionMap.get(base.spaceId) ?? { + useV2: false, + reason: 'feature_not_enabled', + }, + }; + } + + async enrichBaseListV2Status( + baseList: T[] + ): Promise> { + if (!baseList.length) { + return []; + } + + const context = await this.buildBaseListV2Context(baseList); + return baseList.map((base) => ({ + ...base, + ...this.getBaseListV2Info(base, context), + })); + } + + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + private async getRoleByBaseId(baseId: string, spaceId: string) { const userId = this.cls.get('user.id'); const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); @@ -246,7 +324,7 @@ export class BaseService { const allBaseIds = baseList.map((base) => base.id); const allUserIds = uniq([...baseList.map((base) => base.createdBy), ...spaceOwnerMap.values()]); - const [userList, sharedBaseList] = await Promise.all([ + const [userList, sharedBaseList, baseListWithV2Status] = await Promise.all([ this.prismaService.user.findMany({ where: { id: { in: allUserIds } }, select: { id: true, name: true, avatar: true }, @@ -255,12 +333,13 @@ export class BaseService { where: { baseId: { in: allBaseIds }, nodeId: null, enabled: true }, select: { baseId: true }, }), + this.enrichBaseListV2Status(baseList), ]); const userMap = keyBy(userList, 'id'); const sharedBaseIds = new Set(sharedBaseList.map((s) => s.baseId)); - return baseList.map((base) => { + return baseListWithV2Status.map((base) => { const { v2Enabled, ...baseInfo } = base; const isCreatorInSpace = validCreatorSet.has(`${base.spaceId}:${base.createdBy}`); const displayUserId = isCreatorInSpace ? base.createdBy : spaceOwnerMap.get(base.spaceId); @@ -270,7 +349,6 @@ export class BaseService { ...baseInfo, role: roleMap[base.id] || roleMap[base.spaceId], isShared: sharedBaseIds.has(base.id), - v2Status: v2Enabled ? ({ useV2: true, reason: 'new_base' } as const) : undefined, lastModifiedTime: base.lastModifiedTime?.toISOString(), createdTime: base.createdTime?.toISOString(), createdUser: displayUser @@ -294,6 +372,7 @@ export class BaseService { async createBase(createBaseRo: ICreateBaseRo) { const userId = this.cls.get('user.id'); const { name, spaceId, icon } = createBaseRo; + await this.assertSpaceWritable(spaceId); const order = (await this.getMaxOrder(spaceId)) + 1; const base = await this.prismaService.base.create({ @@ -348,6 +427,7 @@ export class BaseService { } async updateBase(baseId: string, updateBaseRo: IUpdateBaseRo) { + await this.assertBaseWritable(baseId); const userId = this.cls.get('user.id'); return this.prismaService.base.update({ @@ -369,6 +449,7 @@ export class BaseService { } async shuffle(spaceId: string) { + await this.assertSpaceWritable(spaceId); const bases = await this.prismaService.base.findMany({ where: { spaceId, deletedTime: null }, select: { id: true }, @@ -389,6 +470,7 @@ export class BaseService { } async updateOrder(baseId: string, orderRo: IUpdateOrderRo) { + await this.assertBaseWritable(baseId); const { anchorId, position } = orderRo; const base = await this.prismaService.base @@ -447,6 +529,7 @@ export class BaseService { } async deleteBase(baseId: string) { + await this.assertBaseWritable(baseId); const userId = this.cls.get('user.id'); await this.prismaService.base.update({ @@ -465,7 +548,9 @@ export class BaseService { params: (ro: IDuplicateBaseRo) => ro as unknown as Record, }) async duplicateBase(duplicateBaseRo: IDuplicateBaseRo) { - const { fromBaseId } = duplicateBaseRo; + const { fromBaseId, spaceId } = duplicateBaseRo; + await this.assertBaseWritable(fromBaseId); + await this.assertSpaceWritable(spaceId); // Regular permission check, base update permission await this.checkBaseUpdatePermission(fromBaseId); @@ -495,7 +580,9 @@ export class BaseService { params: (ro: IDuplicateBaseRo) => ro as unknown as Record, }) async duplicateBaseV2(duplicateBaseRo: IDuplicateBaseRo) { - const { fromBaseId } = duplicateBaseRo; + const { fromBaseId, spaceId } = duplicateBaseRo; + await this.assertBaseWritable(fromBaseId); + await this.assertSpaceWritable(spaceId); // Regular permission check, base update permission await this.checkBaseUpdatePermission(fromBaseId); @@ -516,7 +603,9 @@ export class BaseService { duplicateBaseRo: IDuplicateBaseRo, onProgress?: BaseImportProgressCallback ) { - const { fromBaseId } = duplicateBaseRo; + const { fromBaseId, spaceId } = duplicateBaseRo; + await this.assertBaseWritable(fromBaseId); + await this.assertSpaceWritable(spaceId); await this.checkBaseUpdatePermission(fromBaseId); @@ -554,6 +643,10 @@ export class BaseService { createBaseFromTemplateRo: ICreateBaseFromTemplateRo ): Promise { const { spaceId, templateId, withRecords, baseId } = createBaseFromTemplateRo; + await this.assertSpaceWritable(spaceId); + if (baseId) { + await this.assertBaseWritable(baseId); + } const template = await this.prismaService.template.findUniqueOrThrow({ where: { id: templateId }, select: { @@ -705,6 +798,7 @@ export class BaseService { } async permanentDeleteBase(baseId: string, ignorePermissionCheck: boolean = false) { + await this.assertBaseWritable(baseId); if (!ignorePermissionCheck) { const accessTokenId = this.cls.get('accessTokenId'); await this.permissionService.validPermissions(baseId, ['base|delete'], accessTokenId, true); @@ -849,6 +943,8 @@ export class BaseService { async moveBase(baseId: string, moveBaseRo: IMoveBaseRo) { const { spaceId: targetSpaceId } = moveBaseRo; + await this.assertBaseWritable(baseId); + await this.assertSpaceWritable(targetSpaceId); // check if has the permission to create base in the target space await this.checkBaseCreatePermission(targetSpaceId); @@ -1150,6 +1246,7 @@ export class BaseService { } async publishBase(baseId: string, publishBaseRo: IPublishBaseRo) { + await this.assertBaseWritable(baseId); return await this.prismaService.$tx( async (prisma) => { const template = await prisma.template.findFirst({ diff --git a/apps/nestjs-backend/src/features/base/db-connection.service.spec.ts b/apps/nestjs-backend/src/features/base/db-connection.service.spec.ts index e4a62cd6a8..a76b388646 100644 --- a/apps/nestjs-backend/src/features/base/db-connection.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/db-connection.service.spec.ts @@ -1,9 +1,64 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; +import { DriverClient, HttpErrorCode } from '@teable/core'; +import knex from 'knex'; +import { describe, expect, it, vi } from 'vitest'; import { GlobalModule } from '../../global/global.module'; import { BaseModule } from './base.module'; import { DbConnectionService } from './db-connection.service'; +const baseId = 'bsexxx'; +const migrationInProgressMessage = 'Space data database migration is in progress'; +const readOnlyRole = `read_only_role_${baseId}`; +const publicDatabaseProxy = 'db-proxy.example.com:15432'; +const defaultMaxBaseDBConnections = 10; +const byodbHost = 'byodb.example.com'; +const byodbDataUrl = `postgresql://teable:secret@${byodbHost}:5432/byodb_space?schema=internal_byodb`; +const defaultDataUrl = 'postgresql://teable:secret@default.example.com:5432/teable_data'; +const sqlBuilder = knex({ client: 'pg' }); + +const createPrismaServiceMock = () => { + const txPrisma = { + base: { + findFirstOrThrow: vi.fn().mockResolvedValue({ id: baseId }), + update: vi.fn().mockResolvedValue({ id: baseId }), + }, + }; + const prismaService = { + base: { + findFirst: vi.fn().mockResolvedValue({ id: baseId, schemaPass: 'readonly-pass' }), + }, + $tx: vi.fn((fn) => fn(txPrisma)), + }; + + return { prismaService, txPrisma }; +}; + +const createDbConnectionService = ({ + prismaService, + databaseRouter, + baseConfigOverrides, +}: { + prismaService: unknown; + databaseRouter: unknown; + baseConfigOverrides?: Partial<{ + publicDatabaseProxy?: string; + defaultMaxBaseDBConnections: number; + }>; +}) => + new DbConnectionService( + prismaService as never, + databaseRouter as never, + {} as never, + { driver: DriverClient.Pg } as never, + sqlBuilder as never, + { + publicDatabaseProxy, + defaultMaxBaseDBConnections, + ...baseConfigOverrides, + } as never + ); + describe('DbConnectionService', () => { let service: DbConnectionService; @@ -18,4 +73,241 @@ describe('DbConnectionService', () => { it('should be defined', () => { expect(service).toBeDefined(); }); + + it.each([ + ['default data DB', defaultDataUrl, 'teable_data', 'db-proxy.example.com', 15432, true], + ['BYODB scoped data DB', byodbDataUrl, 'byodb_space', byodbHost, 5432, false], + ])( + 'retrieves an existing %s readonly connection through scoped routing', + async (_, dataUrl, db, host, port, isMetaFallback) => { + const { prismaService } = createPrismaServiceMock(); + const dataPrisma = { + $queryRaw: vi + .fn() + .mockResolvedValueOnce([{ count: 1n }]) + .mockResolvedValueOnce([{ count: 3n }]), + }; + const databaseRouter = { + dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma), + getDataDatabaseForBase: vi.fn().mockResolvedValue({ + url: dataUrl, + isMetaFallback, + }), + }; + const scopedService = createDbConnectionService({ prismaService, databaseRouter }); + + await expect(scopedService.retrieve(baseId)).resolves.toMatchObject({ + dsn: { + driver: DriverClient.Pg, + host, + port, + db, + user: readOnlyRole, + pass: 'readonly-pass', + params: { schema: baseId }, + }, + connection: { + max: defaultMaxBaseDBConnections, + current: 3, + }, + }); + + expect(databaseRouter.dataPrismaForBase).toHaveBeenCalledTimes(2); + expect(databaseRouter.dataPrismaForBase).toHaveBeenNthCalledWith(1, baseId); + expect(databaseRouter.dataPrismaForBase).toHaveBeenNthCalledWith(2, baseId); + expect(databaseRouter.getDataDatabaseForBase).toHaveBeenCalledWith(baseId); + } + ); + + it('creates a BYODB readonly connection on the scoped data DB', async () => { + const { prismaService, txPrisma } = createPrismaServiceMock(); + const dataPrisma = { + $executeRawUnsafe: vi.fn().mockResolvedValue(0), + }; + const databaseRouter = { + getDataDatabaseForBase: vi.fn().mockResolvedValue({ + url: byodbDataUrl, + isMetaFallback: false, + }), + dataPrismaExecutorForBase: vi.fn().mockResolvedValue(dataPrisma), + }; + const scopedService = createDbConnectionService({ prismaService, databaseRouter }); + + const result = await scopedService.create(baseId); + + expect(result).toMatchObject({ + dsn: { + driver: DriverClient.Pg, + host: byodbHost, + port: 5432, + db: 'byodb_space', + user: readOnlyRole, + params: { schema: baseId }, + }, + connection: { + max: defaultMaxBaseDBConnections, + current: 0, + }, + }); + expect(result?.url).toContain(`postgresql://${readOnlyRole}:`); + expect(result?.url).toContain(`@${byodbHost}:5432/byodb_space?schema=bsexxx`); + expect(databaseRouter.getDataDatabaseForBase).toHaveBeenCalledWith(baseId); + expect(databaseRouter.dataPrismaExecutorForBase).toHaveBeenCalledWith(baseId, { + useTransaction: true, + }); + expect(txPrisma.base.update).toHaveBeenCalledWith({ + where: { id: baseId }, + data: { schemaPass: expect.any(String) }, + }); + + const executedSql = dataPrisma.$executeRawUnsafe.mock.calls.map(([sql]) => sql).join('\n'); + expect(executedSql).toContain(`CREATE ROLE "${readOnlyRole}"`); + expect(executedSql).toContain(`GRANT USAGE ON SCHEMA "${baseId}" TO "${readOnlyRole}"`); + expect(executedSql).toContain( + `GRANT SELECT ON ALL TABLES IN SCHEMA "${baseId}" TO "${readOnlyRole}"` + ); + expect(executedSql).toContain( + `ALTER DEFAULT PRIVILEGES IN SCHEMA "${baseId}" GRANT SELECT ON TABLES TO "${readOnlyRole}"` + ); + }); + + it('creates a BYODB readonly connection with the direct scoped database host when no global proxy is configured', async () => { + const { prismaService } = createPrismaServiceMock(); + const databaseRouter = { + getDataDatabaseForBase: vi.fn().mockResolvedValue({ + url: byodbDataUrl, + isMetaFallback: false, + }), + dataPrismaExecutorForBase: vi.fn().mockResolvedValue({ + $executeRawUnsafe: vi.fn().mockResolvedValue(0), + }), + }; + const scopedService = createDbConnectionService({ + prismaService, + databaseRouter, + baseConfigOverrides: { publicDatabaseProxy: undefined }, + }); + + await expect(scopedService.create(baseId)).resolves.toMatchObject({ + dsn: { + host: byodbHost, + port: 5432, + db: 'byodb_space', + }, + }); + }); + + it('removes a BYODB readonly connection from the scoped data DB only', async () => { + const { prismaService, txPrisma } = createPrismaServiceMock(); + const dataPrisma = { + $executeRawUnsafe: vi.fn().mockResolvedValue(0), + }; + const databaseRouter = { + getDataDatabaseForBase: vi.fn(), + dataPrismaExecutorForBase: vi.fn().mockResolvedValue(dataPrisma), + }; + const scopedService = createDbConnectionService({ prismaService, databaseRouter }); + + await expect(scopedService.remove(baseId)).resolves.toBeUndefined(); + + expect(databaseRouter.dataPrismaExecutorForBase).toHaveBeenCalledWith(baseId, { + useTransaction: true, + }); + expect(databaseRouter.getDataDatabaseForBase).not.toHaveBeenCalled(); + expect(txPrisma.base.update).toHaveBeenCalledWith({ + where: { id: baseId }, + data: { schemaPass: null }, + }); + + const executedSql = dataPrisma.$executeRawUnsafe.mock.calls.map(([sql]) => sql).join('\n'); + expect(executedSql).toContain(`REVOKE USAGE ON SCHEMA "${baseId}" FROM "${readOnlyRole}"`); + expect(executedSql).toContain(`DROP ROLE IF EXISTS "${readOnlyRole}"`); + }); + + it('returns an explicit unavailable error when the scoped data DB cannot create readonly roles', async () => { + const { prismaService } = createPrismaServiceMock(); + const dataPrisma = { + $executeRawUnsafe: vi.fn().mockRejectedValue( + Object.assign(new Error('permission denied to create role'), { + code: '42501', + }) + ), + }; + const databaseRouter = { + getDataDatabaseForBase: vi.fn().mockResolvedValue({ + url: byodbDataUrl, + isMetaFallback: false, + }), + dataPrismaExecutorForBase: vi.fn().mockResolvedValue(dataPrisma), + }; + const scopedService = createDbConnectionService({ prismaService, databaseRouter }); + + await expect(scopedService.create(baseId)).rejects.toMatchObject({ + code: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE, + data: { + action: 'create', + reason: 'readonly_role_privilege_unavailable', + }, + }); + }); + + it('rejects readonly connection creation while the base space is migrating', async () => { + const migrationError = new Error(migrationInProgressMessage); + const prismaService = { $tx: vi.fn() }; + const databaseRouter = { + getDataDatabaseForBase: vi.fn(), + dataPrismaExecutorForBase: vi.fn(), + }; + const migrationGuard = { + assertBaseWritable: vi.fn().mockRejectedValue(migrationError), + }; + const guardedService = new DbConnectionService( + prismaService as never, + databaseRouter as never, + {} as never, + { driver: DriverClient.Pg } as never, + {} as never, + { + publicDatabaseProxy: 'db.example.com:5432', + defaultMaxBaseDBConnections: 10, + } as never, + migrationGuard as never + ); + + await expect(guardedService.create(baseId)).rejects.toThrow(migrationInProgressMessage); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledWith(baseId); + expect(databaseRouter.getDataDatabaseForBase).not.toHaveBeenCalled(); + expect(databaseRouter.dataPrismaExecutorForBase).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('rejects readonly connection removal while the base space is migrating', async () => { + const migrationError = new Error(migrationInProgressMessage); + const prismaService = { $tx: vi.fn() }; + const databaseRouter = { + dataPrismaExecutorForBase: vi.fn(), + }; + const migrationGuard = { + assertBaseWritable: vi.fn().mockRejectedValue(migrationError), + }; + const guardedService = new DbConnectionService( + prismaService as never, + databaseRouter as never, + {} as never, + { driver: DriverClient.Pg } as never, + {} as never, + { + publicDatabaseProxy: 'db.example.com:5432', + defaultMaxBaseDBConnections: 10, + } as never, + migrationGuard as never + ); + + await expect(guardedService.remove(baseId)).rejects.toThrow(migrationInProgressMessage); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledWith(baseId); + expect(databaseRouter.dataPrismaExecutorForBase).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); }); diff --git a/apps/nestjs-backend/src/features/base/db-connection.service.ts b/apps/nestjs-backend/src/features/base/db-connection.service.ts index 3dedd78688..0906b8e94a 100644 --- a/apps/nestjs-backend/src/features/base/db-connection.service.ts +++ b/apps/nestjs-backend/src/features/base/db-connection.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import type { IDsn } from '@teable/core'; import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core'; @@ -14,6 +14,20 @@ import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex'; +import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service'; + +const readonlyCapabilityErrorCodes = new Set(['0A000', '42501']); +const readonlyCapabilityErrorMessage = + /permission denied|insufficient privilege|must be owner|not allowed to create role|cannot create role/i; + +type IReadonlyDsnTarget = + | { + available: true; + host: string; + port: number; + db: string; + } + | { available: false }; @Injectable() export class DbConnectionService { @@ -25,9 +39,45 @@ export class DbConnectionService { private readonly configService: ConfigService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(DATA_KNEX) private readonly knex: Knex, - @BaseConfig() private readonly baseConfig: IBaseConfig + @BaseConfig() private readonly baseConfig: IBaseConfig, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + + private isReadonlyCapabilityError(error: unknown): boolean { + if (error instanceof CustomHttpException) { + return false; + } + + const candidate = error as { code?: unknown; message?: unknown }; + const code = typeof candidate.code === 'string' ? candidate.code : undefined; + const message = typeof candidate.message === 'string' ? candidate.message : undefined; + + return ( + (code != null && readonlyCapabilityErrorCodes.has(code)) || + (message != null && readonlyCapabilityErrorMessage.test(message)) + ); + } + + private throwReadonlyUnavailable(error: unknown, action: 'create' | 'remove'): never { + if (!this.isReadonlyCapabilityError(error)) { + throw error; + } + + throw new CustomHttpException( + 'Readonly database connection is unavailable for this data database', + HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE, + { + action, + reason: 'readonly_role_privilege_unavailable', + } + ); + } + private getUrlFromDsn(dsn: IDsn): string { const { driver, host, port, db, user, pass, params } = dsn; if (driver !== DriverClient.Pg) { @@ -49,6 +99,39 @@ export class DbConnectionService { return `postgresql://${user}:${pass}@${host}:${port}/${db}?${paramString}`; } + private getDefaultReadonlyDsnTarget(db: string): IReadonlyDsnTarget { + const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy; + if (!publicDatabaseProxy) { + this.logger.error('PUBLIC_DATABASE_PROXY is not found in env'); + return { available: false }; + } + + const { hostname, port } = new URL(`https://${publicDatabaseProxy}`); + return { + available: true, + host: hostname, + port: Number(port), + db, + }; + } + + private async getReadonlyDsnTarget(baseId: string): Promise { + const resolvedDataDb = await this.databaseRouter.getDataDatabaseForBase(baseId); + const { db, host, port } = parseDsn(resolvedDataDb.url); + const database = db ?? ''; + + if (resolvedDataDb.isMetaFallback) { + return this.getDefaultReadonlyDsnTarget(database); + } + + return { + available: true, + host, + port: Number(port), + db: database, + }; + } + async remove(baseId: string) { if (this.dbProvider.driver !== DriverClient.Pg) { throw new CustomHttpException('Unsupported database driver', HttpErrorCode.VALIDATION_ERROR, { @@ -60,68 +143,73 @@ export class DbConnectionService { }, }); } + await this.assertBaseWritable(baseId); const readOnlyRole = `read_only_role_${baseId}`; const schemaName = baseId; - return this.prismaService.$tx(async (prisma) => { - // Verify if the base exists and if the user is the owner - await prisma.base - .findFirstOrThrow({ - where: { id: baseId, deletedTime: null }, - }) - .catch(() => { - throw new CustomHttpException( - 'Only the base owner can remove a db connection', - HttpErrorCode.RESTRICTED_RESOURCE, - { - localization: { - i18nKey: 'httpErrors.dbConnection.onlyOwnerCanRemove', - context: { - baseId, + try { + return await this.prismaService.$tx(async (prisma) => { + // Verify if the base exists and if the user is the owner + await prisma.base + .findFirstOrThrow({ + where: { id: baseId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException( + 'Only the base owner can remove a db connection', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.dbConnection.onlyOwnerCanRemove', + context: { + baseId, + }, }, - }, - } - ); + } + ); + }); + + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForBase(baseId, { + useTransaction: true, }); - const dataPrisma = await this.databaseRouter.dataPrismaExecutorForBase(baseId, { - useTransaction: true, - }); + // Revoke permissions from the role for the schema + await dataPrisma.$executeRawUnsafe( + this.knex.raw('REVOKE USAGE ON SCHEMA ?? FROM ??', [schemaName, readOnlyRole]).toQuery() + ); - // Revoke permissions from the role for the schema - await dataPrisma.$executeRawUnsafe( - this.knex.raw('REVOKE USAGE ON SCHEMA ?? FROM ??', [schemaName, readOnlyRole]).toQuery() - ); - - await dataPrisma.$executeRawUnsafe( - this.knex - .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [ - schemaName, - readOnlyRole, - ]) - .toQuery() - ); - - // Revoke permissions from the role for the tables in schema - await dataPrisma.$executeRawUnsafe( - this.knex - .raw('REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA ?? FROM ??', [ - schemaName, - readOnlyRole, - ]) - .toQuery() - ); - - // drop the role - await dataPrisma.$executeRawUnsafe( - this.knex.raw('DROP ROLE IF EXISTS ??', [readOnlyRole]).toQuery() - ); - - await prisma.base.update({ - where: { id: baseId }, - data: { schemaPass: null }, + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [ + schemaName, + readOnlyRole, + ]) + .toQuery() + ); + + // Revoke permissions from the role for the tables in schema + await dataPrisma.$executeRawUnsafe( + this.knex + .raw('REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA ?? FROM ??', [ + schemaName, + readOnlyRole, + ]) + .toQuery() + ); + + // drop the role + await dataPrisma.$executeRawUnsafe( + this.knex.raw('DROP ROLE IF EXISTS ??', [readOnlyRole]).toQuery() + ); + + await prisma.base.update({ + where: { id: baseId }, + data: { schemaPass: null }, + }); }); - }); + } catch (error) { + this.throwReadonlyUnavailable(error, 'remove'); + } } private async roleExits(baseId: string, role: string): Promise { @@ -146,13 +234,6 @@ export class DbConnectionService { } const readOnlyRole = `read_only_role_${baseId}`; - const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy; - if (!publicDatabaseProxy) { - this.logger.error('PUBLIC_DATABASE_PROXY is not found in env'); - return null; - } - - const { hostname: dbHostProxy, port: dbPortProxy } = new URL(`https://${publicDatabaseProxy}`); // Check if the base exists and the user is the owner const base = await this.prismaService.base.findFirst({ @@ -178,15 +259,17 @@ export class DbConnectionService { const currentConnections = await this.getConnectionCount(baseId, readOnlyRole); - const databaseUrl = await this.databaseRouter.getDataDatabaseUrlForBase(baseId); - const { db } = parseDsn(databaseUrl); + const readonlyTarget = await this.getReadonlyDsnTarget(baseId); + if (!readonlyTarget.available) { + return null; + } // Construct the DSN for the read-only role const dsn: IDbConnectionVo['dsn'] = { driver: DriverClient.Pg, - host: dbHostProxy, - port: Number(dbPortProxy), - db: db, + host: readonlyTarget.host, + port: readonlyTarget.port, + db: readonlyTarget.db, user: readOnlyRole, pass: base.schemaPass, params: { @@ -218,100 +301,99 @@ export class DbConnectionService { */ async create(baseId: string) { if (this.dbProvider.driver === DriverClient.Pg) { + await this.assertBaseWritable(baseId); + const readOnlyRole = `read_only_role_${baseId}`; const schemaName = baseId; const password = nanoid(); - const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy; - if (!publicDatabaseProxy) { - this.logger.error('PUBLIC_DATABASE_PROXY is not found in env'); + const readonlyTarget = await this.getReadonlyDsnTarget(baseId); + if (!readonlyTarget.available) { return null; } - const { hostname: dbHostProxy, port: dbPortProxy } = new URL( - `https://${publicDatabaseProxy}` - ); - const databaseUrl = await this.databaseRouter.getDataDatabaseUrlForBase(baseId); - const { db } = parseDsn(databaseUrl); - - return this.prismaService.$tx(async (prisma) => { - await prisma.base - .findFirstOrThrow({ - where: { id: baseId, deletedTime: null }, - }) - .catch(() => { - throw new CustomHttpException( - 'Only base owner can create db connection', - HttpErrorCode.RESTRICTED_RESOURCE, - { - localization: { - i18nKey: 'httpErrors.dbConnection.onlyOwnerCanCreate', - context: { - baseId, + try { + return await this.prismaService.$tx(async (prisma) => { + await prisma.base + .findFirstOrThrow({ + where: { id: baseId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException( + 'Only base owner can create db connection', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.dbConnection.onlyOwnerCanCreate', + context: { + baseId, + }, }, - }, - } - ); - }); + } + ); + }); - await prisma.base.update({ - where: { id: baseId }, - data: { schemaPass: password }, - }); - - const dataPrisma = await this.databaseRouter.dataPrismaExecutorForBase(baseId, { - useTransaction: true, - }); + await prisma.base.update({ + where: { id: baseId }, + data: { schemaPass: password }, + }); - // Create a read-only role - await dataPrisma.$executeRawUnsafe( - this.knex - .raw( - `CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT ?`, - [readOnlyRole, password, this.baseConfig.defaultMaxBaseDBConnections] - ) - .toQuery() - ); + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForBase(baseId, { + useTransaction: true, + }); - await dataPrisma.$executeRawUnsafe( - this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [schemaName, readOnlyRole]).toQuery() - ); + // Create a read-only role + await dataPrisma.$executeRawUnsafe( + this.knex + .raw( + `CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT ?`, + [readOnlyRole, password, this.baseConfig.defaultMaxBaseDBConnections] + ) + .toQuery() + ); - await dataPrisma.$executeRawUnsafe( - this.knex - .raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [schemaName, readOnlyRole]) - .toQuery() - ); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [schemaName, readOnlyRole]).toQuery() + ); - await dataPrisma.$executeRawUnsafe( - this.knex - .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ - schemaName, - readOnlyRole, - ]) - .toQuery() - ); + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [schemaName, readOnlyRole]) + .toQuery() + ); - const dsn: IDbConnectionVo['dsn'] = { - driver: DriverClient.Pg, - host: dbHostProxy, - port: Number(dbPortProxy), - db: db, - user: readOnlyRole, - pass: password, - params: { - schema: baseId, - }, - }; + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ + schemaName, + readOnlyRole, + ]) + .toQuery() + ); - return { - dsn, - connection: { - max: this.baseConfig.defaultMaxBaseDBConnections, - current: 0, - }, - url: this.getUrlFromDsn(dsn), - }; - }); + const dsn: IDbConnectionVo['dsn'] = { + driver: DriverClient.Pg, + host: readonlyTarget.host, + port: readonlyTarget.port, + db: readonlyTarget.db, + user: readOnlyRole, + pass: password, + params: { + schema: baseId, + }, + }; + + return { + dsn, + connection: { + max: this.baseConfig.defaultMaxBaseDBConnections, + current: 0, + }, + url: this.getUrlFromDsn(dsn), + }; + }); + } catch (error) { + this.throwReadonlyUnavailable(error, 'create'); + } } throw new CustomHttpException('Unsupported database driver', HttpErrorCode.VALIDATION_ERROR, { diff --git a/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts index 3861189d51..c584e7997f 100644 --- a/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts +++ b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts @@ -1,6 +1,6 @@ import type { CallHandler, ExecutionContext } from '@nestjs/common'; -import { of } from 'rxjs'; -import { describe, expect, it, vi } from 'vitest'; +import { of, throwError } from 'rxjs'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { TEABLE_REQUEST_ATTRIBUTION, V2IndicatorInterceptor, @@ -32,10 +32,13 @@ vi.mock('@sentry/nestjs', () => ({ })); describe('V2IndicatorInterceptor', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it('records request attribution separately from v2 route tags', () => { const setAttributes = vi.fn(); getActiveSpan.mockReturnValue({ setAttributes }); - sentryScope.setTag.mockReset(); const cls = { get: vi.fn((key: string) => { @@ -76,4 +79,81 @@ describe('V2IndicatorInterceptor', () => { }); expect(sentryScope.setTag).toHaveBeenCalledWith(TEABLE_REQUEST_ATTRIBUTION, 'v2'); }); + + it('uses the final cls state after controller fallback', () => { + const values: Record = { + useV2: true, + v2Reason: 'canary', + v2Feature: 'importCsv', + }; + const cls = { + get: vi.fn((key: string) => values[key]), + }; + + const response = { setHeader: vi.fn() }; + const request = { + method: 'POST', + path: '/api/import/bse123', + params: {}, + }; + const context = { + switchToHttp: () => ({ + getResponse: () => response, + getRequest: () => request, + }), + } as unknown as ExecutionContext; + const next = { + handle: () => { + values.useV2 = false; + values.v2Reason = 'unsupported_feature'; + return of('ok'); + }, + } as CallHandler; + + const interceptor = new V2IndicatorInterceptor(cls as never); + interceptor.intercept(context, next).subscribe(); + + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_HEADER, 'false'); + expect(response.setHeader).toHaveBeenCalledWith( + X_TEABLE_V2_REASON_HEADER, + 'unsupported_feature' + ); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_FEATURE_HEADER, 'importCsv'); + }); + + it('stamps the guard attribution before a handler error', () => { + const cls = { + get: vi.fn((key: string) => { + const values: Record = { + useV2: true, + v2Reason: 'space_feature', + v2Feature: 'deleteField', + }; + return values[key]; + }), + }; + + const response = { setHeader: vi.fn() }; + const context = { + switchToHttp: () => ({ + getResponse: () => response, + getRequest: () => ({ + method: 'DELETE', + path: '/api/table/tbl123/field', + params: { tableId: 'tbl123' }, + }), + }), + } as unknown as ExecutionContext; + const exception = new Error('delete failed'); + const next = { handle: () => throwError(() => exception) } as CallHandler; + + const interceptor = new V2IndicatorInterceptor(cls as never); + interceptor.intercept(context, next).subscribe({ error: () => undefined }); + + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_HEADER, 'true'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_REASON_HEADER, 'space_feature'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_FEATURE_HEADER, 'deleteField'); + expect(sentryScope.setTag).toHaveBeenCalledWith(TEABLE_REQUEST_ATTRIBUTION, 'v2'); + expect(sentryScope.setTag).toHaveBeenCalledWith('teable.v2.feature', 'deleteField'); + }); }); diff --git a/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts index 8a5ecd8d74..c70b1d5fe9 100644 --- a/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts +++ b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/naming-convention */ import { Injectable, type NestInterceptor, @@ -6,48 +5,24 @@ import { type CallHandler, Logger, } from '@nestjs/common'; -import * as Sentry from '@sentry/nestjs'; import { trace } from '@opentelemetry/api'; import type { Response } from 'express'; import { ClsService } from 'nestjs-cls'; import type { Observable } from 'rxjs'; import { tap } from 'rxjs/operators'; import type { IClsStore } from '../../../types/cls'; - -export const X_TEABLE_V2_HEADER = 'x-teable-v2'; -export const X_TEABLE_V2_REASON_HEADER = 'x-teable-v2-reason'; -export const X_TEABLE_V2_FEATURE_HEADER = 'x-teable-v2-feature'; -export const TEABLE_REQUEST_ATTRIBUTION = 'teable.request.attribution'; - -type SentryScopeLike = { - setTag(key: string, value: string): void; -}; - -const getSentryScopes = (): SentryScopeLike[] => { - const sentryApi = Sentry as unknown as { - getCurrentScope?: () => SentryScopeLike | undefined; - getIsolationScope?: () => SentryScopeLike | undefined; - getCurrentHub?: () => { getScope?: () => SentryScopeLike | undefined }; - }; - - const scopes = [ - sentryApi.getCurrentScope?.(), - sentryApi.getIsolationScope?.(), - sentryApi.getCurrentHub?.()?.getScope?.(), - ].filter((scope): scope is SentryScopeLike => Boolean(scope)); - - return [...new Set(scopes)]; -}; - -const setSentryTag = (key: string, value: string | undefined) => { - if (value == null) { - return; - } - - for (const scope of getSentryScopes()) { - scope.setTag(key, value); - } -}; +import { + getV2Attribution, + getV2AttributionSpanAttributes, + setV2AttributionHeaders, + setV2AttributionOnCurrentSentryScopes, +} from '../v2-attribution'; +export { + TEABLE_REQUEST_ATTRIBUTION, + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../v2-attribution'; /** * Interceptor that adds V2 indicator to response headers and logs. @@ -65,56 +40,22 @@ export class V2IndicatorInterceptor implements NestInterceptor { constructor(private readonly cls: ClsService) {} - private setHeaderIfPossible(response: Response, name: string, value: string) { - if (response.headersSent || response.writableEnded || response.destroyed) { - return; - } - - response.setHeader(name, value); - } - intercept(context: ExecutionContext, next: CallHandler): Observable { - const useV2 = this.cls.get('useV2'); - const v2Reason = this.cls.get('v2Reason'); - const v2Feature = this.cls.get('v2Feature'); - const response = context.switchToHttp().getResponse(); const request = context.switchToHttp().getRequest(); - // Add V2 indicator headers regardless of useV2 value - // This allows clients to understand why V2 was or wasn't used - this.setHeaderIfPossible(response, X_TEABLE_V2_HEADER, useV2 ? 'true' : 'false'); - if (v2Reason) { - this.setHeaderIfPossible(response, X_TEABLE_V2_REASON_HEADER, v2Reason); - } - if (v2Feature) { - this.setHeaderIfPossible(response, X_TEABLE_V2_FEATURE_HEADER, v2Feature); - } - - // Mirror V2 indicators into Sentry tags so issue search can distinguish v1/v2 requests. - setSentryTag('teable.version', useV2 ? 'v2' : 'v1'); - setSentryTag('teable.v2.enabled', useV2 ? 'true' : 'false'); - setSentryTag('teable.v2.reason', v2Reason); - setSentryTag('teable.v2.feature', v2Feature); - setSentryTag(TEABLE_REQUEST_ATTRIBUTION, useV2 ? 'v2' : 'v1'); - - // Add span attributes for tracing - const span = trace.getActiveSpan(); - if (span) { - span.setAttributes({ - [TEABLE_REQUEST_ATTRIBUTION]: useV2 ? 'v2' : 'v1', - 'teable.v2.enabled': useV2 ?? false, - ...(v2Reason && { 'teable.v2.reason': v2Reason }), - ...(v2Feature && { 'teable.v2.feature': v2Feature }), - }); - } - - if (!useV2) { - return next.handle(); - } + // Stamp the guard decision before the handler runs so thrown requests keep attribution. + this.annotateRequest(response); return next.handle().pipe( tap(() => { + // Re-stamp after success to reflect any controller-level fallback. + const attribution = this.annotateRequest(response); + + if (!attribution.useV2) { + return; + } + // Log V2 usage for tracing this.logger.debug({ message: 'V2 implementation used', @@ -122,10 +63,27 @@ export class V2IndicatorInterceptor implements NestInterceptor { path: request.path, tableId: request.params?.tableId, useV2: true, - v2Reason, - v2Feature, + v2Reason: attribution.v2Reason, + v2Feature: attribution.v2Feature, }); }) ); } + + private annotateRequest(response: Response) { + const attribution = getV2Attribution(this.cls); + + setV2AttributionHeaders(response, attribution); + setV2AttributionOnCurrentSentryScopes(attribution); + + const span = trace.getActiveSpan(); + if (span) { + const attributes = getV2AttributionSpanAttributes(attribution); + if (Object.keys(attributes).length) { + span.setAttributes(attributes); + } + } + + return attribution; + } } diff --git a/apps/nestjs-backend/src/features/canary/v2-attribution.ts b/apps/nestjs-backend/src/features/canary/v2-attribution.ts new file mode 100644 index 0000000000..b5db144e82 --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/v2-attribution.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import * as Sentry from '@sentry/nestjs'; +import type { V2Feature } from '@teable/openapi'; +import type { Response } from 'express'; +import type { ClsService } from 'nestjs-cls'; +import type { IClsStore, IV2Reason } from '../../types/cls'; + +export const X_TEABLE_V2_HEADER = 'x-teable-v2'; +export const X_TEABLE_V2_REASON_HEADER = 'x-teable-v2-reason'; +export const X_TEABLE_V2_FEATURE_HEADER = 'x-teable-v2-feature'; +export const TEABLE_REQUEST_ATTRIBUTION = 'teable.request.attribution'; + +export interface IV2Attribution { + useV2?: boolean; + v2Reason?: IV2Reason; + v2Feature?: V2Feature; +} + +type SentryScopeLike = { + setTag(key: string, value: string): void; +}; + +export const getV2Attribution = (cls?: ClsService): IV2Attribution => { + if (!cls) { + return {}; + } + + try { + return { + useV2: cls.get('useV2'), + v2Reason: cls.get('v2Reason'), + v2Feature: cls.get('v2Feature'), + }; + } catch { + return {}; + } +}; + +const getRequestAttribution = (useV2: boolean | undefined): 'v1' | 'v2' | undefined => { + if (useV2 === true) { + return 'v2'; + } + + if (useV2 === false) { + return 'v1'; + } + + return undefined; +}; + +const getSentryScopes = (): SentryScopeLike[] => { + const sentryApi = Sentry as unknown as { + getCurrentScope?: () => SentryScopeLike | undefined; + getIsolationScope?: () => SentryScopeLike | undefined; + getCurrentHub?: () => { getScope?: () => SentryScopeLike | undefined }; + }; + + const scopes = [ + sentryApi.getCurrentScope?.(), + sentryApi.getIsolationScope?.(), + sentryApi.getCurrentHub?.()?.getScope?.(), + ].filter((scope): scope is SentryScopeLike => Boolean(scope)); + + return [...new Set(scopes)]; +}; + +const setSentryTag = ( + scope: SentryScopeLike, + key: string, + value: string | boolean | undefined +) => { + if (value == null) { + return; + } + + scope.setTag(key, String(value)); +}; + +export const setV2AttributionOnSentryScope = ( + scope: SentryScopeLike, + attribution: IV2Attribution +) => { + const requestAttribution = getRequestAttribution(attribution.useV2); + + setSentryTag(scope, 'teable.version', requestAttribution); + setSentryTag(scope, 'teable.v2.enabled', attribution.useV2); + setSentryTag(scope, 'teable.v2.reason', attribution.v2Reason); + setSentryTag(scope, 'teable.v2.feature', attribution.v2Feature); + setSentryTag(scope, TEABLE_REQUEST_ATTRIBUTION, requestAttribution); +}; + +export const setV2AttributionOnCurrentSentryScopes = (attribution: IV2Attribution) => { + for (const scope of getSentryScopes()) { + setV2AttributionOnSentryScope(scope, attribution); + } +}; + +export const getV2AttributionSpanAttributes = (attribution: IV2Attribution) => { + const requestAttribution = getRequestAttribution(attribution.useV2); + + return { + ...(requestAttribution && { [TEABLE_REQUEST_ATTRIBUTION]: requestAttribution }), + ...(attribution.useV2 != null && { 'teable.v2.enabled': attribution.useV2 }), + ...(attribution.v2Reason && { 'teable.v2.reason': attribution.v2Reason }), + ...(attribution.v2Feature && { 'teable.v2.feature': attribution.v2Feature }), + }; +}; + +export const getV2AttributionLogContext = (attribution: IV2Attribution) => { + const requestAttribution = getRequestAttribution(attribution.useV2); + + if (!requestAttribution && !attribution.v2Reason && !attribution.v2Feature) { + return undefined; + } + + return { + attribution: requestAttribution, + enabled: attribution.useV2, + reason: attribution.v2Reason, + feature: attribution.v2Feature, + }; +}; + +export const setV2AttributionHeaders = ( + response: Response, + attribution: IV2Attribution +) => { + if (response.headersSent || response.writableEnded || response.destroyed) { + return; + } + + if (attribution.useV2 != null) { + response.setHeader(X_TEABLE_V2_HEADER, attribution.useV2 ? 'true' : 'false'); + } + if (attribution.v2Reason) { + response.setHeader(X_TEABLE_V2_REASON_HEADER, attribution.v2Reason); + } + if (attribution.v2Feature) { + response.setHeader(X_TEABLE_V2_FEATURE_HEADER, attribution.v2Feature); + } +}; diff --git a/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts b/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts index 7c7ba4837d..696d9f9046 100644 --- a/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts +++ b/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts @@ -36,6 +36,8 @@ import { import type { IClsStore } from '../../types/cls'; import { getMaxLevelRole } from '../../utils/get-max-level-role'; import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { AuditScope } from '../audit/audit-scope'; +import { Audit } from '../audit/audit.decorator'; @Injectable() export class CollaboratorService { @@ -43,6 +45,7 @@ export class CollaboratorService { private readonly prismaService: PrismaService, private readonly cls: ClsService, private readonly eventEmitterService: EventEmitterService, + private readonly audit: AuditScope, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig @@ -129,7 +132,15 @@ export class CollaboratorService { protected async getBaseCollaboratorBuilder( knex: Knex.QueryBuilder, baseId: string, - options?: { includeSystem?: boolean; search?: string; type?: PrincipalType; role?: IRole[] } + options?: { + includeSystem?: boolean; + search?: string; + type?: PrincipalType; + role?: IRole[]; + // Defaults to true. Set false (e.g. for anonymous share views) to avoid + // turning the picker search into an email-based membership oracle. + searchByEmail?: boolean; + } ) { const base = await this.prismaService .txClient() @@ -139,17 +150,18 @@ export class CollaboratorService { .from('collaborator') .leftJoin('users', 'collaborator.principal_id', 'users.id') .whereIn('collaborator.resource_id', [baseId, base.spaceId]); - const { includeSystem, search, type, role } = options ?? {}; + const { includeSystem, search, type, role, searchByEmail = true } = options ?? {}; if (!includeSystem) { builder.where((db) => { return db.whereNull('users.is_system').orWhere('users.is_system', false); }); } if (search) { - this.dbProvider.searchBuilder(builder, [ - ['users.name', search], - ['users.email', search], - ]); + const searchFields: [string, string][] = [['users.name', search]]; + if (searchByEmail) { + searchFields.push(['users.email', search]); + } + this.dbProvider.searchBuilder(builder, searchFields); } if (role?.length) { @@ -634,6 +646,18 @@ export class CollaboratorService { .txClient() .base.findUniqueOrThrow({ where: { id: resourceId }, select: { spaceId: true } }); spaceId = space.spaceId; + // Base-scope audit (space-scope audit not required by current spec). + await this.audit.emitAtomic({ + action: Events.BASE_COLLABORATOR_DELETE, + resourceId, + params: { + baseId: resourceId, + spaceId, + principalId, + principalType, + oldRole: targetColl.roleName, + }, + }); } this.eventEmitterService.emitAsync( Events.COLLABORATOR_DELETE, @@ -721,6 +745,22 @@ export class CollaboratorService { .txClient() .base.findUniqueOrThrow({ where: { id: resourceId }, select: { spaceId: true } }); spaceId = space.spaceId; + // Base-scope audit. Only emit when role actually changes — same-role PATCHes + // (e.g. no-op idempotency calls) shouldn't bloat the audit log. + if (targetColl.roleName !== role) { + await this.audit.emitAtomic({ + action: Events.BASE_COLLABORATOR_UPDATE, + resourceId, + params: { + baseId: resourceId, + spaceId, + principalId, + principalType, + oldRole: targetColl.roleName, + newRole: role, + }, + }); + } } else if (resourceType === CollaboratorType.Space) { spaceId = resourceId; } @@ -769,6 +809,20 @@ export class CollaboratorService { }; } + @Audit({ + action: Events.BASE_COLLABORATOR_CREATE, + resourceId: (input: { baseId: string }) => input.baseId, + params: (input: { + baseId: string; + role: IBaseRole; + collaborators: { principalId: string; principalType: PrincipalType }[]; + }) => ({ + baseId: input.baseId, + role: input.role, + collaborators: input.collaborators, + }), + emit: true, + }) async createBaseCollaborator({ collaborators, baseId, @@ -1036,7 +1090,10 @@ export class CollaboratorService { return this.getTotalBase(baseId, options); } - async getUserCollaborators(baseId: string, options?: IListBaseCollaboratorUserRo) { + async getUserCollaborators( + baseId: string, + options?: IListBaseCollaboratorUserRo & { searchByEmail?: boolean } + ) { const { skip = 0, take = 50 } = options ?? {}; const builder = this.knex.queryBuilder(); await this.getBaseCollaboratorBuilder(builder, baseId, options); diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts index 52e3b8dd32..4fbc2814c5 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts @@ -80,6 +80,42 @@ const createService = () => {} as never ) as unknown as ITestFieldOpenApiV2Service; +const createV2ContainerService = (commandBus: unknown, tableQueryService: unknown) => { + const tracer = { + startSpan: vi.fn(() => ({ + setAttribute: vi.fn(), + setAttributes: vi.fn(), + recordError: vi.fn(), + end: vi.fn(), + })), + withSpan: vi.fn(async (_span, callback: () => Promise) => callback()), + getActiveSpan: vi.fn(), + }; + const container = { + resolve: vi.fn((token: { description?: string }) => { + if (token.description === 'v2.core.tracer') { + return tracer; + } + if (token.description === 'v2.core.commandBus') { + return commandBus; + } + if (token.description === 'v2.core.tableQueryService') { + return tableQueryService; + } + return undefined; + }), + }; + + return { + getContainerForTable: vi.fn().mockResolvedValue(container), + getContainer: vi.fn().mockResolvedValue(container), + }; +}; + +const createFieldSupplementService = () => ({ + assertSameSpaceLinkTarget: vi.fn(), +}); + describe('FieldOpenApiV2Service mapConvertFieldToV2', () => { it('maps lookup convert options with filter/sort/limit', () => { const service = createService(); @@ -1034,7 +1070,7 @@ describe('FieldOpenApiV2Service normalizeFieldVo', () => { {} as never, {} as never, {} as never, - {} as never, + createFieldSupplementService() as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1424,17 +1460,13 @@ describe('FieldOpenApiV2Service createField', () => { }), }; const service = new FieldOpenApiV2Service( - { - getContainer: async () => ({ - resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(tableQueryService), - }), - } as never, + createV2ContainerService(commandBus, tableQueryService) as never, { createContext: async () => ({ requestId: 'reqTestId' }) } as never, { field: { invalidateTables: vi.fn() } } as never, {} as never, {} as never, {} as never, - {} as never, + createFieldSupplementService() as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1494,17 +1526,13 @@ describe('FieldOpenApiV2Service createField', () => { }), }; const service = new FieldOpenApiV2Service( - { - getContainer: async () => ({ - resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(tableQueryService), - }), - } as never, + createV2ContainerService(commandBus, tableQueryService) as never, { createContext: async () => ({ requestId: 'reqTestId' }) } as never, { field: { invalidateTables: vi.fn() } } as never, {} as never, {} as never, {} as never, - {} as never, + createFieldSupplementService() as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1581,11 +1609,7 @@ describe('FieldOpenApiV2Service createFields', () => { }), }; const service = new FieldOpenApiV2Service( - { - getContainer: async () => ({ - resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(tableQueryService), - }), - } as never, + createV2ContainerService(commandBus, tableQueryService) as never, { createContext: async () => ({ requestId: 'reqTestId' }) } as never, { field: { invalidateTables: vi.fn() } } as never, {} as never, diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts index f9b494905b..36d681c028 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts @@ -45,12 +45,16 @@ import { FieldId, type ICommandBus, type IExecutionContext, + type ISpan, type ITableMapper, + type ITracer, LinkFieldConfig, LinkRelationship, TableId, type Table, type TableQueryService, + TeableSpanAttributes, + bindPreloadedTableToExecutionContext, v2CoreTokens, } from '@teable/v2-core'; import { instanceToPlain } from 'class-transformer'; @@ -102,6 +106,87 @@ type IPreparedLegacyCreateField = { nextAiConfig: IFieldVo['aiConfig'] | undefined; }; +const fieldOpenApiV2Component = 'service'; + +const withV2Span = async ( + context: IExecutionContext | undefined, + operation: string, + callback: () => Promise, + attributes: Record = {} +): Promise => { + const tracer = context?.tracer; + let span: ISpan | undefined; + try { + span = tracer?.startSpan(`teable.FieldOpenApiV2Service.${operation}`, { + [TeableSpanAttributes.VERSION]: 'v2', + [TeableSpanAttributes.COMPONENT]: fieldOpenApiV2Component, + [TeableSpanAttributes.HANDLER]: 'FieldOpenApiV2Service', + [TeableSpanAttributes.OPERATION]: `FieldOpenApiV2Service.${operation}`, + ...attributes, + }); + } catch { + span = undefined; + } + + if (!span || !tracer) { + return callback(); + } + + return tracer.withSpan(span, async () => { + try { + return await callback(); + } catch (error) { + span.recordError(error instanceof Error ? error.message : String(error)); + throw error; + } finally { + span.end(); + } + }); +}; + +const withTracerSpan = async ( + tracer: ITracer | undefined, + operation: string, + callback: () => Promise, + attributes: Record = {} +): Promise => { + let span: ISpan | undefined; + try { + span = tracer?.startSpan(`teable.FieldOpenApiV2Service.${operation}`, { + [TeableSpanAttributes.VERSION]: 'v2', + [TeableSpanAttributes.COMPONENT]: fieldOpenApiV2Component, + [TeableSpanAttributes.HANDLER]: 'FieldOpenApiV2Service', + [TeableSpanAttributes.OPERATION]: `FieldOpenApiV2Service.${operation}`, + ...attributes, + }); + } catch { + span = undefined; + } + + if (!span || !tracer) { + return callback(); + } + + return tracer.withSpan(span, async () => { + try { + return await callback(); + } catch (error) { + span.recordError(error instanceof Error ? error.message : String(error)); + throw error; + } finally { + span.end(); + } + }); +}; + +const getTableIdText = (table: Table): string | undefined => { + try { + return table.id().toString(); + } catch { + return undefined; + } +}; + @Injectable() export class FieldOpenApiV2Service { constructor( @@ -691,31 +776,58 @@ export class FieldOpenApiV2Service { table: Table; }> { const container = await this.v2ContainerService.getContainerForTable(tableId); - const commandBus = container.resolve(v2CoreTokens.commandBus); - const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); - const context = await this.v2ContextFactory.createContext(container); - const tableIdResult = TableId.create(tableId); - if (tableIdResult.isErr()) { - throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); - } + const tracer = container.resolve(v2CoreTokens.tracer); + return withTracerSpan( + tracer, + 'getCreateFieldContext', + async () => { + const commandBus = container.resolve(v2CoreTokens.commandBus); + const tableQueryService = container.resolve( + v2CoreTokens.tableQueryService + ); + const context = await withTracerSpan( + tracer, + 'createExecutionContext', + () => this.v2ContextFactory.createContext(container), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } + ); + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); + } - const tableResult = await tableQueryService.getById(context, tableIdResult.value); - if (tableResult.isErr()) { - const errMsg = tableResult.error.message ?? 'Table not found'; - const isNotFound = - tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); - throw new HttpException( - errMsg, - isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR - ); - } + const tableResult = await withV2Span( + context, + 'loadCreateFieldTable', + () => tableQueryService.getById(context, tableIdResult.value), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } + ); + if (tableResult.isErr()) { + const errMsg = tableResult.error.message ?? 'Table not found'; + const isNotFound = + tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); + throw new HttpException( + errMsg, + isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR + ); + } + bindPreloadedTableToExecutionContext(context, tableResult.value); - return { - commandBus, - tableQueryService, - context, - table: tableResult.value, - }; + return { + commandBus, + tableQueryService, + context, + table: tableResult.value, + }; + }, + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } + ); } private async prepareLegacyCreateField( @@ -729,12 +841,28 @@ export class FieldOpenApiV2Service { const nextAiConfig = hasAiConfig ? (rawFieldRo.aiConfig as IFieldVo['aiConfig'] | null | undefined) ?? null : undefined; - const mappedField = this.mapLegacyCreateFieldToV2(fieldRo); - const v2Field = await this.completeLegacyLinkDbConfigForCreate( - mappedField, - currentTable, - tableQueryService, - context + const currentTableId = getTableIdText(currentTable); + const mappedField = await withV2Span( + context, + 'mapLegacyCreateFieldToV2', + async () => this.mapLegacyCreateFieldToV2(fieldRo), + { + ...(currentTableId ? { [TeableSpanAttributes.TABLE_ID]: currentTableId } : {}), + } + ); + const v2Field = await withV2Span( + context, + 'completeLegacyLinkDbConfigForCreate', + () => + this.completeLegacyLinkDbConfigForCreate( + mappedField, + currentTable, + tableQueryService, + context + ), + { + ...(currentTableId ? { [TeableSpanAttributes.TABLE_ID]: currentTableId } : {}), + } ); return { @@ -780,13 +908,25 @@ export class FieldOpenApiV2Service { forceCompatLookupRead?: boolean; } ): Promise { - const createdFieldFromDomain = await this.extractFieldVoFromDomainTable( - table, - fieldId, - context + const createdFieldFromDomain = await withV2Span( + context, + 'extractFieldVoFromDomainTable', + () => this.extractFieldVoFromDomainTable(table, fieldId, context), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + [TeableSpanAttributes.FIELD_ID]: fieldId, + } ); return options?.forceCompatLookupRead === true || createdFieldFromDomain.isLookup === true - ? await this.getFieldFromV2(tableId, fieldId, context) + ? await withV2Span( + context, + 'getCreatedFieldFromV2Compat', + () => this.getFieldFromV2(tableId, fieldId, context), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + [TeableSpanAttributes.FIELD_ID]: fieldId, + } + ) : createdFieldFromDomain; } @@ -1285,14 +1425,23 @@ export class FieldOpenApiV2Service { ); } - const preparedField = await this.prepareLegacyCreateField( - fieldRo, - table, - tableQueryService, - context + const preparedField = await withV2Span( + context, + 'prepareLegacyCreateField', + () => this.prepareLegacyCreateField(fieldRo, table, tableQueryService, context), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } ); const { hasAiConfig, nextAiConfig, v2Field } = preparedField; - await this.assertCrossSpaceForV2Field(tableId, v2Field); + await withV2Span( + context, + 'assertCrossSpaceForV2Field', + () => this.assertCrossSpaceForV2Field(tableId, v2Field), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } + ); const legacyViewId = fieldRo && typeof fieldRo === 'object' && 'viewId' in fieldRo ? (fieldRo.viewId as string | undefined) @@ -1313,13 +1462,21 @@ export class FieldOpenApiV2Service { orderIndex: legacyOrder.orderIndex, } : undefined; - const commandResult = CreateFieldCommand.create({ - baseId: table.baseId().toString(), - tableId, - field: v2Field, - ...(typeof legacyViewId === 'string' ? { viewId: legacyViewId } : {}), - ...(normalizedOrder ? { order: normalizedOrder } : {}), - }); + const commandResult = await withV2Span( + context, + 'createCreateFieldCommand', + async () => + CreateFieldCommand.create({ + baseId: table.baseId().toString(), + tableId, + field: v2Field, + ...(typeof legacyViewId === 'string' ? { viewId: legacyViewId } : {}), + ...(normalizedOrder ? { order: normalizedOrder } : {}), + }), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } + ); if (commandResult.isErr()) { this.throwV2Error( @@ -1328,9 +1485,13 @@ export class FieldOpenApiV2Service { ); } - const result = await commandBus.execute( + const result = await withV2Span( context, - commandResult.value + 'executeCreateFieldCommand', + () => commandBus.execute(context, commandResult.value), + { + [TeableSpanAttributes.TABLE_ID]: tableId, + } ); if (result.isErr()) { @@ -1343,15 +1504,19 @@ export class FieldOpenApiV2Service { this.invalidateFieldLoader(this.collectFieldInvalidateTableIds(tableId, [v2Field])); if (typeof v2Field.id === 'string') { + const createdFieldId = v2Field.id; const shouldForceCompatLookupRead = v2Field.type === 'lookup' || v2Field.type === 'conditionalLookup'; - const createdField = await this.materializeCreatedFieldVo( - tableId, - result.value.table, - v2Field.id, + const createdField = await withV2Span( context, + 'materializeCreatedFieldVo', + () => + this.materializeCreatedFieldVo(tableId, result.value.table, createdFieldId, context, { + forceCompatLookupRead: shouldForceCompatLookupRead, + }), { - forceCompatLookupRead: shouldForceCompatLookupRead, + [TeableSpanAttributes.TABLE_ID]: tableId, + [TeableSpanAttributes.FIELD_ID]: createdFieldId, } ); diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts index 38f19a3073..fd27cae1a9 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts @@ -1,13 +1,18 @@ /* eslint-disable sonarjs/cognitive-complexity */ /* eslint-disable @typescript-eslint/naming-convention */ -import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, + Optional, +} from '@nestjs/common'; import { CellValueType, ColorConfigType, FieldKeyType, FieldOpBuilder, FieldType, - HttpErrorCode, ViewType, generateFieldId, generateOperationId, @@ -55,13 +60,13 @@ import { groupBy, isEqual, omit, pick } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import { ThresholdConfig, IThresholdConfig } from '../../../configs/threshold.config'; -import { CustomHttpException } from '../../../custom.exception'; import { FieldReferenceCompatibilityException } from '../../../db-provider/filter-query/cell-value-filter.abstract'; import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; import { Events } from '../../../event-emitter/events'; import { IDataDbRoutingOptions } from '../../../global/data-db-client-manager.service'; import { DatabaseRouter } from '../../../global/database-router.service'; import type { IClsStore } from '../../../types/cls'; +import { majorFieldKeysChanged } from '../../../utils/major-field-keys-changed'; import { Timing } from '../../../utils/timing'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; import type { IOpsMap } from '../../calculation/utils/compose-maps'; @@ -70,6 +75,7 @@ import { ComputedOrchestratorService } from '../../record/computed/services/comp import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../../record/query-builder'; import { RecordService } from '../../record/record.service'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { TableIndexService } from '../../table/table-index.service'; import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; import { ViewService } from '../../view/view.service'; @@ -140,9 +146,15 @@ export class FieldOpenApiService { @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, - private readonly computedOrchestrator: ComputedOrchestratorService + private readonly computedOrchestrator: ComputedOrchestratorService, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + async planField(tableId: string, fieldId: string) { return await this.graphService.planField(tableId, fieldId); } @@ -1161,6 +1173,7 @@ export class FieldOpenApiService { options: CreateFieldsOptions = {} ) { if (!fields.length) return; + await this.assertTableWritable(tableId); const orderedFields = this.sortCreateFieldsByDependencies(tableId, fields); @@ -1291,6 +1304,7 @@ export class FieldOpenApiService { routingOptions?: IDataDbRoutingOptions ): Promise { if (!fieldRos.length) return []; + await this.assertTableWritable(tableId); const fieldVos = await this.fieldSupplementService.prepareCreateFields( tableId, fieldRos, @@ -1369,6 +1383,7 @@ export class FieldOpenApiService { windowId?: string, routingOptions?: IDataDbRoutingOptions ) { + await this.assertTableWritable(tableId); const fieldVo = await this.fieldSupplementService.prepareCreateField( tableId, fieldRo, @@ -1444,6 +1459,7 @@ export class FieldOpenApiService { @Timing() async deleteFields(tableId: string, fieldIds: string[], windowId?: string) { + await this.assertTableWritable(tableId); const { fields, fieldVos, columnsMeta, referenceMap, records } = await this.prismaService.$tx( async () => { const fieldRaws = await this.prismaService.txClient().field.findMany({ @@ -1563,6 +1579,7 @@ export class FieldOpenApiService { } async updateField(tableId: string, fieldId: string, updateFieldRo: IUpdateFieldRo) { + await this.assertTableWritable(tableId); const ops: IOtOperation[] = []; if (updateFieldRo.name) { const op = await this.updateUniqProperty(tableId, fieldId, 'name', updateFieldRo.name); @@ -1791,6 +1808,7 @@ export class FieldOpenApiService { updateFieldRo: IConvertFieldRo, windowId?: string ): Promise { + await this.assertTableWritable(tableId); const { oldFieldVo, newFieldVo, modifiedOps, references, supplementChange } = await this.prismaService.$tx( async () => { @@ -1816,9 +1834,15 @@ export class FieldOpenApiService { ); const shouldRecomputeSelf = this.fieldConvertingService.needCalculate(newField, oldField); - const filteredDependentFieldIds = shouldRecomputeSelf - ? dependentFieldIds - : dependentFieldIds.filter((id) => id !== newField.id); + const shouldRecomputeDependents = + shouldRecomputeSelf || + majorFieldKeysChanged(oldField, newField) || + Boolean(analysisResult.supplementChange); + const filteredDependentFieldIds = shouldRecomputeDependents + ? shouldRecomputeSelf + ? dependentFieldIds + : dependentFieldIds.filter((id) => id !== newField.id) + : []; const { compatibilityIssue } = await this.performConvertField({ tableId, @@ -1963,6 +1987,7 @@ export class FieldOpenApiService { tableId: string, fieldId: string ): Promise<{ field: IFieldVo; oldLinkOptions: ILinkFieldOptions }> { + await this.assertTableWritable(tableId); return this.prismaService.$tx( async () => { // Pass `options: {}` explicitly so stageAnalysis assigns the empty diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api-freeze.service.spec.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api-freeze.service.spec.ts new file mode 100644 index 0000000000..b91088544c --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api-freeze.service.spec.ts @@ -0,0 +1,146 @@ +import { HttpErrorCode } from '@teable/core'; +import { SUPPORTEDTYPE } from '@teable/openapi'; +import { vi } from 'vitest'; +import { CustomHttpException } from '../../../custom.exception'; +import { ImportOpenApiV2Service } from './import-open-api-v2.service'; +import { ImportOpenApiService } from './import-open-api.service'; + +describe('Import open API write freeze', () => { + const importCsvUrl = 'https://example.com/import.csv'; + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + + it('rejects v2 create-table imports before resolving the v2 container', async () => { + const service = Object.create(ImportOpenApiV2Service.prototype) as { + createTableFromCsvImport: ImportOpenApiV2Service['createTableFromCsvImport']; + spaceDataDbMigrationGuard: { assertBaseWritable: ReturnType }; + v2ContainerService: { getContainerForBase: ReturnType }; + audit: { withOperation: ReturnType }; + cls: { get: ReturnType }; + }; + service.audit = { + withOperation: vi.fn((_, fn: () => Promise) => fn()), + }; + service.cls = { get: vi.fn() }; + service.spaceDataDbMigrationGuard = { + assertBaseWritable: vi.fn().mockRejectedValue(freezeError), + }; + service.v2ContainerService = { + getContainerForBase: vi.fn(), + }; + + await expect( + service.createTableFromCsvImport('bseImport', { + attachmentUrl: importCsvUrl, + fileType: SUPPORTEDTYPE.CSV, + worksheets: {}, + }) + ).rejects.toBe(freezeError); + + expect(service.spaceDataDbMigrationGuard.assertBaseWritable).toHaveBeenCalledWith('bseImport'); + expect(service.v2ContainerService.getContainerForBase).not.toHaveBeenCalled(); + }); + + it('rejects v2 inplace imports before resolving the v2 container', async () => { + const service = Object.create(ImportOpenApiV2Service.prototype) as { + importRecords: ImportOpenApiV2Service['importRecords']; + spaceDataDbMigrationGuard: { assertTableWritable: ReturnType }; + v2ContainerService: { getContainerForTable: ReturnType }; + audit: { withOperation: ReturnType }; + cls: { get: ReturnType }; + }; + service.audit = { + withOperation: vi.fn((_, fn: () => Promise) => fn()), + }; + service.cls = { get: vi.fn() }; + service.spaceDataDbMigrationGuard = { + assertTableWritable: vi.fn().mockRejectedValue(freezeError), + }; + service.v2ContainerService = { + getContainerForTable: vi.fn(), + }; + + await expect( + service.importRecords('bseImport', 'tblImport', { + attachmentUrl: importCsvUrl, + fileType: SUPPORTEDTYPE.CSV, + insertConfig: { + sourceColumnMap: {}, + sourceWorkSheetKey: 'sheet1', + excludeFirstRow: false, + }, + }) + ).rejects.toBe(freezeError); + + expect(service.spaceDataDbMigrationGuard.assertTableWritable).toHaveBeenCalledWith('tblImport'); + expect(service.v2ContainerService.getContainerForTable).not.toHaveBeenCalled(); + }); + + it('rejects queued create-table imports before checking queue capacity', async () => { + const service = Object.create(ImportOpenApiService.prototype) as { + createTableFromImport: ImportOpenApiService['createTableFromImport']; + spaceDataDbMigrationGuard: { assertBaseWritable: ReturnType }; + importTableCsvChunkQueueProcessor: { + queue: { getJobCountByTypes: ReturnType }; + }; + }; + service.spaceDataDbMigrationGuard = { + assertBaseWritable: vi.fn().mockRejectedValue(freezeError), + }; + service.importTableCsvChunkQueueProcessor = { + queue: { getJobCountByTypes: vi.fn() }, + }; + + await expect( + service.createTableFromImport('bseImport', { + attachmentUrl: importCsvUrl, + fileType: SUPPORTEDTYPE.CSV, + worksheets: {}, + }) + ).rejects.toBe(freezeError); + + expect(service.spaceDataDbMigrationGuard.assertBaseWritable).toHaveBeenCalledWith('bseImport'); + expect( + service.importTableCsvChunkQueueProcessor.queue.getJobCountByTypes + ).not.toHaveBeenCalled(); + }); + + it('rejects queued inplace imports before checking queue capacity', async () => { + const service = Object.create(ImportOpenApiService.prototype) as { + inplaceImportTable: ImportOpenApiService['inplaceImportTable']; + spaceDataDbMigrationGuard: { assertTableWritable: ReturnType }; + importTableCsvChunkQueueProcessor: { + queue: { getJobCountByTypes: ReturnType }; + }; + }; + service.spaceDataDbMigrationGuard = { + assertTableWritable: vi.fn().mockRejectedValue(freezeError), + }; + service.importTableCsvChunkQueueProcessor = { + queue: { getJobCountByTypes: vi.fn() }, + }; + + await expect( + service.inplaceImportTable('bseImport', 'tblImport', { + attachmentUrl: importCsvUrl, + fileType: SUPPORTEDTYPE.CSV, + insertConfig: { + sourceColumnMap: {}, + sourceWorkSheetKey: 'sheet1', + excludeFirstRow: false, + }, + }) + ).rejects.toBe(freezeError); + + expect(service.spaceDataDbMigrationGuard.assertTableWritable).toHaveBeenCalledWith('tblImport'); + expect( + service.importTableCsvChunkQueueProcessor.queue.getJobCountByTypes + ).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts index 7d5f29dbef..6f429a1fb7 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts @@ -1,11 +1,20 @@ -import { Injectable, HttpException, HttpStatus, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; +import { Injectable, HttpException, HttpStatus, Logger, Optional } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; +import { ConfigService } from '@nestjs/config'; import { HttpErrorCode } from '@teable/core'; -import { CreateRecordAction, type IInplaceImportOptionRo } from '@teable/openapi'; +import { + CreateRecordAction, + SUPPORTEDTYPE, + type IImportOptionRo, + type IInplaceImportOptionRo, + type ITableFullVo, +} from '@teable/openapi'; +import { mapDomainErrorToHttpStatus } from '@teable/v2-contract-http'; import { v2CoreTokens, type ICommandBus, + ImportCsvCommand, + type ImportCsvResult, ImportRecordsCommand, type ImportRecordsResult, } from '@teable/v2-core'; @@ -18,6 +27,9 @@ import { Events } from '../../../event-emitter/events'; import type { IClsStore } from '../../../types/cls'; import { AuditScope } from '../../audit/audit-scope'; import { Audit } from '../../audit/audit.decorator'; +import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; +import { TableOpenApiService } from '../../table/open-api/table-open-api.service'; import { V2ContainerService } from '../../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; @@ -36,8 +48,12 @@ export class ImportOpenApiV2Service { private readonly cls: ClsService, private readonly configService: ConfigService, private readonly audit: AuditScope, + private readonly tableOpenApiService: TableOpenApiService, + private readonly fieldOpenApiService: FieldOpenApiService, private readonly eventEmitter: EventEmitter2, - @BaseConfig() private readonly baseConfig: IBaseConfig + @BaseConfig() private readonly baseConfig: IBaseConfig, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} /** @@ -77,6 +93,120 @@ export class ImportOpenApiV2Service { }); } + /** + * Create a new table from a CSV file using V2 architecture via CommandBus. + * + * This adapts the existing V1 OpenAPI payload shape used by the UI + * (attachmentUrl + worksheets) into the V2 ImportCsvCommand URL source. + */ + @Audit({ + rootAction: CreateRecordAction.Import, + resourceId: (baseId: string) => baseId, + params: (_baseId: string, importOptions: IImportOptionRo) => ({ + fileType: importOptions.fileType, + }), + }) + async createTableFromCsvImport( + baseId: string, + importOptions: IImportOptionRo, + maxRowCount?: number + ): Promise { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + + if (importOptions.fileType !== SUPPORTEDTYPE.CSV) { + throw new HttpException( + 'V2 create-table import only supports CSV files', + HttpStatus.BAD_REQUEST + ); + } + + const worksheets = Object.values(importOptions.worksheets); + const worksheet = worksheets.find((item) => item.importData) ?? worksheets[0]; + if (!worksheet) { + return []; + } + + const container = await this.v2ContainerService.getContainerForBase(baseId); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(container); + const resolvedUrl = this.resolveUrl(importOptions.attachmentUrl); + const normalizedMaxRowCount = + maxRowCount !== undefined && maxRowCount > 0 ? maxRowCount : undefined; + + const commandResult = ImportCsvCommand.createFromUrl({ + baseId, + csvUrl: resolvedUrl, + tableName: worksheet.name, + importData: worksheet.importData, + columns: worksheet.columns.length + ? worksheet.columns.map((column) => ({ + name: column.name, + sourceColumnIndex: column.sourceColumnIndex, + type: column.type, + })) + : undefined, + batchSize: normalizedMaxRowCount ? Math.min(normalizedMaxRowCount, 500) : 500, + maxRowCount: normalizedMaxRowCount, + }); + if (commandResult.isErr()) { + this.throwV2Error(commandResult.error, mapDomainErrorToHttpStatus(commandResult.error)); + } + + const result = await commandBus.execute( + context, + commandResult.value + ); + if (result.isErr()) { + this.logger.error('V2 import CSV failed', result.error); + this.eventEmitter.emit(Events.V2_TABLE_IMPORT_FINISH, { + baseId, + status: 'failed', + error: result.error.message, + }); + this.throwV2Error(result.error, mapDomainErrorToHttpStatus(result.error)); + } + + const tableId = result.value.table.id().toString(); + const table = (await this.tableOpenApiService.getTable(baseId, tableId)) as ITableFullVo; + table.fields = await this.fieldOpenApiService.getFields(tableId, {}); + table.records = []; + this.eventEmitter.emit(Events.V2_TABLE_IMPORT_FINISH, { + baseId, + tableId, + status: 'completed', + }); + return [table]; + } + + private validateImportProjection( + sourceColumnMap: Record, + projection?: string[] + ): void { + if (!projection) { + return; + } + + const fieldIds = Object.keys(sourceColumnMap); + const noUpdateFields = difference(fieldIds, projection); + if (noUpdateFields.length === 0) { + return; + } + + const tips = noUpdateFields.join(','); + throw new CustomHttpException( + `There is no permission to update these fields: ${tips}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.updateRecordWithDeniedFields', + context: { + fields: tips, + }, + }, + } + ); + } + /** * Import records using V2 architecture via CommandBus. * Appends records from a file (CSV/Excel) to an existing table. @@ -108,6 +238,8 @@ export class ImportOpenApiV2Service { maxRowCount?: number, projection?: string[] ): Promise<{ totalImported: number }> { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); @@ -116,26 +248,7 @@ export class ImportOpenApiV2Service { const { attachmentUrl, fileType, insertConfig } = importOptions; const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig; - // Validate field permissions if projection is provided - if (projection) { - const fieldIds = Object.keys(sourceColumnMap); - const noUpdateFields = difference(fieldIds, projection); - if (noUpdateFields.length !== 0) { - const tips = noUpdateFields.join(','); - throw new CustomHttpException( - `There is no permission to update these fields: ${tips}`, - HttpErrorCode.RESTRICTED_RESOURCE, - { - localization: { - i18nKey: 'httpErrors.permission.updateRecordWithDeniedFields', - context: { - fields: tips, - }, - }, - } - ); - } - } + this.validateImportProjection(sourceColumnMap, projection); // Resolve relative URL to absolute URL const resolvedUrl = this.resolveUrl(attachmentUrl); @@ -171,6 +284,12 @@ export class ImportOpenApiV2Service { if (result.isErr()) { this.logger.error('V2 import records failed', result.error); + this.eventEmitter.emit(Events.V2_TABLE_IMPORT_FINISH, { + baseId, + tableId, + status: 'failed', + error: result.error.message, + }); // Map domain error to HTTP status const status = @@ -182,25 +301,15 @@ export class ImportOpenApiV2Service { ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR; - // Mirror the V1 worker (import-csv.processor.ts) terminal signal so e2e tests - // and downstream consumers wake up on V2 imports too. V1 emits on lastChunk + - // catch; V2 is synchronous so emits at the return-success/throw boundary. - this.eventEmitter.emit(Events.TABLE_IMPORT_FINISH, { - tableId, - baseId, - status: 'failed', - error: result.error.message, - }); - this.throwV2Error(result.error, status); } // No manual audit emit: ImportRecordsHandler publishes RecordsBatchCreated per batch. // The projection writes one audit_log row per batch naturally, keeping the atomic // record-create action and attaching rootAction=InplaceImport from this operation. - this.eventEmitter.emit(Events.TABLE_IMPORT_FINISH, { - tableId, + this.eventEmitter.emit(Events.V2_TABLE_IMPORT_FINISH, { baseId, + tableId, status: 'completed', }); return { totalImported: result.value.totalImported }; diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts index b168fe42e9..0c44766ad5 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts @@ -16,6 +16,7 @@ import { importOptionRoSchema, IInplaceImportOptionRo, inplaceImportOptionRoSchema, + SUPPORTEDTYPE, } from '@teable/openapi'; import type { ITableFullVo, IAnalyzeVo, IImportStatusVo } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; @@ -55,12 +56,22 @@ export class ImportController { } @Post(':baseId') + @UseV2Feature('importCsv') @Permissions('base|table_import') @TokenAccess() async createTableFromImport( @Param('baseId') baseId: string, @Body(new ZodValidationPipe(importOptionRoSchema)) importRo: IImportOptionRo ): Promise { + if (this.cls.get('useV2') && importRo.fileType === SUPPORTEDTYPE.CSV) { + return await this.importOpenApiV2Service.createTableFromCsvImport(baseId, importRo); + } + + if (this.cls.get('useV2')) { + this.cls.set('useV2', false); + this.cls.set('v2Reason', 'unsupported_feature'); + } + return await this.importOpenService.createTableFromImport(baseId, importRo); } diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts index 30f739a884..612f02a915 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts @@ -4,6 +4,7 @@ import { CanaryModule } from '../../canary/canary.module'; import { FieldOpenApiModule } from '../../field/open-api/field-open-api.module'; import { NotificationModule } from '../../notification/notification.module'; import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; +import { SpaceModule } from '../../space/space.module'; import { TableOpenApiModule } from '../../table/open-api/table-open-api.module'; import { V2Module } from '../../v2/v2.module'; import { ImportMetricsModule } from '../metrics/import-metrics.module'; @@ -23,6 +24,7 @@ import { ImportOpenApiService } from './import-open-api.service'; V2Module, CanaryModule, ImportMetricsModule, + SpaceModule, ], controllers: [ImportController], providers: [ImportOpenApiService, ImportOpenApiV2Service], diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts index c33f009e0a..37cb8b79e4 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts @@ -20,6 +20,7 @@ import { AuditScope } from '../../audit/audit-scope'; import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; import { NotificationService } from '../../notification/notification.service'; import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { DEFAULT_VIEWS, DEFAULT_FIELDS } from '../../table/constant'; import { TableOpenApiService } from '../../table/open-api/table-open-api.service'; import { ImportMetricsService } from '../metrics/import-metrics.service'; @@ -63,6 +64,8 @@ export class ImportOpenApiService { private readonly fieldOpenApiService: FieldOpenApiService, private readonly cacheService: CacheService, private readonly audit: AuditScope, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService, @Optional() private readonly importMetrics?: ImportMetricsService ) {} @@ -113,6 +116,7 @@ export class ImportOpenApiService { } async createTableFromImport(baseId: string, importRo: IImportOptionRo, maxRowCount?: number) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); await this.checkImportConcurrencyLimit(); const userId = this.cls.get('user.id'); @@ -265,6 +269,7 @@ export class ImportOpenApiService { maxRowCount?: number, projection?: string[] ) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); await this.checkImportConcurrencyLimit(); const userId = this.cls.get('user.id'); diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts index 7339a7759e..22aef59c62 100644 --- a/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts @@ -27,7 +27,7 @@ import { ClsService } from 'nestjs-cls'; import { z } from 'zod'; import type { IClsStore } from '../../types/cls'; import { ZodValidationPipe } from '../../zod.validation.pipe'; -import { Permissions } from '../auth/decorators/permissions.decorator'; +import { AnyPermissions, Permissions } from '../auth/decorators/permissions.decorator'; import { PermissionGuard } from '../auth/guard/permission.guard'; import { UseV2Feature } from '../canary/decorators/use-v2-feature.decorator'; import { V2FeatureGuard } from '../canary/guards/v2-feature.guard'; @@ -147,6 +147,7 @@ export class IntegrityV2Controller { @Post('table/:tableId/repair-stream') @Permissions('table|update') + @AnyPermissions(['instance|update']) @UseV2Feature(v2SchemaIntegrityFeature) async repairTable( @Param('tableId') tableId: string, diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts index 97be4a32cd..cefdedb191 100644 --- a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts @@ -14,8 +14,10 @@ import { Table, TableId, TableName, + v2CoreTokens, type ITracer, } from '@teable/v2-core'; +import { ok } from 'neverthrow'; import { describe, expect, it, vi, beforeEach } from 'vitest'; import { IntegrityV2Service } from './integrity-v2.service'; @@ -378,7 +380,7 @@ describe('IntegrityV2Service repair telemetry', () => { ); expect(tableRepository.find).toHaveBeenCalledTimes(1); - expect(tableRepository.find.mock.calls[0]?.[2]).toEqual({ state: 'all' }); + expect(tableRepository.find.mock.calls[0]?.[2]).toEqual({ state: 'activeWithPending' }); expect(tableRepository.find.mock.calls[0]?.[1]).toMatchObject({ left: { baseIdValue: foreignBaseId, @@ -393,6 +395,70 @@ describe('IntegrityV2Service repair telemetry', () => { expect(lookupField.hasError().isError()).toBe(false); }); + it('loads active schema state for integrity table targets', async () => { + const tableId = createTableId('i').toString(); + const table = createTable(); + const tableRepository = { + findOne: vi.fn().mockResolvedValue(ok(table)), + find: vi.fn().mockResolvedValue(ok([table])), + }; + const db = {}; + const container = { + resolve: vi.fn((token) => { + if (token === v2CoreTokens.tableRepository) { + return tableRepository; + } + return db; + }), + }; + const service = new IntegrityV2Service( + { getContainerForTable: vi.fn().mockResolvedValue(container) } as never, + { createContext: vi.fn().mockResolvedValue({}) } as never + ); + + await service['resolveSchemaTarget'](tableId, { includeBaseTables: true }); + + expect(tableRepository.findOne).toHaveBeenCalledWith(expect.anything(), expect.anything(), { + state: 'activeWithPending', + }); + expect(tableRepository.find).toHaveBeenCalledWith(expect.anything(), expect.anything(), { + state: 'activeWithPending', + }); + }); + + it('loads active schema state for integrity base targets', async () => { + const baseId = createBaseId('i').toString(); + const table = createTable(); + const tableRepository = { + find: vi.fn().mockResolvedValue(ok([table])), + }; + const baseRepository = { + findOne: vi.fn().mockResolvedValue(ok({})), + }; + const db = {}; + const container = { + resolve: vi.fn((token) => { + if (token === v2CoreTokens.tableRepository) { + return tableRepository; + } + if (token === v2CoreTokens.baseRepository) { + return baseRepository; + } + return db; + }), + }; + const service = new IntegrityV2Service( + { getContainerForBase: vi.fn().mockResolvedValue(container) } as never, + { createContext: vi.fn().mockResolvedValue({}) } as never + ); + + await service['resolveBaseTarget'](baseId); + + expect(tableRepository.find).toHaveBeenCalledWith(expect.anything(), expect.anything(), { + state: 'activeWithPending', + }); + }); + it('marks metadata reference check results as auto repairable', async () => { const service = new IntegrityV2Service({} as never, {} as never); const table = createTable(); diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts index 8c187477e9..8ace9fd6a2 100644 --- a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; import * as Sentry from '@sentry/nestjs'; import type { IV2BaseSchemaIntegrityRepairRo, @@ -11,7 +11,7 @@ import type { IV2SchemaIntegrityRepairCapability, IV2SchemaIntegrityRepairRo, } from '@teable/openapi'; -import { v2DataDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { v2DataDbTokens, v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { checkTableMetaWithTables, createMetaRepairer, @@ -41,6 +41,7 @@ import { type IExecutionContext, type ITracer, type ITableRepository, + type TableQueryState, } from '@teable/v2-core'; import { V2ContainerService } from '../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; @@ -67,9 +68,15 @@ const integrityFailureKindAttribute = 'teable.integrity.failure_kind'; const integrityRuleIdAttribute = 'teable.integrity.rule_id'; const integrityOutcomeAttribute = 'teable.integrity.outcome'; const integrityRequiredAttribute = 'teable.integrity.required'; +// Physical schema integrity should not validate deleted tables or fields. The repository +// hydrates only fields/views with deleted_time IS NULL for activeWithPending. +const schemaIntegrityTableState: TableQueryState = 'activeWithPending'; +const slowIntegrityTableCheckMs = 10_000; @Injectable() export class IntegrityV2Service { + private readonly logger = new Logger(IntegrityV2Service.name); + constructor( private readonly v2ContainerService: V2ContainerService, private readonly v2ContextFactory: V2ExecutionContextFactory @@ -79,11 +86,12 @@ export class IntegrityV2Service { tableId: string, statuses?: IV2SchemaIntegrityFilterStatus[] ): Promise> { - const { table, tables, db, schema } = await this.resolveSchemaTarget(tableId, { + const { table, tables, db, metaDb, schema } = await this.resolveSchemaTarget(tableId, { includeBaseTables: true, }); const checker = createSchemaChecker({ db, + metaDb, introspector: new PostgresSchemaIntrospector(db), schema, }); @@ -95,16 +103,17 @@ export class IntegrityV2Service { tableId: string, repairRo: IV2SchemaIntegrityRepairRo ): Promise> { - const { table, tables, db, schema, context } = await this.resolveSchemaTarget(tableId, { + const { table, tables, db, metaDb, schema, context } = await this.resolveSchemaTarget(tableId, { includeBaseTables: true, }); const repairer = createSchemaRepairer({ db, + metaDb, introspector: new PostgresSchemaIntrospector(db), schema, }); - const metaRepairer = createMetaRepairer({ db }); + const metaRepairer = createMetaRepairer({ db: metaDb }); if (repairRo.fieldId && isMetaRuleId(repairRo.ruleId)) { return this.decorateRepairStream( @@ -186,9 +195,15 @@ export class IntegrityV2Service { baseId: string, statuses?: IV2SchemaIntegrityFilterStatus[] ): Promise> { - const { tables, metaTables, db, schema } = await this.resolveBaseTarget(baseId); + const startedAt = Date.now(); + this.logger.log(`Schema integrity base check target resolving started baseId=${baseId}`); + const { tables, metaTables, db, metaDb, schema } = await this.resolveBaseTarget(baseId); + this.logger.log( + `Schema integrity base check target resolving completed baseId=${baseId} tableCount=${tables.length} metaTableCount=${metaTables.length} elapsedMs=${Date.now() - startedAt}` + ); const checker = createSchemaChecker({ db, + metaDb, introspector: new PostgresSchemaIntrospector(db), schema, }); @@ -200,13 +215,15 @@ export class IntegrityV2Service { baseId: string, repairRo: IV2BaseSchemaIntegrityRepairRo ): Promise> { - const { tables, metaTables, db, schema, context } = await this.resolveBaseTarget(baseId); + const { tables, metaTables, db, metaDb, schema, context } = + await this.resolveBaseTarget(baseId); const repairer = createSchemaRepairer({ db, + metaDb, introspector: new PostgresSchemaIntrospector(db), schema, }); - const metaRepairer = createMetaRepairer({ db }); + const metaRepairer = createMetaRepairer({ db: metaDb }); return this.streamBaseRepairs(tables, metaTables, repairer, metaRepairer, repairRo, { tracer: context.tracer, @@ -232,7 +249,7 @@ export class IntegrityV2Service { const tableResult = await tableRepository.findOne( context, TableByIdSpec.create(parsedTableId.value), - { state: 'all' } + { state: schemaIntegrityTableState } ); if (tableResult.isErr()) { @@ -240,6 +257,7 @@ export class IntegrityV2Service { } const db = container.resolve(v2DataDbTokens.db); + const metaDb = container.resolve(v2MetaDbTokens.db); const table = tableResult.value; let tables: ReadonlyArray = [table]; @@ -247,7 +265,7 @@ export class IntegrityV2Service { const tablesResult = await tableRepository.find( context, TableByBaseIdSpec.create(table.baseId()), - { state: 'all' } + { state: schemaIntegrityTableState } ); if (tablesResult.isErr()) { @@ -261,6 +279,7 @@ export class IntegrityV2Service { table, tables, db, + metaDb, schema: table.baseId().toString(), context, }; @@ -289,7 +308,7 @@ export class IntegrityV2Service { const tablesResult = await tableRepository.find( context, TableByBaseIdSpec.create(parsedBaseId.value), - { state: 'all' } + { state: schemaIntegrityTableState } ); if (tablesResult.isErr()) { @@ -297,6 +316,7 @@ export class IntegrityV2Service { } const db = container.resolve(v2DataDbTokens.db); + const metaDb = container.resolve(v2MetaDbTokens.db); const metaTables = await this.loadReferencedForeignTables( tablesResult.value, tableRepository, @@ -310,6 +330,7 @@ export class IntegrityV2Service { tables, metaTables, db, + metaDb, schema: parsedBaseId.value.toString(), context, }; @@ -478,7 +499,9 @@ export class IntegrityV2Service { throw new HttpException(specResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); } - const tablesResult = await tableRepository.find(context, specResult.value, { state: 'all' }); + const tablesResult = await tableRepository.find(context, specResult.value, { + state: schemaIntegrityTableState, + }); if (tablesResult.isErr()) { throw new HttpException(tablesResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); } @@ -494,12 +517,28 @@ export class IntegrityV2Service { checker: ReturnType, statuses?: IV2SchemaIntegrityFilterStatus[] ): AsyncGenerator { + const baseId = table.baseId().toString(); + const tableId = table.id().toString(); + const tableName = table.name().toString(); + const schemaStartedAt = Date.now(); + this.logger.log( + `Schema integrity table schema check started baseId=${baseId} tableId=${tableId} tableName=${tableName}` + ); yield* this.decorateCheckStream(table, checker.checkTable(table), statuses); + const schemaElapsedMs = Date.now() - schemaStartedAt; + this.logIntegrityPhaseCompleted('schema', baseId, tableId, tableName, schemaElapsedMs); + + const metaStartedAt = Date.now(); + this.logger.log( + `Schema integrity table meta check started baseId=${baseId} tableId=${tableId} tableName=${tableName}` + ); yield* this.decorateMetaCheckStream( table, checkTableMetaWithTables(table, table.baseId(), allTables), statuses ); + const metaElapsedMs = Date.now() - metaStartedAt; + this.logIntegrityPhaseCompleted('meta', baseId, tableId, tableName, metaElapsedMs); } private async *streamBaseChecks( @@ -508,9 +547,25 @@ export class IntegrityV2Service { checker: ReturnType, statuses?: IV2SchemaIntegrityFilterStatus[] ): AsyncGenerator { + const baseId = tables[0]?.baseId().toString() ?? 'unknown'; + const startedAt = Date.now(); + this.logger.log( + `Schema integrity base check stream started baseId=${baseId} tableCount=${tables.length} metaTableCount=${metaTables.length}` + ); + let checkedTables = 0; for (const table of tables) { + const tableStartedAt = Date.now(); + this.logger.log( + `Schema integrity base check table started baseId=${baseId} tableId=${table.id().toString()} tableName=${table.name().toString()} index=${checkedTables + 1}/${tables.length}` + ); yield* this.streamTableChecks(table, metaTables, checker, statuses); + checkedTables += 1; + const tableElapsedMs = Date.now() - tableStartedAt; + this.logIntegrityTableCompleted(baseId, table, checkedTables, tables.length, tableElapsedMs); } + this.logger.log( + `Schema integrity base check stream completed baseId=${baseId} tableCount=${tables.length} elapsedMs=${Date.now() - startedAt}` + ); } private async *streamBaseRepairs( @@ -789,6 +844,36 @@ export class IntegrityV2Service { return `${table.id().toString()}:${id}`; } + private logIntegrityPhaseCompleted( + phase: 'schema' | 'meta', + baseId: string, + tableId: string, + tableName: string, + elapsedMs: number + ) { + const message = `Schema integrity table ${phase} check completed baseId=${baseId} tableId=${tableId} tableName=${tableName} elapsedMs=${elapsedMs}`; + if (elapsedMs >= slowIntegrityTableCheckMs) { + this.logger.warn(message); + return; + } + this.logger.log(message); + } + + private logIntegrityTableCompleted( + baseId: string, + table: Table, + checkedTables: number, + tableCount: number, + elapsedMs: number + ) { + const message = `Schema integrity base check table completed baseId=${baseId} tableId=${table.id().toString()} tableName=${table.name().toString()} index=${checkedTables}/${tableCount} elapsedMs=${elapsedMs}`; + if (elapsedMs >= slowIntegrityTableCheckMs) { + this.logger.warn(message); + return; + } + this.logger.log(message); + } + private toMutableDetails(details?: SchemaRepairResult['details']) { return details ? { diff --git a/apps/nestjs-backend/src/features/invitation/invitation.service.ts b/apps/nestjs-backend/src/features/invitation/invitation.service.ts index b490c717db..24d6d04c5b 100644 --- a/apps/nestjs-backend/src/features/invitation/invitation.service.ts +++ b/apps/nestjs-backend/src/features/invitation/invitation.service.ts @@ -81,7 +81,26 @@ export class InvitationService { @Audit({ action: Events.INVITATION_EMAIL_SEND, resourceId: (input: { resourceId: string }) => input.resourceId, - params: (input: { resourceType: CollaboratorType }) => ({ resourceType: input.resourceType }), + // Capture the inviter's user.id at decorator-resolve time (before the method + // runs). Inviting a brand-new email path runs `userService.createUser`, which + // mutates CLS user.id via runWith(cls.get(), ...) — that bleeds into the + // outer scope, and by the time the audit listener reads cls.get('user.id') + // it would see the invitee's id instead of the inviter's. Resolving userId + // up front pins the row to the inviter. + userId: (_input, ctx) => ctx.cls.get('user.id'), + params: (input: { + resourceId: string; + resourceType: CollaboratorType; + role: IRole; + emails: string[]; + }) => ({ + resourceType: input.resourceType, + role: input.role, + emails: input.emails, + ...(input.resourceType === CollaboratorType.Base + ? { baseId: input.resourceId } + : { spaceId: input.resourceId }), + }), emit: (_result: unknown, input: { emails: string[] }) => ({ emailCount: input.emails.length }), }) private async emailInvitation({ @@ -246,8 +265,14 @@ export class InvitationService { @Audit({ action: Events.INVITATION_LINK_CREATE, resourceId: (input: { resourceId: string }) => input.resourceId, - params: (input: { resourceType: CollaboratorType }) => ({ resourceType: input.resourceType }), - emit: true, + params: (input: { resourceId: string; resourceType: CollaboratorType; role: IRole }) => ({ + resourceType: input.resourceType, + role: input.role, + ...(input.resourceType === CollaboratorType.Base + ? { baseId: input.resourceId } + : { spaceId: input.resourceId }), + }), + emit: (result: ItemSpaceInvitationLinkVo) => ({ invitationId: result.invitationId }), }) async generateInvitationLink({ role, diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts index 4e4304f1d2..4aedc6458b 100644 --- a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts @@ -1,8 +1,14 @@ -import { Body, Controller, Post } from '@nestjs/common'; +import { Body, Controller, Param, Post } from '@nestjs/common'; import { HttpErrorCode } from '@teable/core'; -import { ITestMailTransportConfigRo, testMailTransportConfigRoSchema } from '@teable/openapi'; +import type { ISendEmailRo, ISendEmailVo } from '@teable/openapi'; +import { + ITestMailTransportConfigRo, + sendEmailRoSchema, + testMailTransportConfigRoSchema, +} from '@teable/openapi'; import { CustomHttpException } from '../../../custom.exception'; import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; import { MailSenderOpenApiService } from './mail-sender-open-api.service'; @Controller('api/mail-sender') @@ -31,4 +37,14 @@ export class MailSenderOpenApiController { ); } } + + @Post(':baseId/send') + @Permissions('base|update') + async sendEmail( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(sendEmailRoSchema)) + sendEmailRo: ISendEmailRo + ): Promise { + return this.mailSenderOpenApiService.sendEmail(baseId, sendEmailRo); + } } diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts index 2f8eec259d..c232726476 100644 --- a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts @@ -1,14 +1,25 @@ import { Injectable } from '@nestjs/common'; -import type { ITestMailTransportConfigRo } from '@teable/openapi'; +import type { ISendEmailRo, ISendEmailVo, ITestMailTransportConfigRo } from '@teable/openapi'; +import { MailBodyType, MailTransporterType, MailType, toEmailArray } from '@teable/openapi'; +import MarkdownIt from 'markdown-it'; import { createTransport } from 'nodemailer'; import { IMailConfig, MailConfig } from '../../../configs/mail.config'; +import { type ISendMailOptions } from '../mail-helpers'; import { MailSenderService } from '../mail-sender.service'; +const markdownIt = MarkdownIt({ html: true, breaks: true }); + +type ISendEmailExtras = { + footerHtml?: string; + list?: ISendMailOptions['list']; + headers?: ISendMailOptions['headers']; +}; + @Injectable() export class MailSenderOpenApiService { constructor( - private readonly mailSenderService: MailSenderService, - @MailConfig() private readonly mailConfig: IMailConfig + protected readonly mailSenderService: MailSenderService, + @MailConfig() protected readonly mailConfig: IMailConfig ) {} async testTransportConfig(testMailTransportConfigRo: ITestMailTransportConfigRo): Promise { @@ -19,4 +30,35 @@ export class MailSenderOpenApiService { const option = await this.mailSenderService.sendTestEmailOptions({ message }); await this.mailSenderService.sendMailByConfig({ to, ...option }, transportConfig); } + + async sendEmail(_baseId: string, ro: ISendEmailRo & ISendEmailExtras): Promise { + const { subject, body, bodyType, replyTo, headers, smtp, footerHtml, list } = ro; + + const rendered = bodyType === MailBodyType.Markdown ? markdownIt.render(body) : body; + const html = rendered + (footerHtml ?? ''); + + const result = await this.mailSenderService.sendMail( + { + to: toEmailArray(ro.to), + cc: toEmailArray(ro.cc), + bcc: toEmailArray(ro.bcc), + subject, + html, + headers, + replyTo, + list, + }, + { + shouldThrow: true, + type: MailType.ApiSendEmailAction, + transportConfig: smtp, + transporterName: MailTransporterType.Automation, + } + ); + + return { + success: !!result, + message: result ? 'Email sent successfully' : 'Failed to send email', + }; + } } diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts index 706697ca35..3230963bfa 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts @@ -1,16 +1,18 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable sonarjs/cognitive-complexity */ -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; +import { Injectable, HttpException, HttpStatus, Optional } from '@nestjs/common'; import { trace } from '@opentelemetry/api'; import { CellFormat, CellValueType, FieldKeyType, FieldType, + HttpErrorCode, TimeFormatting, formatDateToString, isMeTag, parseClipboardText, + type IAttachmentItem, type IDatetimeFormatting, type IFieldVo, type IFilter, @@ -26,13 +28,15 @@ import type { IRecord, ICreateRecordsVo, IGetRecordsRo, + ICreateRecordsRo, + IUpdateRecordsRo, IPasteRo, IPasteVo, IRangesRo, IRecordsVo, IRecordInsertOrderRo, } from '@teable/openapi'; -import { ICreateRecordsRo, IUpdateRecordsRo, RangeType } from '@teable/openapi'; +import { RangeType } from '@teable/openapi'; import { mapDomainErrorToHttpError, mapDomainErrorToHttpStatus } from '@teable/v2-contract-http'; import { executeCreateRecordsEndpoint, @@ -77,6 +81,7 @@ import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exc import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { IClsStore } from '../../../types/cls'; import { AggregationService } from '../../aggregation/aggregation.service'; +import { AttachmentsService } from '../../attachments/attachments.service'; import { AuditScope } from '../../audit/audit-scope'; import { FieldService } from '../../field/field.service'; import type { IFieldInstance } from '../../field/model/factory'; @@ -122,7 +127,8 @@ export class RecordOpenApiV2Service { private readonly recordPermissionService: RecordPermissionService, private readonly aggregationService: AggregationService, private readonly dataDbClientManager: DataDbClientManager, - private readonly audit: AuditScope + private readonly audit: AuditScope, + @Optional() private readonly attachmentsService?: AttachmentsService ) {} private throwV2Error( @@ -650,7 +656,13 @@ export class RecordOpenApiV2Service { throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); } - async updateRecords(tableId: string, updateRecordsRo: IUpdateRecordsRo): Promise { + async updateRecords( + tableId: string, + updateRecordsRo: IUpdateRecordsRo, + options?: { + configureContext?: (context: IExecutionContext) => void; + } + ): Promise { const rawRecords = updateRecordsRo.records ?? []; const records = this.mergeDuplicateRecordUpdates(rawRecords); const recordIds = records.map((record) => record.id); @@ -680,6 +692,7 @@ export class RecordOpenApiV2Service { const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); const context = await this.v2ContextFactory.createContext(container); + options?.configureContext?.(context); const updateResult = await executeUpdateRecordsEndpoint( context, { @@ -711,6 +724,100 @@ export class RecordOpenApiV2Service { return updateResult.body.data.records; } + private async getValidateAttachmentRecord(tableId: string, recordId: string, fieldId: string) { + const field = await this.fieldService.getField(tableId, fieldId); + + if (field.type !== FieldType.Attachment) { + throw new CustomHttpException('Field is not an attachment', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.notAttachment', + }, + }); + } + + if (field.isComputed) { + throw new CustomHttpException('Field is computed', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.isComputed', + }, + }); + } + + const recordData = await this.recordService.getRecordsById(tableId, [recordId]); + const record = recordData.records[0]; + if (!record) { + throw new CustomHttpException(`Record ${recordId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.record.notFound', + }, + }); + } + return record; + } + + async uploadAttachment( + tableId: string, + recordId: string, + fieldId: string, + file?: Express.Multer.File, + fileUrl?: string + ) { + if (!file && !fileUrl) { + throw new CustomHttpException('No file or URL provided', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.record.noFileOrUrlProvided', + }, + }); + } + + if (!this.attachmentsService) { + throw new CustomHttpException(internalServerError, HttpErrorCode.INTERNAL_SERVER_ERROR); + } + + const record = await this.getValidateAttachmentRecord(tableId, recordId, fieldId); + const attachmentItem = file + ? await this.attachmentsService.uploadFile(file) + : await this.attachmentsService.uploadFromUrl(fileUrl as string); + + return await this.updateRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [fieldId]: ((record.fields[fieldId] || []) as IAttachmentItem[]).concat(attachmentItem), + }, + }, + }); + } + + async insertAttachment( + tableId: string, + recordId: string, + fieldId: string, + attachments: IAttachmentItem[], + anchorId?: string + ) { + if (!attachments.length) { + throw new CustomHttpException('No attachments provided', HttpErrorCode.VALIDATION_ERROR); + } + + const record = await this.getValidateAttachmentRecord(tableId, recordId, fieldId); + const current = (record.fields[fieldId] || []) as IAttachmentItem[]; + const anchorIndex = anchorId ? current.findIndex((item) => item.id === anchorId) : -1; + const next = + anchorIndex >= 0 + ? [...current.slice(0, anchorIndex + 1), ...attachments, ...current.slice(anchorIndex + 1)] + : current.concat(attachments); + + return await this.updateRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [fieldId]: next, + }, + }, + }); + } + async createRecords( tableId: string, createRecordsRo: ICreateRecordsRo, diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts index de4af4478c..5d773e1b7c 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts @@ -166,6 +166,7 @@ export class RecordOpenApiController { ); } + @UseV2Feature('updateRecord') @Permissions('record|update') @Post(':recordId/:fieldId/uploadAttachment') @UseInterceptors(FileInterceptor('file')) @@ -185,6 +186,16 @@ export class RecordOpenApiController { }, }); + if (this.cls.get('useV2')) { + return await this.recordOpenApiV2Service.uploadAttachment( + tableId, + recordId, + fieldId, + file, + fileUrl + ); + } + return await this.recordOpenApiService.uploadAttachment( tableId, recordId, @@ -194,6 +205,7 @@ export class RecordOpenApiController { ); } + @UseV2Feature('updateRecord') @Permissions('record|update') @Post(':recordId/:fieldId/insertAttachment') async insertAttachment( @@ -211,6 +223,16 @@ export class RecordOpenApiController { }, }); + if (this.cls.get('useV2')) { + return await this.recordOpenApiV2Service.insertAttachment( + tableId, + recordId, + fieldId, + body.attachments, + body.anchorId + ); + } + return await this.recordOpenApiService.insertAttachment( tableId, recordId, diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts index 54f5edeaa0..db57e9f38d 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts @@ -736,7 +736,8 @@ export class FieldCteVisitor implements IFieldVisitor { state: IMutableQueryBuilderState | undefined, private readonly dialect: IRecordQueryDialectProvider, projection?: string[], - expandFormulaReferences: boolean = true + expandFormulaReferences: boolean = true, + private readonly preferStoredLookupFields: boolean = false ) { this.state = state ?? new RecordQueryBuilderManager('table'); this._table = tables.mustGetEntryTable(); @@ -752,6 +753,15 @@ export class FieldCteVisitor implements IFieldVisitor { return this.state.getFieldCteMap(); } + private shouldUseStoredLookupField(field: FieldCore): boolean { + return ( + this.preferStoredLookupFields && + (field.isLookup || + field.type === FieldType.Rollup || + field.type === FieldType.ConditionalRollup) + ); + } + private unwrapSelectName(selection: IFieldSelectName | string): string { return typeof selection === 'string' ? selection : selection.toQuery(); } @@ -948,7 +958,7 @@ export class FieldCteVisitor implements IFieldVisitor { } } - private getBlockedLinkFieldIds(currentLinkFieldId: string): ReadonlySet | undefined { + private getBlockedLinkFieldIds(_currentLinkFieldId: string): ReadonlySet | undefined { if (!this.linkCteGenerationStack.size) { return undefined; } @@ -1354,10 +1364,6 @@ export class FieldCteVisitor implements IFieldVisitor { foreignAliasUsed, selectVisitor ); - const normalizedExpression = this.coerceConditionalLookupTargetExpression( - rawExpression, - targetField - ); const formattingVisitor = new FieldFormattingVisitor(rawExpression, this.dialect); const formattedExpression = targetField.accept(formattingVisitor); @@ -1944,6 +1950,10 @@ export class FieldCteVisitor implements IFieldVisitor { // This allows selecting lookup/rollup values even when the link fields themselves // are not part of the projection. for (const field of list) { + if (this.shouldUseStoredLookupField(field)) { + continue; + } + const linkFields = !this.expandFormulaReferences && field.type === FieldType.Formula ? [] @@ -1969,6 +1979,9 @@ export class FieldCteVisitor implements IFieldVisitor { } for (const field of list) { + if (this.shouldUseStoredLookupField(field)) { + continue; + } field.accept(this); } } diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts index cd222cd5a8..1cecbc3859 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts @@ -28,8 +28,11 @@ import { DbFieldType, FieldType, isLinkLookupOptions, DriverClient } from '@teab // no driver-specific logic here; use dialect for differences import type { Knex } from 'knex'; import type { IDbProvider } from '../../../db-provider/db.provider.interface'; -import { AUTO_NUMBER_FIELD_NAME } from '../../field/constant'; -import { isSystemUserField } from '../../field/fields-utils'; +import { + AUTO_NUMBER_FIELD_NAME, + CREATED_BY_FIELD_NAME, + LAST_MODIFIED_BY_FIELD_NAME, +} from '../../field/constant'; import type { IFieldSelectName } from './field-select.type'; import type { IRecordSelectionMap, @@ -64,7 +67,8 @@ export class FieldSelectVisitor implements IFieldVisitor { private readonly preferRawFieldReferences: boolean = false, private readonly blockedLinkFieldIds?: ReadonlySet, private readonly readyLinkFieldIds?: ReadonlySet, - private readonly currentLinkFieldId?: string + private readonly currentLinkFieldId?: string, + private readonly preferStoredLookupFields: boolean = false ) {} private get tableAlias() { @@ -174,6 +178,18 @@ export class FieldSelectVisitor implements IFieldVisitor { return selector; } + private selectSystemUserSnapshot( + field: FieldCore, + systemColumnName: typeof CREATED_BY_FIELD_NAME | typeof LAST_MODIFIED_BY_FIELD_NAME + ): IFieldSelectName { + const snapshotRef = this.getColumnSelector(field) as string; + const alias = this.tableAlias; + const idFallbackRef = alias ? `"${alias}"."${systemColumnName}"` : `"${systemColumnName}"`; + const expr = this.dialect.buildUserJsonObjectFromSnapshot(snapshotRef, idFallbackRef); + this.state.setSelection(field.id, expr); + return this.qb.client.raw(expr); + } + // Typed NULL generation is delegated to the dialect implementation /** @@ -183,16 +199,16 @@ export class FieldSelectVisitor implements IFieldVisitor { private checkAndSelectLookupField(field: FieldCore): IFieldSelectName { // Check if this is a Lookup field if (field.isLookup) { + if (this.preferStoredLookupFields) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + const fieldCteMap = this.state.getFieldCteMap(); // Lookup has no standard column in base table. // When building from a materialized view, fallback to the view's column. if (this.shouldSelectRaw()) { - if (isSystemUserField(field) && !field.isLookup) { - const columnSelector = this.getColumnSelector(field) as string; - const expr = this.dialect.buildUserJsonObjectById(columnSelector); - this.state.setSelection(field.id, expr); - return this.qb.client.raw(expr); - } const columnSelector = this.getColumnSelector(field); this.state.setSelection(field.id, columnSelector); return columnSelector; @@ -460,6 +476,12 @@ export class FieldSelectVisitor implements IFieldVisitor { } visitRollupField(field: RollupFieldCore): IFieldSelectName { + if (this.preferStoredLookupFields) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + if (this.shouldSelectRaw()) { // In view context, select the view column directly const columnSelector = this.getColumnSelector(field); @@ -533,6 +555,12 @@ export class FieldSelectVisitor implements IFieldVisitor { } visitConditionalRollupField(field: ConditionalRollupFieldCore): IFieldSelectName { + if (this.preferStoredLookupFields) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + if (field.isLookup) { return this.checkAndSelectLookupField(field); } @@ -631,12 +659,8 @@ export class FieldSelectVisitor implements IFieldVisitor { if (field.isLookup) { return this.checkAndSelectLookupField(field); } - // Build JSON with user info from system column __created_by - const alias = this.tableAlias; - const idRef = alias ? `"${alias}"."__created_by"` : `"__created_by"`; - const expr = this.dialect.buildUserJsonObjectById(idRef); - this.state.setSelection(field.id, expr); - return this.qb.client.raw(expr); + + return this.selectSystemUserSnapshot(field, CREATED_BY_FIELD_NAME); } visitLastModifiedByField(field: LastModifiedByFieldCore): IFieldSelectName { @@ -646,12 +670,7 @@ export class FieldSelectVisitor implements IFieldVisitor { const trackAll = field.isTrackAll(); if (trackAll) { - // Build JSON with user info from system column __last_modified_by - const alias = this.tableAlias; - const idRef = alias ? `"${alias}"."__last_modified_by"` : `"__last_modified_by"`; - const expr = this.dialect.buildUserJsonObjectById(idRef); - this.state.setSelection(field.id, expr); - return this.qb.client.raw(expr); + return this.selectSystemUserSnapshot(field, LAST_MODIFIED_BY_FIELD_NAME); } return this.checkAndSelectLookupField(field); diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts index 9324358393..885f524235 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts @@ -46,6 +46,36 @@ describe('PgRecordQueryDialect#linkExtractTitles', () => { }); }); +describe('PgRecordQueryDialect user snapshots', () => { + const dialect = new PgRecordQueryDialect({} as unknown as Knex); + + it('builds CreatedBy/LastModifiedBy display JSON without users joins', () => { + const sql = dialect.buildUserJsonObjectFromSnapshot( + '"t"."fld_created_by"', + '"t"."__created_by"' + ); + + expect(sql).toContain('to_jsonb("t"."fld_created_by")'); + expect(sql).toContain('"t"."__created_by"'); + expect(sql).toContain('jsonb_build_object'); + expect(sql).not.toContain('users'); + expect(sql).not.toContain('public.users'); + }); + + it('builds formula display text from snapshot with system id fallback', () => { + const sql = dialect.userTitleFromSnapshot( + '"t"."fld_last_modified_by"', + '"t"."__last_modified_by"' + ); + + expect(sql).toContain('to_jsonb("t"."fld_last_modified_by")'); + expect(sql).toContain("->>'title'"); + expect(sql).toContain('"t"."__last_modified_by"'); + expect(sql).not.toContain('users'); + expect(sql).not.toContain('public.users'); + }); +}); + describe('PgRecordQueryDialect#coerceToNumericForCompare', () => { const dialect = new PgRecordQueryDialect({} as unknown as Knex); diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts index 4134964d57..66bb397eb1 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts @@ -294,17 +294,37 @@ export class PgRecordQueryDialect implements IRecordQueryDialectProvider { return `(${selectionSql}->>'title')`; } - selectUserNameById(idRef: string): string { - return `(SELECT u.name FROM users u WHERE u.id = ${idRef})`; + private userSnapshotJson(snapshotRef: string): string { + const snapshotJson = `to_jsonb(${snapshotRef})`; + return `(CASE + WHEN ${snapshotRef} IS NULL THEN NULL::jsonb + WHEN jsonb_typeof(${snapshotJson}) = 'object' THEN ${snapshotJson} + WHEN jsonb_typeof(${snapshotJson}) = 'string' THEN jsonb_build_object('id', ${snapshotRef}, 'title', ${snapshotRef}) + ELSE jsonb_build_object('id', ${snapshotRef}::text, 'title', ${snapshotRef}::text) + END)`; } - buildUserJsonObjectById(idRef: string): string { - return `( - SELECT jsonb_build_object('id', u.id, 'title', u.name, 'email', u.email) || - CASE WHEN u.is_system = true THEN jsonb_build_object('isSystem', true) ELSE '{}'::jsonb END - FROM users u - WHERE u.id = ${idRef} - )`; + userTitleFromSnapshot(snapshotRef: string, idFallbackRef?: string): string { + const snapshotJson = this.userSnapshotJson(snapshotRef); + const fallback = idFallbackRef ?? `${snapshotJson}->>'id'`; + return `COALESCE(${snapshotJson}->>'title', ${snapshotJson}->>'name', ${fallback})`; + } + + buildUserJsonObjectFromSnapshot(snapshotRef: string, idFallbackRef?: string): string { + const snapshotJson = this.userSnapshotJson(snapshotRef); + const fallback = idFallbackRef ?? `${snapshotJson}->>'id'`; + return `jsonb_strip_nulls( + CASE + WHEN ${snapshotJson} IS NULL AND ${idFallbackRef ? `${idFallbackRef} IS NULL` : 'TRUE'} THEN NULL::jsonb + ELSE COALESCE( + ${snapshotJson}, + jsonb_build_object('id', ${fallback}, 'title', ${fallback}) + ) || jsonb_build_object( + 'id', COALESCE(${snapshotJson}->>'id', ${fallback}), + 'title', COALESCE(${snapshotJson}->>'title', ${snapshotJson}->>'name', ${fallback}) + ) + END + )`; } flattenLookupCteValue( diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts index eeaf81580f..7105fba7cf 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts @@ -49,6 +49,12 @@ export interface ICreateRecordQueryBuilderOptions { * Typically used alongside rawProjection when the consumer needs source values (e.g., jsonb) rather than formatted text. */ preferRawFieldReferences?: boolean; + /** + * When true, lookup-like computed fields use their persisted DB columns + * instead of expanding link/conditional CTEs. Intended for read paths where + * persisted computed values are the query source of truth. + */ + preferStoredLookupFields?: boolean; /** * Optional list of record IDs to restrict the query to before generating CTEs. * Useful when the caller intends to apply a final WHERE IN "__id" (...) filter anyway. @@ -90,6 +96,11 @@ export interface ICreateRecordAggregateBuilderOptions { currentUserId?: string; /** Optional projection to minimize CTE/select */ projection?: string[]; + /** + * When true, lookup-like computed fields use their persisted DB columns + * instead of expanding link/conditional CTEs. + */ + preferStoredLookupFields?: boolean; useQueryModel?: boolean; /** * Optional list of record IDs to restrict the query to before generating CTEs. diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts index f4d859a3af..b36ed5e26a 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts @@ -157,7 +157,8 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { tables, state, options.projection, - options.preferRawFieldReferences ?? false + options.preferRawFieldReferences ?? false, + options.preferStoredLookupFields ?? false ); } @@ -181,6 +182,7 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { options: ICreateRecordQueryBuilderOptions ): Promise<{ qb: Knex.QueryBuilder; alias: string; selectionMap: IReadonlyRecordSelectionMap }> { const { tableId, filter, sort, currentUserId, restrictRecordIds } = options; + const preferStoredLookupFields = options.preferStoredLookupFields ?? !options.rawProjection; const { qb, alias, table, state } = await this.createQueryBuilder(from, tableId, { builder: options.builder, useQueryModel: options.useQueryModel, @@ -196,6 +198,7 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { hasSearch: options.hasSearch, restrictRecordIds, preferRawFieldReferences: options.preferRawFieldReferences, + preferStoredLookupFields, }); this.buildSelect( @@ -204,7 +207,8 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { state, options.projection, options.rawProjection, - options.preferRawFieldReferences ?? false + options.preferRawFieldReferences ?? false, + preferStoredLookupFields ); // Selection map collected as fields are visited. @@ -237,6 +241,7 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { defaultOrderField, limit, offset, + preferStoredLookupFields = true, } = options; const usePaginatedRange = limit !== undefined; // The tableCache path skips applyBasePaginationIfNeeded, which would silently @@ -255,9 +260,10 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { limit, offset, paginationMode: usePaginatedRange ? 'full' : undefined, + preferStoredLookupFields, }); - this.buildAggregateSelect(qb, table, state); + this.buildAggregateSelect(qb, table, state, options.projection, preferStoredLookupFields); const selectionMap = state.getSelectionMap(); if (filter) { @@ -305,7 +311,8 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { tables: Tables | undefined, state: IMutableQueryBuilderState, projection?: string[], - preferRawFieldReferences: boolean = false + preferRawFieldReferences: boolean = false, + preferStoredLookupFields: boolean = false ): void { if (!tables) { return; @@ -317,7 +324,8 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { state, this.dialect, projection, - !preferRawFieldReferences + !preferRawFieldReferences, + preferStoredLookupFields ); visitor.build(); } @@ -628,7 +636,8 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { state: IMutableQueryBuilderState, projection?: string[], rawProjection: boolean = false, - preferRawFieldReferences: boolean = false + preferRawFieldReferences: boolean = false, + preferStoredLookupFields: boolean = false ): this { const readyLinkFieldIds = this.getReadyLinkFieldIds(state); const visitor = new FieldSelectVisitor( @@ -641,7 +650,9 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { rawProjection, preferRawFieldReferences, undefined, - readyLinkFieldIds + readyLinkFieldIds, + undefined, + preferStoredLookupFields ); const alias = getTableAliasFromTable(table); @@ -672,7 +683,9 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { private buildAggregateSelect( qb: Knex.QueryBuilder, table: TableDomain, - state: IMutableQueryBuilderState + state: IMutableQueryBuilderState, + projection?: string[], + preferStoredLookupFields: boolean = false ): this { const readyLinkFieldIds = this.getReadyLinkFieldIds(state); const visitor = new FieldSelectVisitor( @@ -685,11 +698,15 @@ export class RecordQueryBuilderService implements IRecordQueryBuilder { false, false, undefined, - readyLinkFieldIds + readyLinkFieldIds, + undefined, + preferStoredLookupFields ); - // Add field-specific selections using visitor pattern - for (const field of table.fields.ordered) { + // Add field-specific selections using visitor pattern. Aggregations only need + // selections for fields referenced by aggregation/group/search/filter callers. + const orderedFields = getOrderedFieldsByProjection(table, projection, true) as FieldCore[]; + for (const field of orderedFields) { field.accept(visitor); } diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts index 20a1730959..f639d92912 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts @@ -139,24 +139,16 @@ export interface IRecordQueryDialectProvider { jsonTitleFromExpr(selectionSql: string): string; /** - * Subquery snippet to select user name by id. - * @example - * ```ts - * dialect.selectUserNameById('"t"."__created_by"') - * // PG: (SELECT u.name FROM users u WHERE u.id = "t"."__created_by") - * ``` + * Extract a display title from a stored user JSON snapshot. + * Falls back to the system user id when the snapshot is missing or still scalar. */ - selectUserNameById(idRef: string): string; + userTitleFromSnapshot(snapshotRef: string, idFallbackRef?: string): string; /** - * Build a JSON object for system user fields: { id, title, email }. - * @example - * ```ts - * dialect.buildUserJsonObjectById('"t"."__created_by"') - * // PG: (SELECT jsonb_build_object('id', u.id, 'title', u.name, 'email', u.email) FROM users u WHERE u.id = "t"."__created_by") - * ``` + * Build a user JSON object from a stored user JSON snapshot. + * Falls back to a minimal { id, title } object from the system user id when missing. */ - buildUserJsonObjectById(idRef: string): string; + buildUserJsonObjectFromSnapshot(snapshotRef: string, idFallbackRef?: string): string; // Lookup CTE helpers diff --git a/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts index 7d38463274..9097d5f628 100644 --- a/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts +++ b/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts @@ -2254,17 +2254,22 @@ export class SelectColumnSqlConversionVisitor extends BaseSqlConversionVisitor { + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + const createService = { + multipleCreateRecords: vi.fn(), + createRecords: vi.fn(), + createRecordsOnlySql: vi.fn(), + }; + const updateService = { + updateRecords: vi.fn(), + simpleUpdateRecords: vi.fn(), + }; + const deleteService = { + deleteRecord: vi.fn(), + deleteRecords: vi.fn(), + }; + const duplicateService = { + duplicateRecord: vi.fn(), + }; + const tableDomainQueryService = { + getTableDomainById: vi.fn(), + }; + const migrationGuard = { + assertTableWritable: vi.fn(), + }; + + const service = () => + new RecordModifyService( + createService as never, + updateService as never, + deleteService as never, + duplicateService as never, + tableDomainQueryService as never, + migrationGuard as never + ); + + beforeEach(() => { + vi.clearAllMocks(); + migrationGuard.assertTableWritable.mockRejectedValue(freezeError); + }); + + it('rejects record create before any table metadata or write work when the space is migrating', async () => { + await expect(service().multipleCreateRecords('tblxxx', { records: [] })).rejects.toBe( + freezeError + ); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(createService.multipleCreateRecords).not.toHaveBeenCalled(); + expect(tableDomainQueryService.getTableDomainById).not.toHaveBeenCalled(); + }); + + it('rejects record update/delete/duplicate before delegating when the space is migrating', async () => { + await expect(service().updateRecords('tblxxx', { records: [] })).rejects.toBe(freezeError); + await expect(service().deleteRecords('tblxxx', ['recxxx'])).rejects.toBe(freezeError); + await expect( + service().duplicateRecord('tblxxx', 'recxxx', { + viewId: 'viwxxx', + anchorId: 'recanchor', + position: 'after', + }) + ).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledTimes(3); + expect(updateService.updateRecords).not.toHaveBeenCalled(); + expect(deleteService.deleteRecords).not.toHaveBeenCalled(); + expect(duplicateService.duplicateRecord).not.toHaveBeenCalled(); + }); + + it('allows non-migrating tables to continue through the normal write path', async () => { + migrationGuard.assertTableWritable.mockResolvedValue(undefined); + createService.multipleCreateRecords.mockResolvedValue({ records: [] }); + + await expect(service().multipleCreateRecords('tblother', { records: [] })).resolves.toEqual({ + records: [], + }); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblother'); + expect(createService.multipleCreateRecords).toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts index ec3d9cd065..fcec72bb1d 100644 --- a/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts +++ b/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import { FieldKeyType } from '@teable/core'; import type { IMakeOptional } from '@teable/core'; import type { @@ -7,6 +7,7 @@ import type { ICreateRecordsVo, IRecordInsertOrderRo, } from '@teable/openapi'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { TableDomainQueryService } from '../../table-domain'; import type { IRecordInnerRo } from '../record.service'; import type { IUpdateRecordsInternalRo } from '../type'; @@ -22,18 +23,26 @@ export class RecordModifyService { private readonly updateService: RecordUpdateService, private readonly deleteService: RecordDeleteService, private readonly duplicateService: RecordDuplicateService, - private readonly tableDomainQueryService: TableDomainQueryService + private readonly tableDomainQueryService: TableDomainQueryService, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + async updateRecords( tableId: string, updateRecordsRo: IUpdateRecordsInternalRo, windowId?: string ) { + await this.assertTableWritable(tableId); return this.updateService.updateRecords(tableId, updateRecordsRo, windowId); } async simpleUpdateRecords(tableId: string, updateRecordsRo: IUpdateRecordsInternalRo) { + await this.assertTableWritable(tableId); return this.updateService.simpleUpdateRecords(tableId, updateRecordsRo); } @@ -42,6 +51,7 @@ export class RecordModifyService { createRecordsRo: ICreateRecordsRo, ignoreMissingFields: boolean = false ): Promise { + await this.assertTableWritable(tableId); return this.createService.multipleCreateRecords(tableId, createRecordsRo, ignoreMissingFields); } @@ -51,6 +61,7 @@ export class RecordModifyService { fieldKeyType?: FieldKeyType, projection?: string[] ): Promise { + await this.assertTableWritable(tableId); const table = await this.tableDomainQueryService.getTableDomainById(tableId); return this.createService.createRecords( table, @@ -61,14 +72,17 @@ export class RecordModifyService { } async createRecordsOnlySql(tableId: string, createRecordsRo: ICreateRecordsRo): Promise { + await this.assertTableWritable(tableId); return this.createService.createRecordsOnlySql(tableId, createRecordsRo); } async deleteRecord(tableId: string, recordId: string, windowId?: string) { + await this.assertTableWritable(tableId); return this.deleteService.deleteRecord(tableId, recordId, windowId); } async deleteRecords(tableId: string, recordIds: string[], windowId?: string) { + await this.assertTableWritable(tableId); return this.deleteService.deleteRecords(tableId, recordIds, windowId); } @@ -78,6 +92,7 @@ export class RecordModifyService { order: IRecordInsertOrderRo, projection?: string[] ): Promise { + await this.assertTableWritable(tableId); return this.duplicateService.duplicateRecord(tableId, recordId, order, projection); } } diff --git a/apps/nestjs-backend/src/features/record/record.service.spec.ts b/apps/nestjs-backend/src/features/record/record.service.spec.ts index 16fe4e29d1..cb2674ea02 100644 --- a/apps/nestjs-backend/src/features/record/record.service.spec.ts +++ b/apps/nestjs-backend/src/features/record/record.service.spec.ts @@ -4,10 +4,72 @@ import { vi } from 'vitest'; import { RecordService } from './record.service'; describe('RecordService', () => { + it('queries only record IDs when resolving doc IDs for count-like callers', async () => { + const dataKnex = Knex({ client: 'pg' }); + const queriedSql: string[] = []; + const alias = 't_tblDocIds'; + const queryBuilder = dataKnex + .from({ [alias]: 'bse_data.tbl_doc_ids' }) + .select(`${alias}.__id`) + .select( + dataKnex.raw( + `(SELECT jsonb_build_object('id', u.id, 'title', u.name) FROM users u WHERE u.id = "${alias}"."__created_by") as "CreatedBy"` + ) + ); + const service = Object.create(RecordService.prototype) as { + knex: ReturnType; + getGroupRelatedData: ReturnType; + buildFilterSortQuery: ReturnType; + getSearchHitIndex: ReturnType; + logger: { debug: ReturnType; error: ReturnType }; + recordPermissionService: { wrapView: ReturnType }; + databaseRouter: { queryDataPrismaForTable: ReturnType }; + getDocIdsByQuery: RecordService['getDocIdsByQuery']; + }; + + service.knex = dataKnex; + service.getGroupRelatedData = vi.fn().mockResolvedValue({ + groupPoints: [], + allGroupHeaderRefs: [], + filter: undefined, + }); + service.buildFilterSortQuery = vi.fn().mockResolvedValue({ + queryBuilder, + dbTableName: 'bse_data.tbl_doc_ids', + alias, + }); + service.getSearchHitIndex = vi.fn().mockResolvedValue(undefined); + service.logger = { + debug: vi.fn(), + error: vi.fn(), + }; + service.recordPermissionService = { + wrapView: vi.fn().mockResolvedValue({ + builder: dataKnex.queryBuilder(), + viewCte: undefined, + }), + }; + service.databaseRouter = { + queryDataPrismaForTable: vi.fn(async (_tableId: string, sql: string) => { + queriedSql.push(sql); + return [{ __id: 'recDocId' }]; + }), + }; + + await expect( + service.getDocIdsByQuery('tblDocIds', { skip: 0, take: 10 }, true) + ).resolves.toMatchObject({ ids: ['recDocId'] }); + + expect(queriedSql[0]).toContain(`"${alias}"."__id"`); + expect(queriedSql[0]).not.toContain('users'); + + await dataKnex.destroy(); + }); + it('writes SQL-only created record history into the routed data DB internal schema', async () => { const dataKnex = Knex({ client: 'pg' }); const executedSql: string[] = []; - const service = Object.create(RecordService.prototype) as RecordService & { + const service = Object.create(RecordService.prototype) as { creditCheck: ReturnType; getFieldsByProjection: ReturnType; getWritableCreatedTimeFieldNames: ReturnType; @@ -18,6 +80,7 @@ describe('RecordService', () => { dataKnexForTable: ReturnType; getDataDatabaseUrlForTable: ReturnType; }; + createRecordsOnlySql: RecordService['createRecordsOnlySql']; }; service.cls = { diff --git a/apps/nestjs-backend/src/features/record/record.service.ts b/apps/nestjs-backend/src/features/record/record.service.ts index d5b7c78267..f179e85c34 100644 --- a/apps/nestjs-backend/src/features/record/record.service.ts +++ b/apps/nestjs-backend/src/features/record/record.service.ts @@ -23,9 +23,9 @@ import type { import { and, CellFormat, - CellValueType, DbFieldType, DriverClient, + extractFieldIdsFromFilter, FieldKeyType, FieldType, generateRecordHistoryId, @@ -67,7 +67,6 @@ import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.confi import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; -import { Events } from '../../event-emitter/events'; import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { RawOpType } from '../../share-db/interface'; @@ -570,6 +569,45 @@ export class RecordService { } } + private resolveAggregateProjection(params: { + groupBy?: IGroup; + filter?: IFilter; + searchFields?: IFieldInstance[]; + allowedFieldIds?: string[]; + }): string[] | undefined { + const { groupBy, filter, searchFields, allowedFieldIds } = params; + const projectionSet = new Set(); + + groupBy?.forEach(({ fieldId }) => { + if (fieldId) { + projectionSet.add(fieldId); + } + }); + + if (filter) { + for (const fieldId of extractFieldIdsFromFilter(filter)) { + projectionSet.add(fieldId); + } + } + + searchFields?.forEach((fieldInstance) => { + projectionSet.add(fieldInstance.id); + }); + + if (projectionSet.size === 0) { + return undefined; + } + + const projectionArray = Array.from(projectionSet); + if (!allowedFieldIds?.length) { + return projectionArray; + } + + const allowedSet = new Set(allowedFieldIds); + const filtered = projectionArray.filter((fieldId) => allowedSet.has(fieldId)); + return filtered.length ? filtered : undefined; + } + private async sanitizeFilterByEnabledFields( tableId: string, filter: IFilter | undefined, @@ -830,6 +868,7 @@ export class RecordService { | 'groupBy' | 'filter' | 'search' + | 'projection' | 'filterLinkCellCandidate' | 'filterLinkCellSelected' | 'collapsedGroupIds' @@ -920,11 +959,19 @@ export class RecordService { } if (search && search[2] && fieldMap) { + // query.projection narrows search to the fields the caller displays + // (e.g. personal view visible columns); intersect it with the permission + // whitelist so it can only ever shrink the searchable set + const searchProjection = query.projection?.length + ? enabledFieldIds + ? query.projection.filter((fieldId) => enabledFieldIds.includes(fieldId)) + : query.projection + : enabledFieldIds; const searchFields = await this.getSearchFields( fieldMap, search, query?.viewId, - enabledFieldIds + searchProjection ); const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); qb.where((builder) => { @@ -2053,7 +2100,7 @@ export class RecordService { }, useQueryModel ); - const { queryBuilder, dbTableName } = await this.buildFilterSortQuery( + const { queryBuilder, dbTableName, alias } = await this.buildFilterSortQuery( tableId, { ...query, @@ -2061,7 +2108,9 @@ export class RecordService { }, useQueryModel ); - // queryBuilder.select(this.knex.ref(`${selectDbTableName}.__id`)); + // This path only needs record IDs. Avoid evaluating display projections such as + // CreatedBy user lookups, which may reference meta-plane tables outside BYODB. + queryBuilder.clearSelect().select(`${alias}.__id`); skip && queryBuilder.offset(skip); if (take !== -1) { @@ -2181,6 +2230,10 @@ export class RecordService { return uniqBy( orderBy( Object.values(fieldInstanceMap) + // shared searchability predicate from @teable/core, also used by + // client-side highlighting; must run before the spread below which + // strips class methods + .filter((field) => field.isSearchable(search[0], { isSearchAllFields })) .map((field) => ({ ...field, isStructuredCellValue: field.isStructuredCellValue, @@ -2205,23 +2258,6 @@ export class RecordService { const searchArr = search?.[1]?.split(',') || []; return searchArr.includes(field.id); }) - .filter((field) => { - if (field.type === FieldType.Button) { - return false; - } - if (field.cellValueType === CellValueType.Boolean) { - return false; - } - if (isSearchAllFields) { - if (field.cellValueType === CellValueType.DateTime) { - return false; - } - if (field.cellValueType === CellValueType.Number && isNaN(Number(search[0]))) { - return false; - } - } - return true; - }) .map((field) => { return { ...field, @@ -2626,7 +2662,8 @@ export class RecordService { filter?: IFilter, search?: [string, string?, boolean?], viewId?: string, - useQueryModel = false + useQueryModel = false, + projection?: string[] ) { const withUserId = this.cls.get('user.id'); const wrap = await this.recordPermissionService.wrapView( @@ -2649,6 +2686,7 @@ export class RecordService { currentUserId: withUserId, useQueryModel, builder: wrap.builder, + projection, } ); @@ -2739,6 +2777,15 @@ export class RecordService { const withUserId = this.cls.get('user.id'); const shouldUseQueryModel = useQueryModel && !viewCte; + const searchFields = search?.[2] + ? await this.getSearchFields(fieldInstanceMap, search, viewId) + : []; + const aggregateProjection = this.resolveAggregateProjection({ + groupBy, + filter: mergedFilter, + searchFields, + allowedFieldIds: enabledFieldIds, + }); const { qb: queryBuilder, selectionMap } = await this.recordQueryBuilder.createRecordAggregateBuilder(viewCte ?? dbTableName, { tableId, @@ -2755,10 +2802,10 @@ export class RecordService { currentUserId: withUserId, useQueryModel: shouldUseQueryModel, builder: permissionBuilder, + projection: aggregateProjection, }); if (search && search[2]) { - const searchFields = await this.getSearchFields(fieldInstanceMap, search, viewId); const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); queryBuilder.where((builder) => { this.dbProvider.searchQuery(builder, searchFields, tableIndex, search, { selectionMap }); @@ -2777,7 +2824,8 @@ export class RecordService { mergedFilter, search, viewId, - useQueryModel + useQueryModel, + aggregateProjection ); try { diff --git a/apps/nestjs-backend/src/features/share/share-link-query.util.ts b/apps/nestjs-backend/src/features/share/share-link-query.util.ts new file mode 100644 index 0000000000..75b94132da --- /dev/null +++ b/apps/nestjs-backend/src/features/share/share-link-query.util.ts @@ -0,0 +1,20 @@ +/** + * A link "selection" query loads records that are already linked or explicitly chosen + * (via `filterLinkCellSelected` or `selectedRecordIds`), as opposed to the candidate + * list of records available to be newly linked (`filterLinkCellCandidate`). + * + * The link field's view scope (`filterByViewId`) and configured filter only constrain + * the candidate list, so selection queries must NOT be filtered by them — otherwise an + * already-linked record that falls outside the configured view disappears (renders + * blank / rowCount 0). See T4864. + * + * Note: `selectedRecordIds` combined with `filterLinkCellCandidate` is the "exclude + * these from the candidate list" case, which must keep the view scope. + */ +export const isLinkRecordSelectionQuery = (query?: { + filterLinkCellSelected?: unknown; + filterLinkCellCandidate?: unknown; + selectedRecordIds?: string[]; +}): boolean => + Boolean(query?.filterLinkCellSelected) || + (Boolean(query?.selectedRecordIds?.length) && !query?.filterLinkCellCandidate); diff --git a/apps/nestjs-backend/src/features/share/share-socket.service.ts b/apps/nestjs-backend/src/features/share/share-socket.service.ts index 5566fd138a..b4a06e41b0 100644 --- a/apps/nestjs-backend/src/features/share/share-socket.service.ts +++ b/apps/nestjs-backend/src/features/share/share-socket.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { HttpErrorCode, type IGetFieldsQuery } from '@teable/core'; import type { IGetRecordsRo } from '@teable/openapi'; import { difference } from 'lodash'; @@ -7,6 +7,7 @@ import { FieldService } from '../field/field.service'; import { RecordService } from '../record/record.service'; import { ViewService } from '../view/view.service'; import type { IShareViewInfo } from './share-auth.service'; +import { isLinkRecordSelectionQuery } from './share-link-query.util'; @Injectable() export class ShareSocketService { @@ -113,10 +114,12 @@ export class ShareSocketService { const { id } = view ?? {}; const { filterByViewId } = linkOptions ?? {}; - const viewId = filterByViewId ?? id; - // if filterLinkCellSelected is not empty, use it as filter - const defaultFilter = linkOptions?.filter ?? query.filter; - const filter = !query.filterLinkCellSelected ? defaultFilter : undefined; + // Queries that load already-linked records (filterLinkCellSelected or explicit + // selectedRecordIds) must return them in full, even when they fall outside the link + // field's view scope. The view scope/filter only constrains the candidate list. T4864. + const isLinkSelectionQuery = Boolean(linkOptions) && isLinkRecordSelectionQuery(query); + const viewId = isLinkSelectionQuery ? id : filterByViewId ?? id; + const filter = isLinkSelectionQuery ? undefined : linkOptions?.filter ?? query.filter; let projection = query.projection; if (linkOptions) { diff --git a/apps/nestjs-backend/src/features/share/share.service.ts b/apps/nestjs-backend/src/features/share/share.service.ts index ee400c4332..367e06a401 100644 --- a/apps/nestjs-backend/src/features/share/share.service.ts +++ b/apps/nestjs-backend/src/features/share/share.service.ts @@ -51,6 +51,7 @@ import { RecordOpenApiService } from '../record/open-api/record-open-api.service import { RecordService } from '../record/record.service'; import { SelectionService } from '../selection/selection.service'; import type { IShareViewInfo } from './share-auth.service'; +import { isLinkRecordSelectionQuery } from './share-link-query.util'; import { ShareSocketService } from './share-socket.service'; export interface IJwtShareInfo { @@ -210,15 +211,19 @@ export class ShareService { const { id } = view ?? {}; const { filterByViewId } = linkOptions ?? {}; - const viewId = filterByViewId ?? id; const tableId = shareInfo.tableId; - // if filterLinkCellSelected is not empty, use it as filter - const defaultFilter = linkOptions?.filter ?? query?.filter; - const filter = query?.filterLinkCellSelected ? undefined : defaultFilter; + // The link field's view scope (filterByViewId)/filter only constrains which records + // can be newly linked (the candidate list). Queries that load already-linked records + // (filterLinkCellSelected or explicit selectedRecordIds) must count them in full, + // even when they fall outside that view — otherwise an already-linked record outside + // the configured view reports rowCount 0. T4864. + const isLinkSelectionQuery = Boolean(linkOptions) && isLinkRecordSelectionQuery(query); + const viewId = isLinkSelectionQuery ? id : filterByViewId ?? id; + const filter = isLinkSelectionQuery ? undefined : linkOptions?.filter ?? query?.filter; const result = await this.aggregationService.performRowCount(tableId, { + ...query, viewId, filter, - ...query, }); return { @@ -248,13 +253,17 @@ export class ShareService { ? fields.filter((f) => visibleFieldIds?.includes(f.id) || f.isPrimary) : fields; - // filterLinkCellSelected applies its own filter; skip the default. - const filter = query?.filterLinkCellSelected ? undefined : query?.filter ?? linkFilter; + // Queries that load already-linked records (filterLinkCellSelected or explicit + // selectedRecordIds) must return them in full, even when they fall outside the link + // field's view scope — otherwise an already-linked record outside the configured + // view renders blank. The view scope/filter only constrains the candidate list. T4864. + const isLinkSelectionQuery = Boolean(linkOptions) && isLinkRecordSelectionQuery(query); + const filter = isLinkSelectionQuery ? undefined : query?.filter ?? linkFilter; return await this.recordService.getRecords( tableId, { - viewId, + viewId: isLinkSelectionQuery ? id : viewId, skip: query?.skip ?? 0, take: query?.take ?? 100, filter, @@ -550,18 +559,12 @@ export class ShareService { const users = await this.prismaService.user.findMany({ where: { id: { in: userIds }, - ...(search - ? { - OR: [ - { name: { contains: search, mode: 'insensitive' } }, - { email: { contains: search, mode: 'insensitive' } }, - ], - } - : {}), + // Match by name only: searching by email would let anonymous viewers + // probe membership by email even though email is not returned. + ...(search ? { name: { contains: search, mode: 'insensitive' } } : {}), }, select: { id: true, - email: true, name: true, avatar: true, }, @@ -569,9 +572,10 @@ export class ShareService { take, }); - return users.map(({ id, email, name, avatar }) => ({ + // Email is intentionally omitted from share responses to avoid leaking the + // member directory to anonymous viewers. Picker selection is by id. + return users.map(({ id, name, avatar }) => ({ userId: id, - email, userName: name, avatar: avatar && getPublicFullStorageUrl(avatar), })); @@ -619,10 +623,13 @@ export class ShareService { skip, take, search, + // Anonymous share views must not allow probing membership by email. + searchByEmail: false, }); + // Email is intentionally omitted from share responses to avoid leaking the + // member directory to anonymous viewers. Picker selection is by id. return list.map((item) => ({ userId: item.id, - email: item.email, userName: item.name, avatar: item.avatar, })); diff --git a/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts b/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts index 6777ea8ac4..065e6b977d 100644 --- a/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts @@ -9,4 +9,8 @@ export class DataDbBaselineService { await this.migrationService.migrate(url, internalSchema); return this.migrationService.getLatestSchemaVersion(); } + + getLatestSchemaVersion() { + return this.migrationService.getLatestSchemaVersion(); + } } diff --git a/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts b/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts index 578a108da7..68cabf628e 100644 --- a/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts +++ b/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts @@ -4,6 +4,25 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DataDbBindingService } from './data-db-binding.service'; import { encryptDataDbUrl } from './data-db-url-secret'; +vi.mock('@teable/db-main-prisma', () => ({ + MetaPrismaService: class MetaPrismaService {}, + Prisma: {}, + PrismaModule: class PrismaModule {}, + PrismaService: class PrismaService {}, + ProvisionState: {}, + getDatabaseUrl: vi.fn(), +})); +vi.mock('@teable/db-data-prisma', () => ({ + DataPrismaModule: class DataPrismaModule {}, + DataPrismaService: class DataPrismaService {}, + PrismaClient: class PrismaClient {}, + getMetaDatabaseUrl: vi.fn(), +})); +vi.mock('@prisma/client', () => ({ + Prisma: {}, + PrismaClient: class PrismaClient {}, +})); + const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; const initializeEmptyTargetMode = 'initialize-empty'; const internalSchema = 'teable_meta_test'; @@ -52,6 +71,10 @@ describe('DataDbBindingService', () => { const dataDbMigrationService = { ensureConnectionMigrated: vi.fn(), }; + const spaceDataDbMigrationService = { + startMigrationForSpace: vi.fn(), + runMigrationJob: vi.fn(), + }; const byodbBinding = { mode: 'byodb', state: 'ready', @@ -83,6 +106,13 @@ describe('DataDbBindingService', () => { baselineService.initialize.mockReset().mockResolvedValue(schemaVersion); dataDbClientManager.invalidateConnection.mockReset(); dataDbMigrationService.ensureConnectionMigrated.mockReset().mockResolvedValue([]); + spaceDataDbMigrationService.startMigrationForSpace.mockReset().mockResolvedValue({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + spaceDataDbMigrationService.runMigrationJob.mockReset().mockResolvedValue({ + state: 'succeeded', + }); prismaService.dataDbConnection.update.mockReset(); prismaService.spaceDataDbBinding.findUnique.mockReset().mockResolvedValue(byodbBinding); prismaService.spaceDataDbBinding.updateMany.mockReset(); @@ -142,6 +172,39 @@ describe('DataDbBindingService', () => { expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); }); + it('returns preflight errors before deriving binding metadata from an invalid URL', async () => { + preflightService.preflight.mockResolvedValue({ + ok: false, + provider: 'postgres', + classification: 'non-empty-unknown', + capabilities, + errors: [{ code: 'INVALID_DATABASE_URL', message: 'Invalid URL' }], + }); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never + ); + + await expect( + service.createBindingForNewSpace('spcxxx', 'usrxxx', { + mode: 'byodb', + url: 'not-a-postgres-url', + targetMode: initializeEmptyTargetMode, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + preflight: expect.objectContaining({ + errors: [expect.objectContaining({ code: 'INVALID_DATABASE_URL' })], + }), + }), + }); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.upsert).not.toHaveBeenCalled(); + }); + it('generates an internal schema for new BYODB spaces when one is not provided', async () => { preflightService.preflight.mockResolvedValue({ ok: true, @@ -167,7 +230,7 @@ describe('DataDbBindingService', () => { expect(preflightService.preflight).toHaveBeenCalledWith({ url: dataUrl, targetMode: initializeEmptyTargetMode, - internalSchema: generatedInternalSchema, + internalSchema: undefined, }); expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, generatedInternalSchema); expect(txClient.dataDbConnection.upsert).toHaveBeenCalledWith( @@ -195,7 +258,7 @@ describe('DataDbBindingService', () => { const dataDb = { mode: 'byodb' as const, url: dataUrl, - targetMode: initializeEmptyTargetMode, + targetMode: 'initialize-empty' as const, internalSchema, }; await service.createBindingForNewSpace('spcxxx1', 'usrxxx', dataDb); @@ -389,7 +452,97 @@ describe('DataDbBindingService', () => { expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); }); - it('does not create a BYODB binding for an existing default space without adopt-existing mode', async () => { + it('queues a migration job for an existing default space with migrate-space target mode', async () => { + prismaService.spaceDataDbBinding.findUnique.mockResolvedValue(null); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never, + spaceDataDbMigrationService as never + ); + + await service.updateBindingForSpace( + 'spcxxx', + 'usrxxx', + { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + confirmLargeMigration: true, + }, + { allowSpaceMigration: true } + ); + + expect(spaceDataDbMigrationService.startMigrationForSpace).toHaveBeenCalledWith( + 'spcxxx', + 'usrxxx', + { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + confirmLargeMigration: true, + } + ); + expect(spaceDataDbMigrationService.runMigrationJob).not.toHaveBeenCalled(); + expect(preflightService.preflight).not.toHaveBeenCalled(); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + }); + + it('rejects migrate-space unless the caller allows admin-only space migration', async () => { + prismaService.spaceDataDbBinding.findUnique.mockResolvedValue(null); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never, + spaceDataDbMigrationService as never + ); + + await expect( + service.updateBindingForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + data: { errorCode: 'SPACE_DATA_DB_ADMIN_ONLY' }, + }); + expect(spaceDataDbMigrationService.startMigrationForSpace).not.toHaveBeenCalled(); + expect(preflightService.preflight).not.toHaveBeenCalled(); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + }); + + it('rejects migrate-space when the migration service is unavailable', async () => { + prismaService.spaceDataDbBinding.findUnique.mockResolvedValue(null); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await expect( + service.updateBindingForSpace( + 'spcxxx', + 'usrxxx', + { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }, + { allowSpaceMigration: true } + ) + ).rejects.toMatchObject({ code: HttpErrorCode.CONFLICT }); + expect(preflightService.preflight).not.toHaveBeenCalled(); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + }); + + it('does not create a BYODB binding for an existing default space without adopt-existing or migrate-space mode', async () => { prismaService.spaceDataDbBinding.findUnique.mockResolvedValue(null); const service = new DataDbBindingService( prismaService as never, @@ -411,6 +564,16 @@ describe('DataDbBindingService', () => { }); it('rejects credential updates that would move the space to a different data DB', async () => { + preflightService.preflight.mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'teable-managed-compatible', + capabilities, + errors: [], + internalSchema, + displayHost: 'other.example.com:5432', + displayDatabase: 'teable_data', + }); const service = new DataDbBindingService( prismaService as never, preflightService as never, diff --git a/apps/nestjs-backend/src/features/space/data-db-binding.service.ts b/apps/nestjs-backend/src/features/space/data-db-binding.service.ts index 1bfffee7a9..c80199b3bb 100644 --- a/apps/nestjs-backend/src/features/space/data-db-binding.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-binding.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import { HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { ICreateSpaceRo, IDataDbPreflightRo, IDataDbPreflightVo } from '@teable/openapi'; @@ -13,8 +13,17 @@ import { getDatabaseUrlDisplayParts, } from './data-db-preflight.service'; import { decryptDataDbUrl, encryptDataDbUrl } from './data-db-url-secret'; +import { + migrateSpaceTargetMode, + spaceDataDbAdminOnlyErrorCode, + spaceDataDbAdminOnlyMessage, +} from './space-data-db-migration.constants'; +import { SpaceDataDbMigrationService } from './space-data-db-migration.service'; type IDataDbCreateOptions = NonNullable; +type IDataDbUpdateOptions = { + allowSpaceMigration?: boolean; +}; type IPreparedDataDbBinding = { encryptedUrl: string; urlFingerprint: string; @@ -24,6 +33,20 @@ type IPreparedDataDbBinding = { schemaVersion: string | null; capabilities: IDataDbPreflightVo['capabilities']; }; +type IExistingDataDbConnection = { + id: string; + encryptedUrl: string; + displayHost: string | null; + displayDatabase: string | null; + internalSchema: string; + createdBy?: string | null; +}; +type IExistingBindingUpdateTarget = { + preflight: IDataDbPreflightVo; + internalSchema: string; + displayHost: string; + displayDatabase: string; +}; const initializeEmptyTargetMode = 'initialize-empty'; const adoptExistingTargetMode = 'adopt-existing'; @@ -43,7 +66,8 @@ export class DataDbBindingService { private readonly preflightService: DataDbPreflightService, private readonly baselineService: DataDbBaselineService, private readonly dataDbClientManager: DataDbClientManager, - private readonly dataDbMigrationService?: DataDbMigrationService + @Optional() private readonly dataDbMigrationService?: DataDbMigrationService, + @Optional() private readonly spaceDataDbMigrationService?: SpaceDataDbMigrationService ) {} async createBindingForNewSpace( @@ -187,7 +211,12 @@ export class DataDbBindingService { return applied; } - async updateBindingForSpace(spaceId: string, updatedBy: string, dataDb: IDataDbPreflightRo) { + async updateBindingForSpace( + spaceId: string, + updatedBy: string, + dataDb: IDataDbPreflightRo, + options: IDataDbUpdateOptions = {} + ) { if (!dataDb.url) { throw new CustomHttpException(dataDbUrlRequiredError, HttpErrorCode.VALIDATION_ERROR); } @@ -197,55 +226,23 @@ export class DataDbBindingService { include: { dataDbConnection: true }, }); if (binding?.mode !== 'byodb' || !binding.dataDbConnection?.encryptedUrl) { - if (dataDb.targetMode === adoptExistingTargetMode) { - await this.createBindingForExistingSpace(spaceId, updatedBy, dataDb); - return; - } - - throw new CustomHttpException( - 'BYODB data database binding was not found', - HttpErrorCode.NOT_FOUND - ); + await this.updateDefaultSpaceBinding(spaceId, updatedBy, dataDb, options); + return; } const current = binding.dataDbConnection; - const internalSchema = resolveDataDbInternalSchema( - dataDb.internalSchema ?? current.internalSchema, - dataDb.url - ); - const nextDisplayParts = getDatabaseUrlDisplayParts(dataDb.url); + const target = await this.prepareExistingBindingUpdateTarget(dataDb, current); + this.assertExistingBindingTargetUnchanged(current, target); - if ( - current.internalSchema !== internalSchema || - current.displayHost !== nextDisplayParts.displayHost || - current.displayDatabase !== nextDisplayParts.displayDatabase - ) { - throw new CustomHttpException( - 'Changing the BYODB database or internal schema is not supported yet', - HttpErrorCode.VALIDATION_ERROR - ); - } - - const preflight = await this.preflightService.preflight({ - url: dataDb.url, - targetMode: initializeEmptyTargetMode, - internalSchema, - }); - if (!preflight.ok) { - throw new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { - preflight, - }); - } - - const schemaVersion = await this.baselineService.initialize(dataDb.url, internalSchema); + const schemaVersion = await this.baselineService.initialize(dataDb.url, target.internalSchema); await this.prismaService.dataDbConnection.update({ where: { id: current.id }, data: { encryptedUrl: encryptDataDbUrl(dataDb.url), - urlFingerprint: fingerprintDataDbConnection(dataDb.url, internalSchema), + urlFingerprint: fingerprintDataDbConnection(dataDb.url, target.internalSchema), status: 'ready', schemaVersion, - capabilities: preflight.capabilities, + capabilities: target.preflight.capabilities, lastValidatedAt: new Date(), lastError: null, createdBy: current.createdBy ?? updatedBy, @@ -258,6 +255,101 @@ export class DataDbBindingService { await this.dataDbClientManager.invalidateConnection(current.id); } + private async updateDefaultSpaceBinding( + spaceId: string, + updatedBy: string, + dataDb: IDataDbPreflightRo, + options: IDataDbUpdateOptions + ) { + if (dataDb.targetMode === adoptExistingTargetMode) { + await this.createBindingForExistingSpace(spaceId, updatedBy, dataDb); + return; + } + if (dataDb.targetMode === migrateSpaceTargetMode) { + this.assertSpaceMigrationAllowed(options); + await this.startSpaceMigration(spaceId, updatedBy, dataDb); + return; + } + + throw new CustomHttpException( + 'BYODB data database binding was not found', + HttpErrorCode.NOT_FOUND + ); + } + + private assertSpaceMigrationAllowed(options: IDataDbUpdateOptions) { + if (options.allowSpaceMigration) { + return; + } + throw new CustomHttpException(spaceDataDbAdminOnlyMessage, HttpErrorCode.RESTRICTED_RESOURCE, { + errorCode: spaceDataDbAdminOnlyErrorCode, + }); + } + + private async startSpaceMigration( + spaceId: string, + updatedBy: string, + dataDb: IDataDbPreflightRo + ) { + if (!this.spaceDataDbMigrationService) { + throw new CustomHttpException( + 'Space data database migration service is unavailable', + HttpErrorCode.CONFLICT + ); + } + await this.spaceDataDbMigrationService.startMigrationForSpace(spaceId, updatedBy, dataDb); + } + + private async prepareExistingBindingUpdateTarget( + dataDb: IDataDbPreflightRo, + current: IExistingDataDbConnection + ): Promise { + const preflight = await this.preflightService.preflight({ + url: dataDb.url, + targetMode: initializeEmptyTargetMode, + internalSchema: dataDb.internalSchema ?? current.internalSchema, + }); + if (!preflight.ok) { + throw new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { + preflight, + }); + } + + const internalSchema = + preflight.internalSchema ?? + resolveDataDbInternalSchema(dataDb.internalSchema ?? current.internalSchema, dataDb.url); + const displayParts = + preflight.displayHost && preflight.displayDatabase != null + ? { + displayHost: preflight.displayHost, + displayDatabase: preflight.displayDatabase, + } + : getDatabaseUrlDisplayParts(dataDb.url); + + return { + preflight, + internalSchema, + ...displayParts, + }; + } + + private assertExistingBindingTargetUnchanged( + current: Pick, + next: Pick + ) { + if ( + current.internalSchema === next.internalSchema && + current.displayHost === next.displayHost && + current.displayDatabase === next.displayDatabase + ) { + return; + } + throw new CustomHttpException( + 'Changing the BYODB database or internal schema is not supported yet', + HttpErrorCode.VALIDATION_ERROR + ); + } + async createBindingForExistingSpace( spaceId: string, createdBy: string, @@ -328,11 +420,10 @@ export class DataDbBindingService { dataDb: IDataDbPreflightRo, targetMode: IDataDbPreflightRo['targetMode'] ): Promise { - const internalSchema = resolveDataDbInternalSchema(dataDb.internalSchema, dataDb.url); const preflight = await this.preflightService.preflight({ url: dataDb.url, targetMode, - internalSchema, + internalSchema: dataDb.internalSchema, }); if (!preflight.ok) { throw new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { @@ -340,9 +431,17 @@ export class DataDbBindingService { }); } + const internalSchema = + preflight.internalSchema ?? resolveDataDbInternalSchema(dataDb.internalSchema, dataDb.url); const schemaVersion = await this.baselineService.initialize(dataDb.url, internalSchema); - const { displayHost, displayDatabase } = getDatabaseUrlDisplayParts(dataDb.url); + const { displayHost, displayDatabase } = + preflight.displayHost && preflight.displayDatabase != null + ? { + displayHost: preflight.displayHost, + displayDatabase: preflight.displayDatabase, + } + : getDatabaseUrlDisplayParts(dataDb.url); return { encryptedUrl: encryptDataDbUrl(dataDb.url), urlFingerprint: fingerprintDataDbConnection(dataDb.url, internalSchema), diff --git a/apps/nestjs-backend/src/features/space/data-db-migration.service.ts b/apps/nestjs-backend/src/features/space/data-db-migration.service.ts index d38e22dcf2..dd344b1f82 100644 --- a/apps/nestjs-backend/src/features/space/data-db-migration.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-migration.service.ts @@ -24,6 +24,7 @@ export const DATA_DB_MIGRATION_TABLE = '__teable_data_schema_migrations'; const migrationsRootCandidates = [ join(process.cwd(), 'community/packages/db-data-prisma/prisma/migrations'), + join(process.cwd(), '../../packages/db-data-prisma/prisma/migrations'), join(process.cwd(), '../../community/packages/db-data-prisma/prisma/migrations'), ]; const defaultMigrationStatementTimeoutMs = 300_000; diff --git a/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts b/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts index 8e9941f77a..a1c0dc9bf2 100644 --- a/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts +++ b/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts @@ -6,8 +6,10 @@ import type { IDataDbPreflightClient } from './data-db-preflight.service'; import { DataDbPreflightService, fingerprintDatabaseUrl, + fingerprintDataDbConnection, maskDatabaseUrl, } from './data-db-preflight.service'; +import { encryptDataDbUrl } from './data-db-url-secret'; type IFakeDbState = { schemas?: string[]; @@ -15,9 +17,15 @@ type IFakeDbState = { functions?: string[]; databases?: string[]; failCreateSchema?: boolean; + failCreateTable?: boolean; + failCreateIndex?: boolean; + failCreateFunction?: boolean; + failCreateTrigger?: boolean; failConnect?: boolean; failMessage?: string; failCode?: string; + canCreateRole?: boolean; + canGrantPrivileges?: boolean; }; class FakePreflightClient implements IDataDbPreflightClient { @@ -32,14 +40,26 @@ class FakePreflightClient implements IDataDbPreflightClient { if (sql.includes('CREATE SCHEMA') && this.state.failCreateSchema) { throw new Error('permission denied for database'); } + if (sql.includes('CREATE TABLE') && this.state.failCreateTable) { + throw new Error('permission denied for table'); + } + if (sql.includes('CREATE INDEX') && this.state.failCreateIndex) { + throw new Error('permission denied for index'); + } + if (sql.includes('CREATE OR REPLACE FUNCTION') && this.state.failCreateFunction) { + throw new Error('permission denied for function'); + } + if (sql.includes('CREATE TRIGGER') && this.state.failCreateTrigger) { + throw new Error('permission denied for trigger'); + } if (sql.includes('SHOW server_version')) { return { rows: [{ server_version: '14.12' }] as T[] }; } if (sql.includes('has_database_privilege')) { - return { rows: [{ can_create: true }] as T[] }; + return { rows: [{ can_create: this.state.canGrantPrivileges ?? true }] as T[] }; } if (sql.includes('pg_roles')) { - return { rows: [{ can_create_role: false }] as T[] }; + return { rows: [{ can_create_role: this.state.canCreateRole ?? false }] as T[] }; } if (sql.includes('pg_stat_activity')) { return { rows: [{ count: '0' }] as T[] }; @@ -89,6 +109,27 @@ const DATA_SCHEMA_MIGRATION_TABLE = '__teable_data_schema_migrations'; const createService = (state: IFakeDbState) => new DataDbPreflightService(undefined, () => new FakePreflightClient(state)); +const defaultRelatedSpacesPrisma = () => ({ + field: { + findMany: vi.fn().mockResolvedValue([]), + }, + tableMeta: { + findMany: vi.fn().mockResolvedValue([]), + }, + space: { + findMany: vi.fn().mockResolvedValue([ + { + id: 'spcxxx', + name: 'Space', + baseGroup: [], + }, + ]), + }, + spaceDataDbBinding: { + findMany: vi.fn().mockResolvedValue([]), + }, +}); + describe('database URL helpers', () => { it('masks database URL passwords', () => { expect(maskDatabaseUrl(DATA_URL)).toBe('postgresql://teable:***@example.com:5432/teable_data'); @@ -241,6 +282,106 @@ describe('DataDbPreflightService', () => { expect(result.errors.map((error) => error.code)).toContain('DDL_PRIVILEGE_CHECK_FAILED'); }); + it('reports missing table or index privileges with partial DDL capabilities', async () => { + const missingTable = await createService({ + schemas: ['public'], + failCreateTable: true, + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + }); + + expect(missingTable.ok).toBe(false); + expect(missingTable.capabilities).toMatchObject({ + createSchema: true, + createTable: false, + createFunction: false, + createTrigger: false, + }); + expect(missingTable.errors.map((error) => error.code)).toContain('DDL_PRIVILEGE_CHECK_FAILED'); + + const missingIndex = await createService({ + schemas: ['public'], + failCreateIndex: true, + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + }); + + expect(missingIndex.ok).toBe(false); + expect(missingIndex.capabilities).toMatchObject({ + createSchema: true, + createTable: true, + createFunction: false, + createTrigger: false, + }); + expect(missingIndex.errors.map((error) => error.code)).toContain('DDL_PRIVILEGE_CHECK_FAILED'); + }); + + it('reports missing function or trigger privileges with partial DDL capabilities', async () => { + const missingFunction = await createService({ + schemas: ['public'], + failCreateFunction: true, + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + }); + + expect(missingFunction.ok).toBe(false); + expect(missingFunction.capabilities).toMatchObject({ + createSchema: true, + createTable: true, + createFunction: false, + createTrigger: false, + }); + expect(missingFunction.errors.map((error) => error.code)).toContain( + 'DDL_PRIVILEGE_CHECK_FAILED' + ); + + const missingTrigger = await createService({ + schemas: ['public'], + failCreateTrigger: true, + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + }); + + expect(missingTrigger.ok).toBe(false); + expect(missingTrigger.capabilities).toMatchObject({ + createSchema: true, + createTable: true, + createFunction: true, + createTrigger: false, + }); + expect(missingTrigger.errors.map((error) => error.code)).toContain( + 'DDL_PRIVILEGE_CHECK_FAILED' + ); + }); + + it('reports readonly sharing capability separately from core BYODB usability', async () => { + const result = await createService({ + schemas: ['public'], + tables: [], + canCreateRole: false, + canGrantPrivileges: false, + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + }); + + expect(result.ok).toBe(true); + expect(result.classification).toBe('empty'); + expect(result.capabilities).toMatchObject({ + createSchema: true, + createTable: true, + createFunction: true, + createTrigger: true, + createRole: false, + grantPrivileges: false, + inspectActivity: true, + }); + }); + it('does not leak passwords in connection errors', async () => { const result = await createService({ failConnect: true, @@ -335,20 +476,172 @@ describe('DataDbPreflightService', () => { it('returns default summary when a space has no explicit binding', async () => { const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), spaceDataDbBinding: { + ...defaultRelatedSpacesPrisma().spaceDataDbBinding, findUnique: vi.fn().mockResolvedValue(null), }, + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + }, } as never); - await expect(service.getSummary('spcxxx')).resolves.toEqual({ + await expect(service.getSummary('spcxxx')).resolves.toMatchObject({ mode: 'default', state: 'ready', + relatedSpaces: expect.objectContaining({ + primarySpaceId: 'spcxxx', + hasCrossSpaceLinks: false, + }), }); }); + it('returns default summary when the migration job table is not available yet', async () => { + const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), + spaceDataDbBinding: { + ...defaultRelatedSpacesPrisma().spaceDataDbBinding, + findUnique: vi.fn().mockResolvedValue(null), + }, + spaceDataDbMigrationJob: { + findFirst: vi + .fn() + .mockRejectedValue( + Object.assign( + new Error('The table `public.space_data_db_migration_job` does not exist'), + { code: 'P2021' } + ) + ), + findMany: vi.fn().mockResolvedValue([]), + }, + } as never); + + await expect(service.getSummary('spcxxx')).resolves.toMatchObject({ + mode: 'default', + state: 'ready', + relatedSpaces: expect.objectContaining({ + primarySpaceId: 'spcxxx', + hasCrossSpaceLinks: false, + }), + }); + }); + + it('returns migrating BYODB summary when a default space has an active switch migration job', async () => { + const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), + spaceDataDbBinding: { + ...defaultRelatedSpacesPrisma().spaceDataDbBinding, + findUnique: vi.fn().mockResolvedValue(null), + }, + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue({ + id: 'sdmjxxx', + state: 'copying', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + lastError: null, + targetConnection: { + provider: 'postgres', + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema, + schemaVersion: '20260421000000_init_data_db_baseline', + lastValidatedAt: new Date('2026-05-06T00:00:00.000Z'), + lastError: null, + encryptedUrl: 'encrypted-secret', + capabilities: { + createSchema: true, + createTable: true, + createFunction: true, + createTrigger: true, + createRole: false, + grantPrivileges: true, + inspectActivity: true, + }, + }, + }), + findMany: vi.fn().mockResolvedValue([]), + }, + } as never); + + const summary = await service.getSummary('spcxxx'); + + expect(summary).toMatchObject({ + mode: 'byodb', + state: 'migrating', + provider: 'postgres', + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema, + migration: { + jobId: 'sdmjxxx', + state: 'copying', + targetInternalSchema: internalSchema, + }, + }); + expect(JSON.stringify(summary)).not.toContain('encrypted-secret'); + }); + + it('keeps a default-space summary while a test-only migration job is active', async () => { + const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), + spaceDataDbBinding: { + ...defaultRelatedSpacesPrisma().spaceDataDbBinding, + findUnique: vi.fn().mockResolvedValue(null), + }, + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue({ + id: 'sdmjxxx', + state: 'waiting_worker', + switchOnCompletion: false, + targetInternalSchema: internalSchema, + lastError: null, + targetConnection: { + provider: 'postgres', + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema, + schemaVersion: '20260421000000_init_data_db_baseline', + lastValidatedAt: new Date('2026-05-06T00:00:00.000Z'), + lastError: null, + encryptedUrl: 'encrypted-secret', + capabilities: { + createSchema: true, + createTable: true, + createFunction: true, + createTrigger: true, + createRole: false, + grantPrivileges: true, + inspectActivity: true, + }, + }, + }), + findMany: vi.fn().mockResolvedValue([]), + }, + } as never); + + const summary = await service.getSummary('spcxxx'); + + expect(summary).toMatchObject({ + mode: 'default', + state: 'ready', + migration: { + jobId: 'sdmjxxx', + state: 'waiting_worker', + targetInternalSchema: internalSchema, + switchOnCompletion: false, + }, + }); + expect(summary.displayHost).toBeUndefined(); + expect(JSON.stringify(summary)).not.toContain('encrypted-secret'); + }); + it('returns a BYODB summary without encrypted URL material', async () => { const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), spaceDataDbBinding: { + ...defaultRelatedSpacesPrisma().spaceDataDbBinding, findUnique: vi.fn().mockResolvedValue({ mode: 'byodb', state: 'ready', @@ -373,6 +666,10 @@ describe('DataDbPreflightService', () => { }, }), }, + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + }, } as never); const summary = await service.getSummary('spcxxx'); @@ -389,4 +686,44 @@ describe('DataDbPreflightService', () => { }); expect(JSON.stringify(summary)).not.toContain('encrypted-secret'); }); + + it('includes physical database fingerprints for related BYODB spaces', async () => { + const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([ + { + spaceId: 'spcxxx', + mode: 'byodb', + state: 'ready', + dataDbConnection: { + id: 'dcnxxx', + encryptedUrl: encryptDataDbUrl(DATA_URL), + urlFingerprint: fingerprintDataDbConnection(DATA_URL, internalSchema), + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema, + }, + }, + ]), + }, + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + }, + } as never); + + await expect(service.getSummary('spcxxx')).resolves.toMatchObject({ + relatedSpaces: { + spaces: [ + expect.objectContaining({ + spaceId: 'spcxxx', + dataDbUrlFingerprint: fingerprintDataDbConnection(DATA_URL, internalSchema), + dataDbDatabaseFingerprint: fingerprintDatabaseUrl(DATA_URL), + }), + ], + }, + }); + }); }); diff --git a/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts b/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts index 195f6dc361..31d9ad0fcf 100644 --- a/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts @@ -14,6 +14,8 @@ import type { import type { Knex } from 'knex'; import createKnex from 'knex'; import { resolveDataDbInternalSchema } from './data-db-internal-schema'; +import { activeSpaceDataDbMigrationStates } from './space-data-db-migration.constants'; +import { resolveSpaceDataDbRelatedSpaces } from './space-data-db-related-spaces'; type IPreflightCapabilities = IDataDbPreflightVo['capabilities']; type IPreflightClassification = IDataDbPreflightVo['classification']; @@ -77,6 +79,32 @@ const normalizeRawRows = (result: { rows?: T[] } | T[]): T[] => { return result.rows ?? []; }; +type IActiveMigrationSummary = { + id: string; + state: string; + targetInternalSchema: string; + switchOnCompletion?: boolean | null; + lastError: string | null; + inventory: unknown; + targetConnection: { + provider: 'postgres'; + displayHost: string | null; + displayDatabase: string | null; + internalSchema: string; + schemaVersion: string | null; + lastValidatedAt: Date | null; + lastError: string | null; + capabilities: unknown; + } | null; +}; + +type IDataDbSummaryPrisma = PrismaService & { + spaceDataDbMigrationJob?: { + findFirst(args: unknown): Promise; + findMany(args: unknown): Promise; + }; +}; + export const maskDatabaseUrl = (url: string): string => { const parsed = new URL(url); if (parsed.password) { @@ -267,6 +295,12 @@ export class DataDbPreflightService { async getSummary(spaceId: string): Promise { if (this.prismaService) { + const relatedSpaces = await resolveSpaceDataDbRelatedSpaces(this.prismaService, spaceId); + const migrationSummary = await this.getActiveMigrationSummary(spaceId); + if (migrationSummary) { + return migrationSummary; + } + const binding = await this.prismaService.spaceDataDbBinding.findUnique({ where: { spaceId }, include: { dataDbConnection: true }, @@ -285,8 +319,14 @@ export class DataDbPreflightService { capabilities: binding.dataDbConnection.capabilities as | IDataDbConnectionSummaryVo['capabilities'] | undefined, + relatedSpaces, }; } + return { + mode: 'default', + state: 'ready', + relatedSpaces, + }; } return { mode: 'default', @@ -294,6 +334,126 @@ export class DataDbPreflightService { }; } + private async getActiveMigrationSummary( + spaceId: string + ): Promise { + const migrationClient = (this.prismaService as IDataDbSummaryPrisma | undefined) + ?.spaceDataDbMigrationJob; + if (!migrationClient) { + return null; + } + + const job = await migrationClient + .findFirst({ + where: { + spaceId, + state: { in: [...activeSpaceDataDbMigrationStates] }, + }, + include: { targetConnection: true }, + orderBy: { createdTime: 'desc' }, + }) + .catch((error) => { + if (this.isMissingMigrationJobTableError(error)) { + return null; + } + throw error; + }); + const resolvedJob = + job ?? + (await migrationClient + .findMany({ + where: { + state: { in: [...activeSpaceDataDbMigrationStates] }, + }, + include: { targetConnection: true }, + orderBy: { createdTime: 'desc' }, + }) + .then((jobs) => + jobs.find((candidate) => + this.migrationInventoryContainsSpace(candidate.inventory, spaceId) + ) + ) + .catch((error) => { + if (this.isMissingMigrationJobTableError(error)) { + return null; + } + throw error; + })); + if (!resolvedJob) return null; + + const connection = resolvedJob.targetConnection; + const inventory = + resolvedJob.inventory && typeof resolvedJob.inventory === 'object' + ? (resolvedJob.inventory as { relatedSpaces?: IDataDbConnectionSummaryVo['relatedSpaces'] }) + : {}; + const migration = { + jobId: resolvedJob.id, + state: resolvedJob.state as NonNullable['state'], + targetInternalSchema: resolvedJob.targetInternalSchema, + switchOnCompletion: resolvedJob.switchOnCompletion === true, + lastError: resolvedJob.lastError, + }; + + if (resolvedJob.switchOnCompletion !== true) { + return { + mode: 'default', + state: 'ready', + migration, + relatedSpaces: inventory.relatedSpaces, + }; + } + + return { + mode: 'byodb', + state: 'migrating', + provider: connection?.provider ?? 'postgres', + displayHost: connection?.displayHost ?? undefined, + displayDatabase: connection?.displayDatabase ?? undefined, + internalSchema: connection?.internalSchema ?? resolvedJob.targetInternalSchema, + schemaVersion: connection?.schemaVersion ?? null, + lastValidatedAt: connection?.lastValidatedAt?.toISOString(), + lastError: resolvedJob.lastError ?? connection?.lastError ?? undefined, + capabilities: connection?.capabilities as + | IDataDbConnectionSummaryVo['capabilities'] + | undefined, + migration, + relatedSpaces: inventory.relatedSpaces, + }; + } + + private migrationInventoryContainsSpace(inventory: unknown, spaceId: string) { + if (!inventory || typeof inventory !== 'object') { + return false; + } + const candidate = inventory as { spaceIds?: unknown; relatedSpaces?: { spaces?: unknown } }; + return ( + (Array.isArray(candidate.spaceIds) && candidate.spaceIds.includes(spaceId)) || + (Array.isArray(candidate.relatedSpaces?.spaces) && + candidate.relatedSpaces.spaces.some( + (space) => + space && + typeof space === 'object' && + (space as { spaceId?: unknown }).spaceId === spaceId + )) + ); + } + + private isMissingMigrationJobTableError(error: unknown) { + const code = + typeof error === 'object' && error !== null && 'code' in error + ? String((error as { code?: unknown }).code) + : ''; + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('space_data_db_migration_job') && + (code === 'P2021' || + code === 'P2022' || + code === '42P01' || + message.includes('does not exist') || + message.includes('relation')) + ); + } + private buildResult({ errors, maskedUrl, diff --git a/apps/nestjs-backend/src/features/space/space-data-db-admin-gate.test.ts b/apps/nestjs-backend/src/features/space/space-data-db-admin-gate.test.ts new file mode 100644 index 0000000000..795092f093 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-admin-gate.test.ts @@ -0,0 +1,124 @@ +import { HttpErrorCode } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SpaceController } from './space.controller'; + +vi.mock('@teable/db-main-prisma', () => ({ + MetaPrismaService: class MetaPrismaService {}, + Prisma: {}, + PrismaModule: class PrismaModule {}, + PrismaService: class PrismaService {}, + ProvisionState: {}, + getDatabaseUrl: vi.fn(), +})); +vi.mock('@teable/db-data-prisma', () => ({ + DataPrismaModule: class DataPrismaModule {}, + DataPrismaService: class DataPrismaService {}, + PrismaClient: class PrismaClient {}, + getMetaDatabaseUrl: vi.fn(), +})); +vi.mock('@prisma/client', () => ({ + Prisma: {}, + PrismaClient: class PrismaClient {}, +})); +vi.mock('../invitation/invitation.service', () => ({ + InvitationService: class InvitationService {}, +})); +vi.mock('../collaborator/collaborator.service', () => ({ + CollaboratorService: class CollaboratorService {}, +})); +vi.mock('./data-db-binding.service', () => ({ + DataDbBindingService: class DataDbBindingService {}, +})); +vi.mock('./data-db-preflight.service', () => ({ + DataDbPreflightService: class DataDbPreflightService {}, +})); +vi.mock('./space-data-db-migration.service', () => ({ + SpaceDataDbMigrationService: class SpaceDataDbMigrationService {}, +})); +vi.mock('./space.service', () => ({ + SpaceService: class SpaceService {}, +})); + +describe('SpaceController data DB admin gate', () => { + let controller: SpaceController; + const dataDbPreflightService = { + preflight: vi.fn(), + getSummary: vi.fn(), + }; + const dataDbBindingService = { + updateBindingForSpace: vi.fn(), + }; + const cls = { + get: vi.fn().mockReturnValue('usrxxx'), + }; + const spaceDataDbMigrationService = { + cancelMigrationForSpace: vi.fn(), + getMigrationJobStatus: vi.fn(), + rollbackMigrationForSpace: vi.fn(), + }; + + beforeEach(() => { + dataDbPreflightService.preflight.mockReset(); + dataDbPreflightService.getSummary.mockReset(); + dataDbBindingService.updateBindingForSpace.mockReset(); + cls.get.mockClear(); + spaceDataDbMigrationService.cancelMigrationForSpace.mockReset(); + spaceDataDbMigrationService.getMigrationJobStatus.mockReset(); + spaceDataDbMigrationService.rollbackMigrationForSpace.mockReset(); + controller = new SpaceController( + {} as never, + {} as never, + {} as never, + dataDbPreflightService as never, + dataDbBindingService as never, + cls as never, + spaceDataDbMigrationService as never + ); + }); + + it('rejects migrate-space preflight requests from the non-admin space API', async () => { + await expect( + controller.preflightDataDb({ + url: 'postgresql://teable:secret@example.com:5432/teable_data', + targetMode: 'migrate-space', + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + data: { errorCode: 'SPACE_DATA_DB_ADMIN_ONLY' }, + }); + expect(dataDbPreflightService.preflight).not.toHaveBeenCalled(); + }); + + it('rejects migrate-space updates from the non-admin space API', async () => { + await expect( + controller.updateSpaceDataDb('spcxxx', { + url: 'postgresql://teable:secret@example.com:5432/teable_data', + targetMode: 'migrate-space', + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + data: { errorCode: 'SPACE_DATA_DB_ADMIN_ONLY' }, + }); + expect(dataDbBindingService.updateBindingForSpace).not.toHaveBeenCalled(); + }); + + it('rejects migration job operations from the non-admin space API', async () => { + await expect(controller.getSpaceDataDbMigration('spcxxx', 'sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + data: { errorCode: 'SPACE_DATA_DB_ADMIN_ONLY' }, + }); + await expect(controller.cancelSpaceDataDbMigration('spcxxx', 'sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + data: { errorCode: 'SPACE_DATA_DB_ADMIN_ONLY' }, + }); + await expect( + controller.rollbackSpaceDataDbMigration('spcxxx', 'sdmjxxx') + ).rejects.toMatchObject({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + data: { errorCode: 'SPACE_DATA_DB_ADMIN_ONLY' }, + }); + expect(spaceDataDbMigrationService.getMigrationJobStatus).not.toHaveBeenCalled(); + expect(spaceDataDbMigrationService.cancelMigrationForSpace).not.toHaveBeenCalled(); + expect(spaceDataDbMigrationService.rollbackMigrationForSpace).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-copy-plan.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-copy-plan.spec.ts new file mode 100644 index 0000000000..0e9be30f87 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-copy-plan.spec.ts @@ -0,0 +1,459 @@ +import { describe, expect, it } from 'vitest'; +import { + buildBaseSchemaDumpRestorePlan, + buildBaseSchemaDumpStreamRestorePlan, + buildBaseSchemaPgcopydbPlan, + buildBaseSchemaRestorePlan, + buildMigrationSharedTablePostgresFdwCopyPlans, + buildMigrationSharedTablePsqlCopyPlans, + buildSharedTablePostgresFdwCopyPlan, + buildSharedTableCopyPlan, + buildSharedTablePsqlCopyPlan, + postgresToolUrl, +} from './space-data-db-copy-plan'; + +const sourceHost = 'source.example'; +const targetHost = 'target.example'; +const sourceUrl = `postgresql://${sourceHost}/teable`; +const targetUrl = `postgresql://${targetHost}/teable`; +const workDir = '/tmp/sdmjxxx'; +const dumpFile = '/tmp/sdmjxxx/base-schemas.dump'; +const noOwnerArg = '--no-owner'; +const noAclArg = '--no-acl'; +const jobsArg = '--jobs'; +const pgcopydbWorkDirectory = '/tmp/sdmjxxx/pgcopydb-base-schemas'; +const pgcopydbFilterFile = '/tmp/sdmjxxx/pgcopydb-base-schemas.filter.ini'; +const psqlCommandArgs = ['--no-psqlrc', '--set', 'ON_ERROR_STOP=1', '--command']; +const fdwSourceUrl = + 'postgresql://source_user:source_secret@source.example:15432/teable_source?sslmode=require'; + +describe('space data DB copy plan', () => { + it('strips Prisma-only query params from URLs passed to PostgreSQL client tools', () => { + expect( + postgresToolUrl( + 'postgresql://teable:secret@127.0.0.1:5432/teable?schema=public&statement_cache_size=1&sslmode=require' + ) + ).toBe('postgresql://teable:secret@127.0.0.1:5432/teable?sslmode=require'); + }); + + it('percent-encodes user info for PostgreSQL client tool URIs', () => { + expect( + postgresToolUrl( + 'postgresql://user.name:pa%ss$word@target.example:5432/teable?schema=public&sslmode=require' + ) + ).toBe('postgresql://user.name:pa%25ss%24word@target.example:5432/teable?sslmode=require'); + }); + + it('does not double-encode already encoded PostgreSQL client tool user info', () => { + expect(postgresToolUrl('postgresql://user%40name:pa%25ss%24word@target.example/teable')).toBe( + 'postgresql://user%40name:pa%25ss%24word@target.example/teable' + ); + }); + + it('builds pg_dump and pg_restore spawn-array plans for base schemas', () => { + const plan = buildBaseSchemaDumpRestorePlan({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + workDir, + jobs: 4, + }); + + expect(plan.dumpFile).toBe(dumpFile); + expect(plan.dump).toEqual({ + command: 'pg_dump', + args: [ + '--format=custom', + noOwnerArg, + noAclArg, + '--file', + dumpFile, + '--schema', + '"bseaaa"', + '--schema', + '"bsebbb"', + sourceUrl, + ], + }); + expect(plan.restore).toEqual({ + command: 'pg_restore', + args: [noOwnerArg, noAclArg, jobsArg, '4', '--dbname', targetUrl, dumpFile], + }); + }); + + it('requires at least one schema for base schema copy planning', () => { + expect(() => + buildBaseSchemaDumpRestorePlan({ + sourceUrl, + targetUrl, + schemaNames: [], + workDir, + }) + ).toThrow('At least one base schema'); + }); + + it('builds streaming pg_dump to pg_restore plans for base schemas', () => { + const plan = buildBaseSchemaDumpStreamRestorePlan({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + }); + + expect(plan).toEqual({ + label: 'base-schemas', + schemaNames: ['bseaaa', 'bsebbb'], + source: { + command: 'pg_dump', + args: [ + '--format=custom', + noOwnerArg, + noAclArg, + '--schema', + '"bseaaa"', + '--schema', + '"bsebbb"', + sourceUrl, + ], + }, + target: { + command: 'pg_restore', + args: [noOwnerArg, noAclArg, '--dbname', targetUrl], + }, + }); + }); + + it('adds a pg_restore use-list file when base schema restore needs filtered TOC entries', () => { + const plan = buildBaseSchemaRestorePlan({ + targetUrl, + dumpFile, + jobs: 2, + restoreListFile: '/tmp/sdmjxxx/base-schemas.restore.list', + }); + + expect(plan).toEqual({ + command: 'pg_restore', + args: [ + noOwnerArg, + noAclArg, + jobsArg, + '2', + '--use-list', + '/tmp/sdmjxxx/base-schemas.restore.list', + '--dbname', + targetUrl, + dumpFile, + ], + }); + }); + + it('uses PostgreSQL-tool-compatible URLs for physical copy commands', () => { + const plan = buildBaseSchemaDumpRestorePlan({ + sourceUrl: + 'postgresql://source.example/teable?schema=public&statement_cache_size=1&sslmode=require', + targetUrl: + 'postgresql://target.example/teable?schema=public&connection_limit=10&sslmode=require', + schemaNames: ['bsexxx'], + workDir, + }); + + expect(plan.dump.args.at(-1)).toBe('postgresql://source.example/teable?sslmode=require'); + expect(plan.dump.args).toContain('"bsexxx"'); + expect(plan.restore.args).toContain('postgresql://target.example/teable?sslmode=require'); + }); + + it('uses percent-encoded PostgreSQL-tool-compatible URLs in physical copy commands', () => { + const rawSourceUrl = 'postgresql://source_user:source%secret$@source.example/teable'; + const rawTargetUrl = 'postgresql://target_user:target%secret$@target.example/teable'; + const safeSourceUrl = 'postgresql://source_user:source%25secret%24@source.example/teable'; + const safeTargetUrl = 'postgresql://target_user:target%25secret%24@target.example/teable'; + + const dumpRestorePlan = buildBaseSchemaDumpRestorePlan({ + sourceUrl: rawSourceUrl, + targetUrl: rawTargetUrl, + schemaNames: ['bsexxx'], + workDir, + }); + const streamPlan = buildBaseSchemaDumpStreamRestorePlan({ + sourceUrl: rawSourceUrl, + targetUrl: rawTargetUrl, + schemaNames: ['bsexxx'], + }); + const psqlPlan = buildSharedTablePsqlCopyPlan({ + sourceUrl: rawSourceUrl, + targetUrl: rawTargetUrl, + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + table: 'record_history', + columns: ['id'], + whereSql: `"table_id" = ANY(ARRAY['tblxxx']::text[])`, + }); + const pgcopydbPlan = buildBaseSchemaPgcopydbPlan({ + sourceUrl: rawSourceUrl, + targetUrl: rawTargetUrl, + schemaNames: ['bsexxx'], + workDir, + }); + + expect(dumpRestorePlan.dump.args.at(-1)).toBe(safeSourceUrl); + expect(dumpRestorePlan.restore.args).toContain(safeTargetUrl); + expect(streamPlan.source.args.at(-1)).toBe(safeSourceUrl); + expect(streamPlan.target.args).toContain(safeTargetUrl); + expect(psqlPlan.source.args.at(-1)).toBe(safeSourceUrl); + expect(psqlPlan.target.args.at(-1)).toBe(safeTargetUrl); + expect(pgcopydbPlan.copy.args).toContain(safeSourceUrl); + expect(pgcopydbPlan.copy.args).toContain(safeTargetUrl); + }); + + it('builds an explicitly selected pgcopydb plan with a schema include filter', () => { + const plan = buildBaseSchemaPgcopydbPlan({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + workDir, + jobs: 3, + }); + + expect(plan.workDirectory).toBe(pgcopydbWorkDirectory); + expect(plan.filterFile).toBe(pgcopydbFilterFile); + expect(plan.filterFileContent).toBe('[include-only-schema]\n"bseaaa"\n"bsebbb"\n'); + expect(plan.copy).toEqual({ + command: 'pgcopydb', + args: [ + 'copy', + 'db', + '--source', + sourceUrl, + '--target', + targetUrl, + '--dir', + pgcopydbWorkDirectory, + '--table-jobs', + '3', + '--index-jobs', + '3', + '--restore-jobs', + '3', + noOwnerArg, + noAclArg, + '--skip-large-objects', + '--filters', + pgcopydbFilterFile, + '--fail-fast', + '--restart', + ], + }); + }); + + it('uses PostgreSQL-tool-compatible URLs for pgcopydb commands', () => { + const plan = buildBaseSchemaPgcopydbPlan({ + sourceUrl: 'postgresql://source.example/teable?schema=public', + targetUrl: 'postgresql://target.example/teable?statement_cache_size=1', + schemaNames: ['bsexxx'], + workDir, + }); + + expect(plan.copy.args).toContain('postgresql://source.example/teable'); + expect(plan.copy.args).toContain('postgresql://target.example/teable'); + }); + + it('builds streaming COPY plans with explicit columns and scoped filters', () => { + const plan = buildSharedTableCopyPlan({ + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + table: 'record_history', + columns: ['id', 'table_id', 'record_id', 'snapshot'], + whereSql: '"table_id" = ANY($1::text[])', + sourceBindings: [['tblxxx', 'tblyyy']], + }); + + expect(plan).toEqual({ + sourceSql: + 'COPY (SELECT "id", "table_id", "record_id", "snapshot" FROM "public"."record_history" WHERE "table_id" = ANY($1::text[])) TO STDOUT', + sourceBindings: [['tblxxx', 'tblyyy']], + targetSql: + 'COPY "teable_meta_target"."record_history" ("id", "table_id", "record_id", "snapshot") FROM STDIN', + }); + }); + + it('builds psql process plans for streaming shared-table COPY', () => { + const plan = buildSharedTablePsqlCopyPlan({ + sourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + table: 'record_history', + columns: ['id', 'table_id', 'record_id'], + whereSql: `"table_id" = ANY(ARRAY['tblxxx']::text[])`, + }); + + expect(plan.table).toBe('record_history'); + expect(plan.sourceSql).toBe( + `COPY (SELECT "id", "table_id", "record_id" FROM "public"."record_history" WHERE "table_id" = ANY(ARRAY['tblxxx']::text[])) TO STDOUT` + ); + expect(plan.targetSql).toBe( + 'COPY "teable_meta_target"."record_history" ("id", "table_id", "record_id") FROM STDIN' + ); + expect(plan.source).toEqual({ + command: 'psql', + args: [...psqlCommandArgs, plan.sourceSql, sourceUrl], + }); + expect(plan.target).toEqual({ + command: 'psql', + args: [...psqlCommandArgs, plan.targetSql, targetUrl], + }); + }); + + it('uses PostgreSQL-tool-compatible URLs for psql shared-table copy commands', () => { + const plan = buildSharedTablePsqlCopyPlan({ + sourceUrl: 'postgresql://source.example/teable?schema=public', + targetUrl: 'postgresql://target.example/teable?statement_cache_size=1', + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + table: 'record_history', + columns: ['id'], + whereSql: `"table_id" = ANY(ARRAY['tblxxx']::text[])`, + }); + + expect(plan.source.args.at(-1)).toBe('postgresql://source.example/teable'); + expect(plan.target.args.at(-1)).toBe('postgresql://target.example/teable'); + }); + + it('builds postgres_fdw plans for explicitly selected shared-table DB-to-DB inserts', () => { + const plan = buildSharedTablePostgresFdwCopyPlan({ + sourceUrl: fdwSourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + table: 'record_history', + columns: ['id', 'table_id', 'record_id'], + whereSql: `"table_id" = ANY(ARRAY['tblxxx']::text[])`, + fdwSchema: 'sdmjxxx_fdw_0', + serverName: 'sdmjxxx_srv_0', + }); + + expect(plan.table).toBe('record_history'); + expect(plan.sql).toContain('CREATE EXTENSION IF NOT EXISTS postgres_fdw;'); + expect(plan.sql).toContain('CREATE SCHEMA "sdmjxxx_fdw_0";'); + expect(plan.sql).toContain( + `CREATE SERVER "sdmjxxx_srv_0" FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'source.example', port '15432', dbname 'teable_source', sslmode 'require');` + ); + expect(plan.sql).toContain( + `CREATE USER MAPPING FOR CURRENT_USER SERVER "sdmjxxx_srv_0" OPTIONS (user 'source_user', password 'source_secret');` + ); + expect(plan.sql).toContain( + 'IMPORT FOREIGN SCHEMA "public" LIMIT TO ("record_history") FROM SERVER "sdmjxxx_srv_0" INTO "sdmjxxx_fdw_0";' + ); + expect(plan.sql).toContain( + `INSERT INTO "teable_meta_target"."record_history" ("id", "table_id", "record_id") SELECT "id", "table_id", "record_id" FROM "sdmjxxx_fdw_0"."record_history" WHERE "table_id" = ANY(ARRAY['tblxxx']::text[]);` + ); + expect(plan.sql).toContain('DROP SERVER "sdmjxxx_srv_0" CASCADE;'); + expect(plan.target).toEqual({ + command: 'psql', + args: [...psqlCommandArgs, plan.sql, targetUrl], + }); + }); + + it('builds scoped psql COPY plans for all migration shared tables', () => { + const plans = buildMigrationSharedTablePsqlCopyPlans({ + sourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + spaceId: "spc'x", + baseIds: ['bsexxx', 'bseyyy'], + tableIds: ['tblxxx', 'tblyyy'], + }); + + expect(plans.map((plan) => plan.table)).toEqual([ + 'record_history', + 'table_trash', + 'record_trash', + 'computed_update_outbox', + 'computed_update_dead_letter', + 'computed_update_outbox_seed', + 'computed_update_pause_scope', + '__undo_log', + ]); + expect(plans[0].sourceSql).toContain(`"table_id" = ANY(ARRAY['tblxxx', 'tblyyy']::text[])`); + expect(plans[3].sourceSql).toContain(`"base_id" = ANY(ARRAY['bsexxx', 'bseyyy']::text[])`); + expect(plans[5].sourceSql).toContain( + 'FROM "public"."computed_update_outbox" WHERE "base_id" = ANY' + ); + expect(plans[6].sourceSql).toContain(`"scope_id" = ANY(ARRAY['spc''x']::text[])`); + expect(plans[7].sourceSql).toContain( + `split_part("table_name", '.', 1) = ANY(ARRAY['bsexxx', 'bseyyy']::text[])` + ); + expect(plans.every((plan) => plan.source.args.includes(sourceUrl))).toBe(true); + expect(plans.every((plan) => plan.target.args.includes(targetUrl))).toBe(true); + }); + + it('uses deleted-table shared scope only for shared rows that reference table_id directly', () => { + const plans = buildMigrationSharedTablePsqlCopyPlans({ + sourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + spaceId: 'spcxxx', + baseIds: ['bsexxx'], + tableIds: ['tblactive'], + sharedTableIds: ['tblactive', 'tbldeleted'], + }); + + expect(plans.find((plan) => plan.table === 'record_history')?.sourceSql).toContain( + `"table_id" = ANY(ARRAY['tblactive', 'tbldeleted']::text[])` + ); + expect(plans.find((plan) => plan.table === 'table_trash')?.sourceSql).toContain( + `"table_id" = ANY(ARRAY['tblactive', 'tbldeleted']::text[])` + ); + expect(plans.find((plan) => plan.table === 'record_trash')?.sourceSql).toContain( + `"table_id" = ANY(ARRAY['tblactive', 'tbldeleted']::text[])` + ); + expect(plans.find((plan) => plan.table === 'computed_update_outbox_seed')?.sourceSql).toContain( + `"table_id" = ANY(ARRAY['tblactive']::text[])` + ); + expect(plans.find((plan) => plan.table === 'computed_update_pause_scope')?.sourceSql).toContain( + `"scope_id" = ANY(ARRAY['tblactive', 'tbldeleted']::text[])` + ); + }); + + it('builds scoped postgres_fdw plans for all migration shared tables', () => { + const plans = buildMigrationSharedTablePostgresFdwCopyPlans({ + sourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + spaceId: "spc'x", + baseIds: ['bsexxx', 'bseyyy'], + tableIds: ['tblxxx', 'tblyyy'], + fdwSchemaPrefix: 'sdmjxxx_fdw', + serverNamePrefix: 'sdmjxxx_srv', + }); + + expect(plans.map((plan) => plan.table)).toEqual([ + 'record_history', + 'table_trash', + 'record_trash', + 'computed_update_outbox', + 'computed_update_dead_letter', + 'computed_update_outbox_seed', + 'computed_update_pause_scope', + '__undo_log', + ]); + expect(plans[0].sql).toContain('FROM "sdmjxxx_fdw_0"."record_history"'); + expect(plans[5].sql).toContain('FROM "sdmjxxx_fdw_5"."computed_update_outbox"'); + expect(plans[6].sql).toContain(`"scope_id" = ANY(ARRAY['spc''x']::text[])`); + expect(plans.every((plan) => plan.target.args.includes(targetUrl))).toBe(true); + }); + + it('rejects unscoped shared table copy plans', () => { + expect(() => + buildSharedTableCopyPlan({ + sourceSchema: 'public', + targetSchema: 'teable_meta_target', + table: 'record_history', + columns: ['id'], + whereSql: '', + }) + ).toThrow('requires a scoped WHERE clause'); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-copy-plan.ts b/apps/nestjs-backend/src/features/space/space-data-db-copy-plan.ts new file mode 100644 index 0000000000..6a6ee210ea --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-copy-plan.ts @@ -0,0 +1,590 @@ +import path from 'path'; + +export type ISpaceDataDbProcessPlan = { + command: string; + args: string[]; +}; + +export type ISpaceDataDbProcessPipelinePlan = { + source: ISpaceDataDbProcessPlan; + target: ISpaceDataDbProcessPlan; + label?: string; +}; + +export type ISpaceDataDbDumpRestorePlan = { + dumpFile: string; + dump: ISpaceDataDbProcessPlan; + restoreList?: ISpaceDataDbProcessPlan; + restoreListFile?: string; + restore: ISpaceDataDbProcessPlan; +}; + +export type ISpaceDataDbDumpStreamRestorePlan = ISpaceDataDbProcessPipelinePlan & { + schemaNames: string[]; +}; + +export type ISpaceDataDbPgcopydbPlan = { + workDirectory: string; + filterFile: string; + filterFileContent: string; + copy: ISpaceDataDbProcessPlan; +}; + +export type ISharedTableCopyPlan = { + sourceSql: string; + sourceBindings: unknown[]; + targetSql: string; +}; + +export type ISharedTablePsqlCopyPlan = ISpaceDataDbProcessPipelinePlan & { + table: string; + sourceSql: string; + targetSql: string; +}; + +export type ISharedTablePostgresFdwCopyPlan = { + table: string; + sql: string; + target: ISpaceDataDbProcessPlan; +}; + +const quoteIdent = (identifier: string) => `"${identifier.replace(/"/g, '""')}"`; + +const qualify = (schema: string, table: string) => `${quoteIdent(schema)}.${quoteIdent(table)}`; + +const normalizeJobs = (jobs?: number) => Math.max(1, Math.floor(jobs ?? 1)); + +const literal = (value: string) => `'${value.replace(/'/g, "''")}'`; +const noOwnerArg = '--no-owner'; +const noAclArg = '--no-acl'; +const jobsArg = '--jobs'; +const pgcopydbBaseSchemasDir = 'pgcopydb-base-schemas'; +const pgcopydbFilterFileName = 'pgcopydb-base-schemas.filter.ini'; +const includeOnlySchemaSection = '[include-only-schema]'; +const prismaOnlyPostgresUrlParams = [ + 'schema', + 'statement_cache_size', + 'connection_limit', + 'pool_timeout', + 'pgbouncer', +] as const; + +const textArray = (values: string[]) => + values.length ? `ARRAY[${values.map(literal).join(', ')}]::text[]` : 'ARRAY[]::text[]'; + +const textArrayPredicate = (column: string, values: string[]) => + `${quoteIdent(column)} = ANY(${textArray(values)})`; + +const decodePostgresToolUserInfoComponent = (value: string) => { + try { + return decodeURIComponent(value); + } catch { + const escapedInvalidPercent = value.replace(/%(?![0-9a-f]{2})/gi, '%25'); + try { + return decodeURIComponent(escapedInvalidPercent); + } catch { + return value; + } + } +}; + +const encodePostgresToolUserInfoComponent = (value: string) => + encodeURIComponent(decodePostgresToolUserInfoComponent(value)); + +const postgresToolUserInfo = (parsed: URL) => { + if (!parsed.username) { + return ''; + } + const username = encodePostgresToolUserInfoComponent(parsed.username); + const password = parsed.password + ? `:${encodePostgresToolUserInfoComponent(parsed.password)}` + : ''; + return `${username}${password}@`; +}; + +export const postgresToolUrl = (url: string) => { + const parsed = new URL(url); + for (const param of prismaOnlyPostgresUrlParams) { + parsed.searchParams.delete(param); + } + return `${parsed.protocol}//${postgresToolUserInfo(parsed)}${parsed.host}${parsed.pathname}${ + parsed.search + }${parsed.hash}`; +}; + +export const buildBaseSchemaRestoreListPlan = (dumpFile: string): ISpaceDataDbProcessPlan => ({ + command: 'pg_restore', + args: ['--list', dumpFile], +}); + +export const buildBaseSchemaRestorePlan = (input: { + targetUrl: string; + dumpFile: string; + jobs?: number; + restoreListFile?: string; +}): ISpaceDataDbProcessPlan => { + const jobs = String(normalizeJobs(input.jobs)); + const targetUrl = postgresToolUrl(input.targetUrl); + return { + command: 'pg_restore', + args: [ + noOwnerArg, + noAclArg, + jobsArg, + jobs, + ...(input.restoreListFile ? ['--use-list', input.restoreListFile] : []), + '--dbname', + targetUrl, + input.dumpFile, + ], + }; +}; + +const psqlArgs = (url: string, command: string) => [ + '--no-psqlrc', + '--set', + 'ON_ERROR_STOP=1', + '--command', + command, + postgresToolUrl(url), +]; + +const parsePostgresUrl = (url: string) => { + const parsed = new URL(url); + const dbname = decodeURIComponent(parsed.pathname.replace(/^\//, '')); + if (!dbname) { + throw new Error('Source PostgreSQL URL must include a database name for postgres_fdw copy'); + } + + return { + host: parsed.hostname, + port: parsed.port || '5432', + dbname, + user: decodeURIComponent(parsed.username), + password: decodeURIComponent(parsed.password), + sslmode: parsed.searchParams.get('sslmode') ?? undefined, + }; +}; + +const fdwOptions = (options: Record) => + Object.entries(options) + .filter((entry): entry is [string, string] => Boolean(entry[1])) + .map(([key, value]) => `${key} ${literal(value)}`) + .join(', '); + +export const buildBaseSchemaDumpRestorePlan = (input: { + sourceUrl: string; + targetUrl: string; + schemaNames: string[]; + workDir: string; + jobs?: number; +}): ISpaceDataDbDumpRestorePlan => { + const schemaNames = [...input.schemaNames].sort(); + if (!schemaNames.length) { + throw new Error('At least one base schema is required for pg_dump planning'); + } + + const dumpFile = path.join(input.workDir, 'base-schemas.dump'); + const schemaArgs = schemaNames.flatMap((schemaName) => ['--schema', quoteIdent(schemaName)]); + const sourceUrl = postgresToolUrl(input.sourceUrl); + const targetUrl = postgresToolUrl(input.targetUrl); + + return { + dumpFile, + dump: { + command: 'pg_dump', + args: ['--format=custom', noOwnerArg, noAclArg, '--file', dumpFile, ...schemaArgs, sourceUrl], + }, + restore: buildBaseSchemaRestorePlan({ + targetUrl, + dumpFile, + jobs: input.jobs, + }), + }; +}; + +export const buildBaseSchemaDumpStreamRestorePlan = (input: { + sourceUrl: string; + targetUrl: string; + schemaNames: string[]; +}): ISpaceDataDbDumpStreamRestorePlan => { + const schemaNames = [...input.schemaNames].sort(); + if (!schemaNames.length) { + throw new Error('At least one base schema is required for pg_dump planning'); + } + + const schemaArgs = schemaNames.flatMap((schemaName) => ['--schema', quoteIdent(schemaName)]); + const sourceUrl = postgresToolUrl(input.sourceUrl); + const targetUrl = postgresToolUrl(input.targetUrl); + + return { + label: 'base-schemas', + schemaNames, + source: { + command: 'pg_dump', + args: ['--format=custom', noOwnerArg, noAclArg, ...schemaArgs, sourceUrl], + }, + target: { + command: 'pg_restore', + args: [noOwnerArg, noAclArg, '--dbname', targetUrl], + }, + }; +}; + +export const buildBaseSchemaPgcopydbPlan = (input: { + sourceUrl: string; + targetUrl: string; + schemaNames: string[]; + workDir: string; + jobs?: number; +}): ISpaceDataDbPgcopydbPlan => { + const schemaNames = [...input.schemaNames].sort(); + if (!schemaNames.length) { + throw new Error('At least one base schema is required for pgcopydb planning'); + } + + const jobs = String(normalizeJobs(input.jobs)); + const workDirectory = path.join(input.workDir, pgcopydbBaseSchemasDir); + const filterFile = path.join(input.workDir, pgcopydbFilterFileName); + const filterFileContent = [includeOnlySchemaSection, ...schemaNames.map(quoteIdent), ''].join( + '\n' + ); + + return { + workDirectory, + filterFile, + filterFileContent, + copy: { + command: 'pgcopydb', + args: [ + 'copy', + 'db', + '--source', + postgresToolUrl(input.sourceUrl), + '--target', + postgresToolUrl(input.targetUrl), + '--dir', + workDirectory, + '--table-jobs', + jobs, + '--index-jobs', + jobs, + '--restore-jobs', + jobs, + noOwnerArg, + noAclArg, + '--skip-large-objects', + '--filters', + filterFile, + '--fail-fast', + '--restart', + ], + }, + }; +}; + +export const buildSharedTableCopyPlan = (input: { + sourceSchema: string; + targetSchema: string; + table: string; + columns: string[]; + whereSql: string; + sourceBindings?: unknown[]; +}): ISharedTableCopyPlan => { + if (!input.columns.length) { + throw new Error(`Shared table ${input.table} requires an explicit column list`); + } + if (!input.whereSql.trim()) { + throw new Error(`Shared table ${input.table} requires a scoped WHERE clause`); + } + + const columns = input.columns.map(quoteIdent).join(', '); + return { + sourceSql: `COPY (SELECT ${columns} FROM ${qualify(input.sourceSchema, input.table)} WHERE ${ + input.whereSql + }) TO STDOUT`, + sourceBindings: input.sourceBindings ?? [], + targetSql: `COPY ${qualify(input.targetSchema, input.table)} (${columns}) FROM STDIN`, + }; +}; + +export const buildSharedTablePsqlCopyPlan = (input: { + sourceUrl: string; + targetUrl: string; + sourceSchema: string; + targetSchema: string; + table: string; + columns: string[]; + whereSql: string; +}): ISharedTablePsqlCopyPlan => { + const plan = buildSharedTableCopyPlan(input); + return { + table: input.table, + label: `shared-table:${input.table}`, + sourceSql: plan.sourceSql, + targetSql: plan.targetSql, + source: { + command: 'psql', + args: psqlArgs(input.sourceUrl, plan.sourceSql), + }, + target: { + command: 'psql', + args: psqlArgs(input.targetUrl, plan.targetSql), + }, + }; +}; + +export const buildSharedTablePostgresFdwCopyPlan = (input: { + sourceUrl: string; + targetUrl: string; + sourceSchema: string; + targetSchema: string; + table: string; + columns: string[]; + whereSql: string; + fdwSchema: string; + serverName: string; +}): ISharedTablePostgresFdwCopyPlan => { + if (!input.columns.length) { + throw new Error(`Shared table ${input.table} requires an explicit column list`); + } + if (!input.whereSql.trim()) { + throw new Error(`Shared table ${input.table} requires a scoped WHERE clause`); + } + + const source = parsePostgresUrl(input.sourceUrl); + const columns = input.columns.map(quoteIdent).join(', '); + const serverOptions = fdwOptions({ + host: source.host, + port: source.port, + dbname: source.dbname, + sslmode: source.sslmode, + }); + const userMappingOptions = fdwOptions({ + user: source.user, + password: source.password, + }); + const importLimit = quoteIdent(input.table); + const targetTable = qualify(input.targetSchema, input.table); + const foreignTable = qualify(input.fdwSchema, input.table); + + const sql = [ + 'BEGIN', + 'CREATE EXTENSION IF NOT EXISTS postgres_fdw', + `CREATE SCHEMA ${quoteIdent(input.fdwSchema)}`, + `CREATE SERVER ${quoteIdent( + input.serverName + )} FOREIGN DATA WRAPPER postgres_fdw OPTIONS (${serverOptions})`, + userMappingOptions + ? `CREATE USER MAPPING FOR CURRENT_USER SERVER ${quoteIdent( + input.serverName + )} OPTIONS (${userMappingOptions})` + : '', + `IMPORT FOREIGN SCHEMA ${quoteIdent(input.sourceSchema)} LIMIT TO (${importLimit}) FROM SERVER ${quoteIdent( + input.serverName + )} INTO ${quoteIdent(input.fdwSchema)}`, + `INSERT INTO ${targetTable} (${columns}) SELECT ${columns} FROM ${foreignTable} WHERE ${input.whereSql}`, + `DROP SERVER ${quoteIdent(input.serverName)} CASCADE`, + `DROP SCHEMA ${quoteIdent(input.fdwSchema)} CASCADE`, + 'COMMIT', + ] + .filter(Boolean) + .map((statement) => `${statement};`) + .join('\n'); + + return { + table: input.table, + sql, + target: { + command: 'psql', + args: psqlArgs(input.targetUrl, sql), + }, + }; +}; + +const computedOutboxColumns = [ + 'id', + 'base_id', + 'seed_table_id', + 'seed_record_ids', + 'change_type', + 'steps', + 'edges', + 'status', + 'attempts', + 'max_attempts', + 'next_run_at', + 'locked_at', + 'locked_by', + 'last_error', + 'estimated_complexity', + 'plan_hash', + 'dirty_stats', + 'run_id', + 'origin_run_ids', + 'run_total_steps', + 'run_completed_steps_before', + 'affected_table_ids', + 'affected_field_ids', + 'sync_max_level', + 'created_at', + 'updated_at', +]; + +const recordHistoryColumns = [ + 'id', + 'table_id', + 'record_id', + 'field_id', + 'before', + 'after', + 'created_time', + 'created_by', +]; + +const buildMigrationSharedTableDefinitions = (input: { + sourceSchema: string; + spaceId: string; + spaceIds?: string[]; + baseIds: string[]; + tableIds: string[]; + sharedTableIds?: string[]; +}) => { + const sharedTableIds = input.sharedTableIds?.length ? input.sharedTableIds : input.tableIds; + const tablePredicate = textArrayPredicate('table_id', sharedTableIds); + const basePredicate = textArrayPredicate('base_id', input.baseIds); + const spaceIds = input.spaceIds?.length ? input.spaceIds : [input.spaceId]; + const outboxSeedPredicate = [ + textArrayPredicate('table_id', input.tableIds), + `"task_id" IN (SELECT "id" FROM ${qualify( + input.sourceSchema, + 'computed_update_outbox' + )} WHERE ${basePredicate})`, + ].join(' AND '); + const pauseScopePredicate = [ + `("scope_type" = 'space' AND "scope_id" = ANY(${textArray(spaceIds)}))`, + `("scope_type" = 'base' AND "scope_id" = ANY(${textArray(input.baseIds)}))`, + `("scope_type" = 'table' AND "scope_id" = ANY(${textArray(sharedTableIds)}))`, + ].join(' OR '); + const undoPredicate = `split_part("table_name", '.', 1) = ANY(${textArray(input.baseIds)})`; + + return [ + { + table: 'record_history', + columns: recordHistoryColumns, + whereSql: tablePredicate, + }, + { + table: 'table_trash', + columns: ['id', 'table_id', 'resource_type', 'snapshot', 'created_time', 'created_by'], + whereSql: tablePredicate, + }, + { + table: 'record_trash', + columns: ['id', 'table_id', 'record_id', 'snapshot', 'created_time', 'created_by'], + whereSql: tablePredicate, + }, + { + table: 'computed_update_outbox', + columns: computedOutboxColumns, + whereSql: basePredicate, + }, + { + table: 'computed_update_dead_letter', + columns: [...computedOutboxColumns, 'trace_data', 'failed_at'], + whereSql: basePredicate, + }, + { + table: 'computed_update_outbox_seed', + columns: ['id', 'task_id', 'table_id', 'record_id'], + whereSql: outboxSeedPredicate, + }, + { + table: 'computed_update_pause_scope', + columns: [ + 'id', + 'scope_type', + 'scope_id', + 'paused_at', + 'paused_by', + 'resume_at', + 'reason', + 'updated_at', + 'updated_by', + ], + whereSql: pauseScopePredicate, + }, + { + table: '__undo_log', + columns: [ + 'id', + 'batch_id', + 'operation', + 'table_name', + 'record_id', + 'old_row', + 'new_row', + 'created_at', + ], + whereSql: undoPredicate, + }, + ]; +}; + +export const buildMigrationSharedTablePsqlCopyPlans = (input: { + sourceUrl: string; + targetUrl: string; + sourceSchema: string; + targetSchema: string; + spaceId: string; + spaceIds?: string[]; + baseIds: string[]; + tableIds: string[]; + sharedTableIds?: string[]; +}): ISharedTablePsqlCopyPlan[] => { + const shared = buildMigrationSharedTableDefinitions(input); + + return shared.map((item) => + buildSharedTablePsqlCopyPlan({ + sourceUrl: input.sourceUrl, + targetUrl: input.targetUrl, + sourceSchema: input.sourceSchema, + targetSchema: input.targetSchema, + table: item.table, + columns: item.columns, + whereSql: item.whereSql, + }) + ); +}; + +export const buildMigrationSharedTablePostgresFdwCopyPlans = (input: { + sourceUrl: string; + targetUrl: string; + sourceSchema: string; + targetSchema: string; + spaceId: string; + spaceIds?: string[]; + baseIds: string[]; + tableIds: string[]; + sharedTableIds?: string[]; + fdwSchemaPrefix: string; + serverNamePrefix: string; +}): ISharedTablePostgresFdwCopyPlan[] => { + const shared = buildMigrationSharedTableDefinitions(input); + + return shared.map((_item, index) => { + const fdwSchema = `${input.fdwSchemaPrefix}_${index}`; + const item = buildMigrationSharedTableDefinitions({ ...input, sourceSchema: fdwSchema })[index]; + return buildSharedTablePostgresFdwCopyPlan({ + sourceUrl: input.sourceUrl, + targetUrl: input.targetUrl, + sourceSchema: input.sourceSchema, + targetSchema: input.targetSchema, + table: item.table, + columns: item.columns, + whereSql: item.whereSql, + fdwSchema, + serverName: `${input.serverNamePrefix}_${index}`, + }); + }); +}; diff --git a/apps/nestjs-backend/src/features/space/space-data-db-copy.integration.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-copy.integration.spec.ts new file mode 100644 index 0000000000..3921045efd --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-copy.integration.spec.ts @@ -0,0 +1,1250 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { spawnSync } from 'child_process'; +import { mkdtemp, mkdir, rm } from 'fs/promises'; +import { createServer } from 'net'; +import { tmpdir } from 'os'; +import path from 'path'; +import createKnex from 'knex'; +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; +import { DataDbRuntimeCacheService } from '../../global/data-db-runtime-cache.service'; +import { dataDbKnexClientFactory } from './data-db-preflight.service'; +import { encryptDataDbUrl } from './data-db-url-secret'; +import { buildMigrationSharedTablePsqlCopyPlans } from './space-data-db-copy-plan'; +import { SpaceDataDbCopyService } from './space-data-db-copy.service'; +import { SpaceDataDbMigrationService } from './space-data-db-migration.service'; +import { SpaceDataDbProcessRunnerService } from './space-data-db-process-runner.service'; + +const requiredPostgresBins = ['initdb', 'pg_ctl', 'pg_dump', 'pg_restore', 'psql']; +const hasPostgresBinaries = requiredPostgresBins.every( + (command) => spawnSync('which', [command], { stdio: 'ignore' }).status === 0 +); +const describeWithPostgres = hasPostgresBinaries ? describe : describe.skip; +const sourceDatabase = 'source_data_db'; +const targetDatabase = 'target_data_db'; +const targetMismatchDatabase = 'target_mismatch_data_db'; +const targetSchema = 'teable_meta_test'; +const targetMismatchSchema = 'teable_meta_mismatch'; +const baseId = 'bsecopy'; +const otherBaseId = 'bseother'; +const tableId = 'tblcopy'; +const linkedTableId = 'tbllinkedcopy'; +const spaceId = 'spccopy'; +const otherSpaceId = 'spcother'; +const jobId = 'sdmjcopy'; +const targetConnectionId = 'dcncopy'; +const mismatchJobId = 'sdmjmismatch'; +const mismatchTargetConnectionId = 'dcnmismatch'; +const mainRelationName = 'sheet1'; +const linkedRelationName = 'sheet2'; +const junctionRelationName = 'sheet1_sheet2_junction'; + +const buildCopyInventory = () => ({ + baseIds: [baseId], + tableIds: [tableId, linkedTableId], + dbTableNames: [ + `${baseId}.${mainRelationName}`, + `${baseId}.${linkedRelationName}`, + `${baseId}.${junctionRelationName}`, + ], + physicalSchemas: [ + { + schemaName: baseId, + totalBytes: 3072, + estimatedRows: 6, + relations: [ + { + schemaName: baseId, + relationName: mainRelationName, + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 2, + }, + { + schemaName: baseId, + relationName: linkedRelationName, + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 2, + }, + { + schemaName: baseId, + relationName: junctionRelationName, + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 2, + }, + ], + }, + ], +}); + +const execFile = async (command: string, args: string[], options: { timeout?: number } = {}) => { + const { execFile: nodeExecFile } = await import('child_process'); + return await new Promise((resolve, reject) => { + nodeExecFile(command, args, { timeout: options.timeout ?? 30_000 }, (error, stdout, stderr) => { + if (!error) { + resolve(); + return; + } + reject( + new Error(`${command} ${args.join(' ')} failed: ${stderr || stdout || error.message}`) + ); + }); + }); +}; + +const getFreePort = async () => + await new Promise((resolve, reject) => { + const server = createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (!address || typeof address === 'string') { + reject(new Error('Could not allocate a local PostgreSQL test port')); + return; + } + server.close(() => resolve(address.port)); + }); + }); + +const currentUser = () => encodeURIComponent(process.env.USER || 'postgres'); + +const pgUrl = (port: number, database: string) => + `postgresql://${currentUser()}@127.0.0.1:${port}/${database}`; + +const queryCount = async (client: Client, sql: string, values: unknown[] = []) => { + const result = await client.query<{ count: string }>(sql, values); + return Number(result.rows[0]?.count ?? 0); +}; + +const expectProcessTiming = (result: { + startedAt: string; + completedAt: string; + durationMs: number; +}) => { + expect(Date.parse(result.startedAt)).not.toBeNaN(); + expect(Date.parse(result.completedAt)).not.toBeNaN(); + expect(result.durationMs).toBeGreaterThanOrEqual(0); +}; + +const createSharedTables = async (client: Client, schema: string) => { + await client.query(`CREATE SCHEMA IF NOT EXISTS "${schema}"`); + await client.query(` + CREATE OR REPLACE FUNCTION "${schema}"."__teable_capture_undo_row"() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + RETURN NEW; + END; + $$ + `); + await client.query(` + CREATE TABLE "${schema}"."record_history" ( + "id" text PRIMARY KEY, + "table_id" text, + "record_id" text, + "field_id" text, + "before" jsonb, + "after" jsonb, + "created_time" timestamp, + "created_by" text + ) + `); + await client.query(` + CREATE TABLE "${schema}"."table_trash" ( + "id" text PRIMARY KEY, + "table_id" text, + "resource_type" text, + "snapshot" jsonb, + "created_time" timestamp, + "created_by" text + ) + `); + await client.query(` + CREATE TABLE "${schema}"."record_trash" ( + "id" text PRIMARY KEY, + "table_id" text, + "record_id" text, + "snapshot" jsonb, + "created_time" timestamp, + "created_by" text + ) + `); + await client.query(` + CREATE TABLE "${schema}"."computed_update_outbox" ( + "id" text PRIMARY KEY, + "base_id" text, + "seed_table_id" text, + "seed_record_ids" text, + "change_type" text, + "steps" text, + "edges" text, + "status" text, + "attempts" text, + "max_attempts" text, + "next_run_at" text, + "locked_at" timestamp, + "locked_by" text, + "last_error" text, + "estimated_complexity" text, + "plan_hash" text, + "dirty_stats" text, + "run_id" text, + "origin_run_ids" text, + "run_total_steps" text, + "run_completed_steps_before" text, + "affected_table_ids" text, + "affected_field_ids" text, + "sync_max_level" text, + "created_at" timestamp, + "updated_at" timestamp + ) + `); + await client.query(` + CREATE TABLE "${schema}"."computed_update_dead_letter" ( + "id" text PRIMARY KEY, + "base_id" text, + "seed_table_id" text, + "seed_record_ids" text, + "change_type" text, + "steps" text, + "edges" text, + "status" text, + "attempts" text, + "max_attempts" text, + "next_run_at" text, + "locked_at" timestamp, + "locked_by" text, + "last_error" text, + "estimated_complexity" text, + "plan_hash" text, + "dirty_stats" text, + "run_id" text, + "origin_run_ids" text, + "run_total_steps" text, + "run_completed_steps_before" text, + "affected_table_ids" text, + "affected_field_ids" text, + "sync_max_level" text, + "created_at" timestamp, + "updated_at" timestamp, + "trace_data" jsonb, + "failed_at" timestamp + ) + `); + await client.query(` + CREATE TABLE "${schema}"."computed_update_outbox_seed" ( + "id" text PRIMARY KEY, + "task_id" text, + "table_id" text, + "record_id" text + ) + `); + await client.query(` + CREATE TABLE "${schema}"."computed_update_pause_scope" ( + "id" text PRIMARY KEY, + "scope_type" text, + "scope_id" text, + "paused_at" timestamp, + "paused_by" text, + "resume_at" timestamp, + "reason" text, + "updated_at" timestamp, + "updated_by" text + ) + `); + await client.query(` + CREATE TABLE "${schema}"."__undo_log" ( + "id" text PRIMARY KEY, + "batch_id" text, + "operation" text, + "table_name" text, + "record_id" text, + "old_row" jsonb, + "new_row" jsonb, + "created_at" timestamp + ) + `); +}; + +const seedSourceData = async (client: Client) => { + await client.query(`CREATE SCHEMA "${baseId}"`); + await client.query(` + CREATE TABLE "${baseId}"."${mainRelationName}" ( + "id" text PRIMARY KEY, + "__created_time" timestamp, + "__last_modified_time" timestamp, + "name" text, + CONSTRAINT "sheet1_name_nonempty" CHECK (length("name") > 0) + ) + `); + await client.query( + `CREATE INDEX "sheet1_name_idx" ON "${baseId}"."${mainRelationName}" ("name")` + ); + await client.query(` + CREATE OR REPLACE FUNCTION "${baseId}"."touch_sheet1_modified_time"() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + NEW."__last_modified_time" = COALESCE(NEW."__last_modified_time", now()); + RETURN NEW; + END; + $$ + `); + await client.query(` + CREATE TRIGGER "sheet1_touch_modified_time" + BEFORE INSERT OR UPDATE ON "${baseId}"."${mainRelationName}" + FOR EACH ROW + EXECUTE FUNCTION "${baseId}"."touch_sheet1_modified_time"() + `); + await client.query( + `INSERT INTO "${baseId}"."${mainRelationName}" VALUES + ('rec1', now(), now(), 'A'), + ('rec2', now(), now(), 'B')` + ); + await client.query(` + CREATE TABLE "${baseId}"."${linkedRelationName}" ( + "id" text PRIMARY KEY, + "__created_time" timestamp, + "title" text + ) + `); + await client.query( + `INSERT INTO "${baseId}"."${linkedRelationName}" VALUES + ('rec-linked-1', now(), 'Linked A'), + ('rec-linked-2', now(), 'Linked B')` + ); + await client.query(` + CREATE TABLE "${baseId}"."${junctionRelationName}" ( + "id" text PRIMARY KEY, + "sheet1_id" text NOT NULL REFERENCES "${baseId}"."${mainRelationName}" ("id"), + "sheet2_id" text NOT NULL REFERENCES "${baseId}"."${linkedRelationName}" ("id") + ) + `); + await client.query( + `INSERT INTO "${baseId}"."${junctionRelationName}" VALUES + ('lnk1', 'rec1', 'rec-linked-1'), + ('lnk2', 'rec2', 'rec-linked-2')` + ); + await client.query(`CREATE SCHEMA "${otherBaseId}"`); + await client.query(` + CREATE TABLE "${otherBaseId}"."${mainRelationName}" ( + "id" text PRIMARY KEY, + "__created_time" timestamp, + "name" text + ) + `); + await client.query( + `INSERT INTO "${otherBaseId}"."${mainRelationName}" VALUES ('rec-other-1', now(), 'Other space row')` + ); + + await createSharedTables(client, 'public'); + await client.query( + `INSERT INTO "public"."record_history" VALUES + ('rh1', $1, 'rec1', 'fld1', '{}'::jsonb, '{"name":"A"}'::jsonb, now(), 'usr'), + ('rh2', 'tblother', 'rec9', 'fld1', '{}'::jsonb, '{}'::jsonb, now(), 'usr')`, + [tableId] + ); + await client.query( + `INSERT INTO "public"."table_trash" VALUES + ('tt1', $1, 'table', '{}'::jsonb, now(), 'usr'), + ('tt2', 'tblother', 'table', '{}'::jsonb, now(), 'usr')`, + [tableId] + ); + await client.query( + `INSERT INTO "public"."record_trash" VALUES + ('rt1', $1, 'rec1', '{}'::jsonb, now(), 'usr'), + ('rt2', 'tblother', 'rec9', '{}'::jsonb, now(), 'usr')`, + [tableId] + ); + await client.query( + `INSERT INTO "public"."computed_update_outbox" + ("id", "base_id", "status", "created_at", "updated_at") + VALUES + ('cuo1', $1, 'pending', now(), now()), + ('cuo2', $2, 'pending', now(), now())`, + [baseId, otherBaseId] + ); + await client.query( + `INSERT INTO "public"."computed_update_dead_letter" + ("id", "base_id", "status", "created_at", "updated_at", "trace_data", "failed_at") + VALUES + ('cudl1', $1, 'failed', now(), now(), '{}'::jsonb, now()), + ('cudl2', $2, 'failed', now(), now(), '{}'::jsonb, now())`, + [baseId, otherBaseId] + ); + await client.query( + `INSERT INTO "public"."computed_update_outbox_seed" VALUES + ('seed1', 'cuo1', $1, 'rec1'), + ('seed2', 'cuo2', $1, 'rec9')`, + [tableId] + ); + await client.query( + `INSERT INTO "public"."computed_update_pause_scope" + ("id", "scope_type", "scope_id", "paused_at", "paused_by", "reason", "updated_at", "updated_by") + VALUES + ('pause-space', 'space', $1, now(), 'usr', $5, now(), 'usr'), + ('pause-base', 'base', $2, now(), 'usr', 'migration', now(), 'usr'), + ('pause-table', 'table', $3, now(), 'usr', 'migration', now(), 'usr'), + ('pause-other', 'space', $4, now(), 'usr', 'other', now(), 'usr')`, + [spaceId, baseId, tableId, otherSpaceId, `space-data-db-migration:${jobId}`] + ); + await client.query( + `INSERT INTO "public"."__undo_log" VALUES + ('undo1', 'batch1', 'update', $1, 'rec1', '{}'::jsonb, '{}'::jsonb, now()), + ('undo2', 'batch2', 'update', $2, 'rec9', '{}'::jsonb, '{}'::jsonb, now())`, + [`${baseId}.${mainRelationName}`, `${otherBaseId}.${mainRelationName}`] + ); +}; + +const waitUntil = async (predicate: () => boolean, timeoutMs = 1000, pollMs = 10) => { + const startedAt = Date.now(); + for (;;) { + if (predicate()) { + return; + } + if (Date.now() - startedAt >= timeoutMs) { + throw new Error('Timed out waiting for condition'); + } + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } +}; + +describeWithPostgres('SpaceDataDbCopyService integration', () => { + let rootDir: string; + let dataDir: string; + let socketDir: string; + let port: number; + + beforeAll(async () => { + rootDir = await mkdtemp(path.join(tmpdir(), 'teable-byodb-copy-')); + dataDir = path.join(rootDir, 'pgdata'); + socketDir = path.join(rootDir, 'socket'); + port = await getFreePort(); + await mkdir(socketDir); + await execFile('initdb', ['-D', dataDir, '--no-instructions', '--auth=trust']); + await execFile('pg_ctl', [ + '-D', + dataDir, + '-o', + `-F -p ${port} -k ${socketDir} -c listen_addresses=127.0.0.1`, + '-l', + path.join(rootDir, 'postgres.log'), + '-w', + 'start', + ]); + + const admin = new Client({ connectionString: pgUrl(port, 'postgres') }); + await admin.connect(); + try { + await admin.query(`CREATE DATABASE ${sourceDatabase}`); + await admin.query(`CREATE DATABASE ${targetDatabase}`); + await admin.query(`CREATE DATABASE ${targetMismatchDatabase}`); + } finally { + await admin.end(); + } + + const source = new Client({ connectionString: pgUrl(port, sourceDatabase) }); + const target = new Client({ connectionString: pgUrl(port, targetDatabase) }); + const targetMismatch = new Client({ connectionString: pgUrl(port, targetMismatchDatabase) }); + await Promise.all([source.connect(), target.connect(), targetMismatch.connect()]); + try { + await seedSourceData(source); + await createSharedTables(target, targetSchema); + await createSharedTables(targetMismatch, targetMismatchSchema); + } finally { + await Promise.all([source.end(), target.end(), targetMismatch.end()]); + } + }, 30_000); + + afterAll(async () => { + if (dataDir) { + await execFile('pg_ctl', ['-D', dataDir, '-m', 'fast', '-w', 'stop']).catch(() => undefined); + } + if (rootDir) { + await rm(rootDir, { recursive: true, force: true }); + } + }, 30_000); + + it('copies base schemas and validates/switches the migration job through real PostgreSQL tools', async () => { + const sourceUrl = pgUrl(port, sourceDatabase); + const targetUrl = pgUrl(port, targetDatabase); + const service = new SpaceDataDbCopyService(new SpaceDataDbProcessRunnerService()); + await mkdir(path.join(rootDir, 'work')); + + const targetConnection = { + id: targetConnectionId, + status: 'migrating', + internalSchema: targetSchema, + encryptedUrl: encryptDataDbUrl(targetUrl), + displayHost: '127.0.0.1', + displayDatabase: targetDatabase, + urlFingerprint: 'dbfp_copy_target', + }; + let currentBinding: { + mode: 'byodb'; + state: 'ready'; + dataDbConnection: typeof targetConnection; + } | null = null; + const txClient = { + dataDbConnection: { + update: vi.fn().mockImplementation(async (args) => { + if (args.where?.id === targetConnectionId) { + targetConnection.status = args.data.status ?? targetConnection.status; + } + return undefined; + }), + }, + spaceDataDbBinding: { + upsert: vi.fn().mockImplementation(async (args) => { + if (args.where?.spaceId === spaceId) { + currentBinding = { + mode: 'byodb', + state: 'ready', + dataDbConnection: targetConnection, + }; + } + return undefined; + }), + }, + spaceDataDbMigrationJob: { + update: vi.fn().mockResolvedValue(undefined), + }, + }; + const migrationJob = { + id: jobId, + spaceId, + targetConnectionId, + targetInternalSchema: targetSchema, + createdBy: 'usr', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: null, + inventory: buildCopyInventory(), + copyStats: null, + validationStats: null, + targetConnection: { + encryptedUrl: encryptDataDbUrl(targetUrl), + }, + }; + const prismaService = { + $tx: vi.fn(async (fn: (client: typeof txClient) => Promise) => fn(txClient)), + base: { + findUnique: vi.fn().mockImplementation(async (args) => { + if (args.where?.id === baseId) { + return { spaceId }; + } + if (args.where?.id === otherBaseId) { + return { spaceId: otherSpaceId }; + } + return null; + }), + }, + spaceDataDbBinding: { + findUnique: vi + .fn() + .mockImplementation(async (args) => + args.where?.spaceId === spaceId ? currentBinding : null + ), + }, + spaceDataDbMigrationJob: { + findUnique: vi.fn().mockResolvedValue(migrationJob), + update: vi.fn().mockResolvedValue(undefined), + }, + }; + const baselineService = { + getLatestSchemaVersion: vi.fn().mockReturnValue(null), + }; + const dataDbClientManager = { + getDataDatabaseForSpace: vi.fn().mockImplementation((_, options) => { + if (options?.previewBinding) { + return Promise.resolve({ + cacheKey: targetConnectionId, + connectionId: targetConnectionId, + internalSchema: targetSchema, + isMetaFallback: false, + url: targetUrl, + }); + } + return Promise.resolve({ + cacheKey: 'meta-fallback', + connectionId: undefined, + internalSchema: undefined, + isMetaFallback: true, + url: sourceUrl, + }); + }), + invalidateConnection: vi.fn(), + }; + const runtimeCache = new DataDbRuntimeCacheService(); + const sourceFallbackKnex = createKnex({ + client: 'pg', + connection: sourceUrl, + pool: { min: 0, max: 1 }, + }); + const routingManager = new DataDbClientManager( + prismaService as never, + {} as never, + sourceFallbackKnex, + runtimeCache + ); + const migrationService = new SpaceDataDbMigrationService( + prismaService as never, + {} as never, + baselineService as never, + dataDbClientManager as never, + service as never, + dataDbKnexClientFactory + ); + + try { + await expect(routingManager.dataKnexForBase(baseId)).resolves.toBe(sourceFallbackKnex); + + const baseCopyStats = await migrationService.copyBaseSchemasForJob(jobId, { + workDir: path.join(rootDir, 'work'), + jobs: 2, + timeoutMs: 30_000, + }); + + expect(baseCopyStats).toMatchObject({ + phase: 'base_schemas_completed', + baseSchemas: { + copiedRelationCount: 3, + totalCopiedRows: 6, + copiedRelations: expect.arrayContaining([ + expect.objectContaining({ + schemaName: baseId, + relationName: mainRelationName, + copiedRows: 2, + estimatedRows: 2, + }), + expect.objectContaining({ + schemaName: baseId, + relationName: linkedRelationName, + copiedRows: 2, + estimatedRows: 2, + }), + expect.objectContaining({ + schemaName: baseId, + relationName: junctionRelationName, + copiedRows: 2, + estimatedRows: 2, + }), + ]), + }, + }); + expectProcessTiming(baseCopyStats.baseSchemas.dump); + expectProcessTiming(baseCopyStats.baseSchemas.restore); + + const sharedResults = await service.copySharedTables( + buildMigrationSharedTablePsqlCopyPlans({ + sourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema, + spaceId, + baseIds: [baseId], + tableIds: [tableId, linkedTableId], + }), + { timeoutMs: 30_000 } + ); + + expect(sharedResults).toEqual( + expect.arrayContaining([ + expect.objectContaining({ table: 'record_history', copiedRows: 1 }), + expect.objectContaining({ table: 'table_trash', copiedRows: 1 }), + expect.objectContaining({ table: 'record_trash', copiedRows: 1 }), + expect.objectContaining({ table: 'computed_update_outbox', copiedRows: 1 }), + expect.objectContaining({ table: 'computed_update_dead_letter', copiedRows: 1 }), + expect.objectContaining({ table: 'computed_update_outbox_seed', copiedRows: 1 }), + expect.objectContaining({ table: 'computed_update_pause_scope', copiedRows: 3 }), + expect.objectContaining({ table: '__undo_log', copiedRows: 1 }), + ]) + ); + const historyCopy = sharedResults.find((result) => result.table === 'record_history'); + expect(historyCopy).toBeDefined(); + if (!historyCopy) { + throw new Error('Expected record_history copy result'); + } + expectProcessTiming(historyCopy.source); + expectProcessTiming(historyCopy.target); + + const target = new Client({ connectionString: targetUrl }); + await target.connect(); + try { + await expect( + queryCount(target, `SELECT COUNT(*) AS count FROM "${baseId}"."${mainRelationName}"`) + ).resolves.toBe(2); + await expect( + queryCount(target, `SELECT COUNT(*) AS count FROM "${baseId}"."${linkedRelationName}"`) + ).resolves.toBe(2); + await expect( + queryCount(target, `SELECT COUNT(*) AS count FROM "${baseId}"."${junctionRelationName}"`) + ).resolves.toBe(2); + await expect( + queryCount( + target, + ` + SELECT COUNT(*) AS count + FROM pg_indexes + WHERE schemaname = $1 + AND indexname = 'sheet1_name_idx' + `, + [baseId] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + ` + SELECT COUNT(*) AS count + FROM pg_constraint con + JOIN pg_class c ON c.oid = con.conrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 + AND c.relname = $2 + AND con.conname = 'sheet1_name_nonempty' + `, + [baseId, mainRelationName] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + ` + SELECT COUNT(*) AS count + FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 + AND c.relname = $2 + AND tg.tgname = 'sheet1_touch_modified_time' + AND NOT tg.tgisinternal + `, + [baseId, mainRelationName] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."record_history" WHERE "table_id" = $1`, + [tableId] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."record_history" WHERE "table_id" = 'tblother'` + ) + ).resolves.toBe(0); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."table_trash" WHERE "table_id" = $1`, + [tableId] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."table_trash" WHERE "table_id" = 'tblother'` + ) + ).resolves.toBe(0); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."record_trash" WHERE "table_id" = $1`, + [tableId] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."record_trash" WHERE "table_id" = 'tblother'` + ) + ).resolves.toBe(0); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_outbox" WHERE "base_id" = $1`, + [baseId] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_outbox" WHERE "base_id" = $1`, + [otherBaseId] + ) + ).resolves.toBe(0); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_dead_letter" WHERE "base_id" = $1`, + [baseId] + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_dead_letter" WHERE "base_id" = $1`, + [otherBaseId] + ) + ).resolves.toBe(0); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_outbox_seed"` + ) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_outbox_seed" WHERE "task_id" = 'cuo2'` + ) + ).resolves.toBe(0); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_pause_scope"` + ) + ).resolves.toBe(3); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."computed_update_pause_scope" WHERE "scope_id" = $1`, + [otherSpaceId] + ) + ).resolves.toBe(0); + await expect( + queryCount(target, `SELECT COUNT(*) AS count FROM "${targetSchema}"."__undo_log"`) + ).resolves.toBe(1); + await expect( + queryCount( + target, + `SELECT COUNT(*) AS count FROM "${targetSchema}"."__undo_log" WHERE "table_name" = $1`, + [`${otherBaseId}.${mainRelationName}`] + ) + ).resolves.toBe(0); + } finally { + await target.end(); + } + + await expect(migrationService.validateAndSwitchJob(jobId)).resolves.toMatchObject({ + state: 'succeeded', + validationStats: expect.objectContaining({ + phase: 'validation_completed', + routeSmoke: expect.objectContaining({ + ok: true, + connectionId: targetConnectionId, + internalSchema: targetSchema, + }), + baseSchemas: expect.arrayContaining([ + expect.objectContaining({ object: `base:${baseId}.${mainRelationName}` }), + expect.objectContaining({ object: `base:${baseId}.${linkedRelationName}` }), + expect.objectContaining({ object: `base:${baseId}.${junctionRelationName}` }), + ]), + }), + }); + + const postSwitchKnex = await routingManager.dataKnexForBase(baseId); + expect(postSwitchKnex).not.toBe(sourceFallbackKnex); + await postSwitchKnex.raw( + `INSERT INTO "${baseId}"."${mainRelationName}" ("id", "__created_time", "name") VALUES (?, now(), ?)`, + ['rec-post-switch', 'BYODB target'] + ); + const otherSpaceKnex = await routingManager.dataKnexForBase(otherBaseId); + expect(otherSpaceKnex).toBe(sourceFallbackKnex); + await otherSpaceKnex.raw( + `INSERT INTO "${otherBaseId}"."${mainRelationName}" ("id", "__created_time", "name") VALUES (?, now(), ?)`, + ['rec-other-post-switch', 'Default source'] + ); + + const source = new Client({ connectionString: sourceUrl }); + const targetAfterSwitch = new Client({ connectionString: targetUrl }); + await Promise.all([source.connect(), targetAfterSwitch.connect()]); + try { + await expect( + queryCount( + source, + `SELECT COUNT(*) AS count FROM "${baseId}"."${mainRelationName}" WHERE "id" = 'rec-post-switch'` + ) + ).resolves.toBe(0); + await expect( + queryCount( + targetAfterSwitch, + `SELECT COUNT(*) AS count FROM "${baseId}"."${mainRelationName}" WHERE "id" = 'rec-post-switch'` + ) + ).resolves.toBe(1); + await expect( + queryCount( + source, + `SELECT COUNT(*) AS count FROM "${otherBaseId}"."${mainRelationName}" WHERE "id" = 'rec-other-post-switch'` + ) + ).resolves.toBe(1); + await expect( + queryCount( + targetAfterSwitch, + ` + SELECT COUNT(*) AS count + FROM information_schema.schemata + WHERE schema_name = $1 + `, + [otherBaseId] + ) + ).resolves.toBe(0); + } finally { + await Promise.all([source.end(), targetAfterSwitch.end()]); + } + + expect(txClient.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: targetConnectionId }, + data: expect.objectContaining({ status: 'ready', lastError: null }), + }); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith({ + where: { spaceId }, + create: { + spaceId, + dataDbConnectionId: targetConnectionId, + mode: 'byodb', + state: 'ready', + createdBy: 'usr', + }, + update: { + dataDbConnectionId: targetConnectionId, + mode: 'byodb', + state: 'ready', + }, + }); + expect(txClient.spaceDataDbMigrationJob.update).toHaveBeenCalledWith({ + where: { id: jobId }, + data: expect.objectContaining({ state: 'succeeded', lastError: null }), + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith(targetConnectionId); + } finally { + await Promise.all([routingManager.onModuleDestroy(), sourceFallbackKnex.destroy()]); + } + }, 60_000); + + it('fails validation and keeps source routing when target base rows drift before switch', async () => { + const sourceUrl = pgUrl(port, sourceDatabase); + const targetUrl = pgUrl(port, targetMismatchDatabase); + const service = new SpaceDataDbCopyService(new SpaceDataDbProcessRunnerService()); + await mkdir(path.join(rootDir, 'work-mismatch')); + + const targetConnection = { + id: mismatchTargetConnectionId, + status: 'migrating', + internalSchema: targetMismatchSchema, + encryptedUrl: encryptDataDbUrl(targetUrl), + displayHost: '127.0.0.1', + displayDatabase: targetMismatchDatabase, + urlFingerprint: 'dbfp_copy_target_mismatch', + }; + let currentBinding: { + mode: 'byodb'; + state: 'ready'; + dataDbConnection: typeof targetConnection; + } | null = null; + const txClient = { + dataDbConnection: { + update: vi.fn().mockImplementation(async (args) => { + if (args.where?.id === mismatchTargetConnectionId) { + targetConnection.status = args.data.status ?? targetConnection.status; + } + return undefined; + }), + }, + spaceDataDbBinding: { + upsert: vi.fn().mockImplementation(async (args) => { + if (args.where?.spaceId === spaceId) { + currentBinding = { + mode: 'byodb', + state: 'ready', + dataDbConnection: targetConnection, + }; + } + return undefined; + }), + }, + spaceDataDbMigrationJob: { + update: vi.fn().mockResolvedValue(undefined), + }, + }; + const migrationJob = { + id: mismatchJobId, + spaceId, + targetConnectionId: mismatchTargetConnectionId, + targetInternalSchema: targetMismatchSchema, + createdBy: 'usr', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: null, + inventory: buildCopyInventory(), + copyStats: null, + validationStats: null, + targetConnection: { + encryptedUrl: encryptDataDbUrl(targetUrl), + }, + }; + const prismaService = { + $tx: vi.fn(async (fn: (client: typeof txClient) => Promise) => fn(txClient)), + base: { + findUnique: vi.fn().mockImplementation(async (args) => { + if (args.where?.id === baseId) { + return { spaceId }; + } + return null; + }), + }, + spaceDataDbBinding: { + findUnique: vi + .fn() + .mockImplementation(async (args) => + args.where?.spaceId === spaceId ? currentBinding : null + ), + }, + spaceDataDbMigrationJob: { + findUnique: vi.fn().mockResolvedValue(migrationJob), + update: vi.fn().mockResolvedValue(undefined), + }, + }; + const dataDbClientManager = { + getDataDatabaseForSpace: vi.fn().mockImplementation((_, options) => { + if (options?.previewBinding) { + return Promise.resolve({ + cacheKey: mismatchTargetConnectionId, + connectionId: mismatchTargetConnectionId, + internalSchema: targetMismatchSchema, + isMetaFallback: false, + url: targetUrl, + }); + } + return Promise.resolve({ + cacheKey: 'meta-fallback', + connectionId: undefined, + internalSchema: undefined, + isMetaFallback: true, + url: sourceUrl, + }); + }), + invalidateConnection: vi.fn(), + }; + const sourceFallbackKnex = createKnex({ + client: 'pg', + connection: sourceUrl, + pool: { min: 0, max: 1 }, + }); + const routingManager = new DataDbClientManager( + prismaService as never, + {} as never, + sourceFallbackKnex, + new DataDbRuntimeCacheService() + ); + const migrationService = new SpaceDataDbMigrationService( + prismaService as never, + {} as never, + { getLatestSchemaVersion: vi.fn().mockReturnValue(null) } as never, + dataDbClientManager as never, + service as never, + dataDbKnexClientFactory + ); + + try { + await migrationService.copyBaseSchemasForJob(mismatchJobId, { + workDir: path.join(rootDir, 'work-mismatch'), + jobs: 2, + timeoutMs: 30_000, + }); + await service.copySharedTables( + buildMigrationSharedTablePsqlCopyPlans({ + sourceUrl, + targetUrl, + sourceSchema: 'public', + targetSchema: targetMismatchSchema, + spaceId, + baseIds: [baseId], + tableIds: [tableId, linkedTableId], + }), + { timeoutMs: 30_000 } + ); + + const target = new Client({ connectionString: targetUrl }); + await target.connect(); + try { + await target.query( + `INSERT INTO "${baseId}"."${mainRelationName}" ("id", "__created_time", "name") + VALUES ('rec-target-extra', now(), 'Target extra row')` + ); + } finally { + await target.end(); + } + + await expect(migrationService.validateAndSwitchJob(mismatchJobId)).rejects.toMatchObject({ + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: expect.arrayContaining([ + expect.objectContaining({ + object: `base:${baseId}.${mainRelationName}`, + sourceCount: 2, + targetCount: 3, + }), + ]), + }), + }); + await expect(routingManager.dataKnexForBase(baseId)).resolves.toBe(sourceFallbackKnex); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.update).not.toHaveBeenCalled(); + expect(dataDbClientManager.invalidateConnection).not.toHaveBeenCalled(); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: mismatchJobId }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'Space data database migration validation failed', + validationStats: expect.objectContaining({ phase: 'validation_failed' }), + }), + }) + ); + } finally { + await Promise.all([routingManager.onModuleDestroy(), sourceFallbackKnex.destroy()]); + } + }, 60_000); + + it('waits for active computed tasks in the source data DB before draining', async () => { + const sourceUrl = pgUrl(port, sourceDatabase); + const source = new Client({ connectionString: sourceUrl }); + await source.connect(); + try { + await source.query( + `INSERT INTO "public"."computed_update_outbox" + ("id", "base_id", "status", "locked_at", "created_at", "updated_at") + VALUES + ('cuo-active-drain', $1, 'processing', now(), now(), now()), + ('cuo-active-other-base', 'bseother', 'processing', now(), now(), now()) + ON CONFLICT ("id") DO UPDATE + SET "base_id" = EXCLUDED."base_id", + "status" = EXCLUDED."status", + "locked_at" = EXCLUDED."locked_at", + "updated_at" = EXCLUDED."updated_at"`, + [baseId] + ); + } finally { + await source.end(); + } + + const updates: unknown[] = []; + const migrationJob = { + id: jobId, + spaceId, + targetInternalSchema: targetSchema, + createdBy: 'usr', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + inventory: { + baseIds: [baseId], + tableIds: [tableId], + dbTableNames: [`${baseId}.sheet1`], + physicalSchemas: [], + }, + targetConnection: null, + }; + const prismaService = { + spaceDataDbMigrationJob: { + findUnique: vi.fn().mockResolvedValue(migrationJob), + findFirst: vi.fn().mockResolvedValue({ id: jobId, spaceId, state: 'copying' }), + update: vi.fn().mockImplementation(async (args: unknown) => { + updates.push(args); + return undefined; + }), + }, + }; + const dataDbClientManager = { + getDataDatabaseForSpace: vi.fn().mockResolvedValue({ + cacheKey: 'meta-fallback', + connectionId: undefined, + internalSchema: undefined, + isMetaFallback: true, + url: sourceUrl, + }), + invalidateConnection: vi.fn(), + }; + const migrationService = new SpaceDataDbMigrationService( + prismaService as never, + {} as never, + { getLatestSchemaVersion: vi.fn().mockReturnValue(null) } as never, + dataDbClientManager as never, + new SpaceDataDbCopyService(new SpaceDataDbProcessRunnerService()) as never, + dataDbKnexClientFactory + ); + + const drain = migrationService.waitForSourceComputedDrainForJob(jobId, { + timeoutMs: 2000, + pollMs: 20, + processingLeaseMs: 120_000, + }); + + await waitUntil(() => + updates.some((item) => { + const args = item as { data?: { copyStats?: { phase?: string; computedDrain?: unknown } } }; + const computedDrain = args.data?.copyStats?.computedDrain as + | { activeCount?: number } + | undefined; + return ( + args.data?.copyStats?.phase === 'computed_draining' && computedDrain?.activeCount === 1 + ); + }) + ); + + const finisher = new Client({ connectionString: sourceUrl }); + await finisher.connect(); + try { + await finisher.query( + `UPDATE "public"."computed_update_outbox" + SET "status" = 'pending', + "locked_at" = NULL, + "updated_at" = now() + WHERE "id" = 'cuo-active-drain'` + ); + } finally { + await finisher.end(); + } + + await expect(drain).resolves.toMatchObject({ + activeCount: 0, + reclaimableCount: 0, + }); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: jobId }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'computed_drained', + computedDrain: expect.objectContaining({ + activeCount: 0, + reclaimableCount: 0, + }), + }), + }), + }) + ); + + const verifier = new Client({ connectionString: sourceUrl }); + await verifier.connect(); + try { + await expect( + queryCount( + verifier, + `SELECT COUNT(*) AS count + FROM "public"."computed_update_outbox" + WHERE "id" = 'cuo-active-other-base' + AND "base_id" = 'bseother' + AND "status" = 'processing'` + ) + ).resolves.toBe(1); + } finally { + await verifier.end(); + } + }, 30_000); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-copy.service.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-copy.service.spec.ts new file mode 100644 index 0000000000..d0ae1147b0 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-copy.service.spec.ts @@ -0,0 +1,599 @@ +import { mkdtemp, readFile, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import path from 'path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + filterPgRestoreListForForeignKeys, + PG_RESTORE_LIST_STDOUT_LIMIT, + postgresCopyToolsForStrategy, + parsePsqlInsertRowCount, + parsePsqlCopyRowCount, + REQUIRED_PGCOPYDB_COPY_TOOLS, + REQUIRED_POSTGRES_COPY_TOOLS, + SpaceDataDbCopyService, +} from './space-data-db-copy.service'; + +const sourceUrl = 'postgresql://source.example/teable'; +const targetUrl = 'postgresql://target.example/teable'; +const workDir = '/tmp/sdmjxxx'; +const psqlCommand = 'psql'; +const historyTargetArgs = ['history-target']; +const trashTargetArgs = ['trash-target']; + +describe('SpaceDataDbCopyService', () => { + const processRunner = { + run: vi.fn(), + runPipeline: vi.fn(), + }; + + beforeEach(() => { + processRunner.run.mockReset(); + processRunner.runPipeline.mockReset(); + }); + + it('parses psql COPY row counts from command output', () => { + expect(parsePsqlCopyRowCount('COPY 42\n')).toBe(42); + expect(parsePsqlCopyRowCount('notice\nCOPY 3\nCOPY 4\n')).toBe(4); + expect(parsePsqlCopyRowCount('')).toBeNull(); + }); + + it('parses psql INSERT row counts from command output', () => { + expect(parsePsqlInsertRowCount('INSERT 0 42\n')).toBe(42); + expect(parsePsqlInsertRowCount('BEGIN\nINSERT 0 3\nINSERT 0 4\nCOMMIT\n')).toBe(4); + expect(parsePsqlInsertRowCount('')).toBeNull(); + }); + + it('checks required PostgreSQL client tools before copy work starts', async () => { + processRunner.run.mockImplementation((plan: { command: string; args: string[] }) => + Promise.resolve({ + command: plan.command, + args: plan.args, + exitCode: 0, + signal: null, + stderr: '', + stdout: '', + startedAt: '2026-05-06T00:00:00.000Z', + completedAt: '2026-05-06T00:00:01.000Z', + durationMs: 1000, + }) + ); + const service = new SpaceDataDbCopyService(processRunner as never); + + await expect( + service.assertPostgresToolsAvailable('pg_dump_restore', { timeoutMs: 5000 }) + ).resolves.toHaveLength(REQUIRED_POSTGRES_COPY_TOOLS.length); + + for (const [index, command] of REQUIRED_POSTGRES_COPY_TOOLS.entries()) { + expect(processRunner.run).toHaveBeenNthCalledWith( + index + 1, + { command, args: ['--version'] }, + { timeoutMs: 5000 } + ); + } + }); + + it('requires pgcopydb only when the pgcopydb base-schema strategy is selected', () => { + expect(postgresCopyToolsForStrategy('pg_dump_restore')).toEqual([ + 'pg_dump', + 'pg_restore', + 'psql', + ]); + expect(postgresCopyToolsForStrategy('pg_dump_stream_restore')).toEqual([ + 'pg_dump', + 'pg_restore', + 'psql', + ]); + expect(postgresCopyToolsForStrategy('pgcopydb')).toEqual(REQUIRED_PGCOPYDB_COPY_TOOLS); + }); + + it('streams pg_dump into pg_restore for base schema copies by default', async () => { + processRunner.runPipeline.mockResolvedValueOnce({ + source: { command: 'pg_dump', args: [], exitCode: 0, signal: null, stderr: '', stdout: '' }, + target: { + command: 'pg_restore', + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout: '', + }, + }); + const service = new SpaceDataDbCopyService(processRunner as never); + + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + workDir, + jobs: 2, + processOptions: { timeoutMs: 10_000 }, + }) + ).resolves.toMatchObject({ + strategy: 'pg_dump_stream_restore', + stream: { + source: { command: 'pg_dump' }, + target: { command: 'pg_restore' }, + }, + }); + + expect(processRunner.run).not.toHaveBeenCalled(); + expect(processRunner.runPipeline).toHaveBeenCalledWith( + expect.objectContaining({ + source: expect.objectContaining({ + command: 'pg_dump', + args: expect.arrayContaining(['--schema', '"bseaaa"', '--schema', '"bsebbb"']), + }), + target: expect.objectContaining({ command: 'pg_restore' }), + }), + { timeoutMs: 10_000 } + ); + }); + + it('executes pg_dump before pg_restore when the dump/restore strategy is selected', async () => { + processRunner.run + .mockResolvedValueOnce({ command: 'pg_dump', exitCode: 0, args: [], stderr: '', stdout: '' }) + .mockResolvedValueOnce({ + command: 'pg_restore', + exitCode: 0, + args: [], + stderr: '', + stdout: '', + }); + const service = new SpaceDataDbCopyService(processRunner as never); + + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + workDir, + jobs: 2, + strategy: 'pg_dump_restore', + processOptions: { timeoutMs: 10_000 }, + }) + ).resolves.toMatchObject({ + strategy: 'pg_dump_restore', + dump: { command: 'pg_dump' }, + restore: { command: 'pg_restore' }, + }); + + expect(processRunner.run).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + command: 'pg_dump', + args: expect.arrayContaining(['--schema', '"bseaaa"', '--schema', '"bsebbb"']), + }), + { timeoutMs: 10_000 } + ); + expect(processRunner.run).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ command: 'pg_restore' }), + { timeoutMs: 10_000 } + ); + }); + + it('streams pg_dump into pg_restore for base schema copies when selected', async () => { + processRunner.runPipeline.mockResolvedValueOnce({ + source: { command: 'pg_dump', args: [], exitCode: 0, signal: null, stderr: '', stdout: '' }, + target: { + command: 'pg_restore', + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout: '', + }, + }); + const service = new SpaceDataDbCopyService(processRunner as never); + + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + workDir, + strategy: 'pg_dump_stream_restore', + processOptions: { timeoutMs: 10_000 }, + }) + ).resolves.toMatchObject({ + strategy: 'pg_dump_stream_restore', + stream: { + source: { command: 'pg_dump' }, + target: { command: 'pg_restore' }, + }, + }); + + expect(processRunner.run).not.toHaveBeenCalled(); + expect(processRunner.runPipeline).toHaveBeenCalledWith( + expect.objectContaining({ + source: expect.objectContaining({ + command: 'pg_dump', + args: expect.arrayContaining(['--format=custom']), + }), + target: expect.objectContaining({ command: 'pg_restore' }), + }), + { timeoutMs: 10_000 } + ); + expect(processRunner.runPipeline.mock.calls[0][0].source.args).not.toEqual( + expect.arrayContaining(['--file', '-']) + ); + }); + + it('filters out-of-space foreign keys from pg_restore list before restore', async () => { + const workDir = await mkdtemp(path.join(tmpdir(), 'teable-pg-restore-list-')); + processRunner.run + .mockResolvedValueOnce({ command: 'pg_dump', exitCode: 0, args: [], stderr: '', stdout: '' }) + .mockResolvedValueOnce({ + command: 'pg_restore', + exitCode: 0, + args: ['--list'], + stderr: '', + stdout: [ + '; archive TOC', + '181; 1259 1 TABLE bse9Jpr5JmgTTXRYHWh Biao_GeyaAMdfes85 postgres', + '182; 2606 2 FK CONSTRAINT bse9Jpr5JmgTTXRYHWh Biao_GeyaAMdfes85 fk___fk_fldP7S7LNeXsLsEFFhr postgres', + '183; 2606 3 FK CONSTRAINT bse9Jpr5JmgTTXRYHWh Biao_GeyaAMdfes85 fk_keep_in_scope postgres', + '', + ].join('\n'), + }) + .mockResolvedValueOnce({ + command: 'pg_restore', + exitCode: 0, + args: [], + stderr: '', + stdout: '', + }); + const service = new SpaceDataDbCopyService(processRunner as never); + + try { + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bse9Jpr5JmgTTXRYHWh'], + workDir, + jobs: 2, + excludedForeignKeys: [ + { + schemaName: 'bse9Jpr5JmgTTXRYHWh', + tableName: 'Biao_GeyaAMdfes85', + constraintName: 'fk___fk_fldP7S7LNeXsLsEFFhr', + referencedSchemaName: 'bsemNMekh61Et', + referencedTableName: 'Newtable_tblXTN8jAZ9i7omvC8u', + }, + ], + processOptions: { timeoutMs: 10_000 }, + }) + ).resolves.toMatchObject({ + strategy: 'pg_dump_restore', + restoreList: { command: 'pg_restore' }, + filteredRestoreList: { + requestedForeignKeyCount: 1, + excludedEntryCount: 1, + }, + }); + + const restoreCall = processRunner.run.mock.calls[2][0]; + const restoreListOptions = processRunner.run.mock.calls[1][1]; + expect(restoreListOptions).toMatchObject({ + timeoutMs: 10_000, + stdoutLimit: PG_RESTORE_LIST_STDOUT_LIMIT, + }); + + expect(restoreCall).toEqual( + expect.objectContaining({ + command: 'pg_restore', + args: expect.arrayContaining([ + '--use-list', + path.join(workDir, 'base-schemas.restore.list'), + ]), + }) + ); + + const listFile = await readFile(path.join(workDir, 'base-schemas.restore.list'), 'utf8'); + expect(listFile).not.toContain('fk___fk_fldP7S7LNeXsLsEFFhr'); + expect(listFile).toContain('fk_keep_in_scope'); + } finally { + await rm(workDir, { recursive: true, force: true }); + } + }); + + it('filters pg_restore list entries by schema table and foreign key names', () => { + const filtered = filterPgRestoreListForForeignKeys( + [ + '182; 2606 2 FK CONSTRAINT bsexxx sheet1 fk_out_of_scope postgres', + '183; 2606 3 FK CONSTRAINT bsexxx sheet2 fk_out_of_scope postgres', + '184; 2606 4 CONSTRAINT bsexxx sheet1 fk_out_of_scope postgres', + '', + ].join('\n'), + [ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ] + ); + + expect(filtered.excludedEntryCount).toBe(1); + expect(filtered.content).not.toContain('182;'); + expect(filtered.content).toContain('183;'); + expect(filtered.content).toContain('184;'); + }); + + it('attaches separate progress hooks to pg_dump and pg_restore', async () => { + processRunner.run + .mockResolvedValueOnce({ command: 'pg_dump', exitCode: 0, args: [], stderr: '', stdout: '' }) + .mockResolvedValueOnce({ + command: 'pg_restore', + exitCode: 0, + args: [], + stderr: '', + stdout: '', + }); + const service = new SpaceDataDbCopyService(processRunner as never); + const onDumpProgressPoll = vi.fn(); + const onRestoreProgressPoll = vi.fn(); + const baseOnPoll = vi.fn(); + + await service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsexxx'], + workDir, + strategy: 'pg_dump_restore', + processOptions: { timeoutMs: 10_000, pollMs: 250, onPoll: baseOnPoll }, + hooks: { + onDumpProgressPoll, + onRestoreProgressPoll, + }, + }); + + const dumpOptions = processRunner.run.mock.calls[0][1]; + const restoreOptions = processRunner.run.mock.calls[1][1]; + await dumpOptions.onPoll(); + await restoreOptions.onPoll(); + + expect(baseOnPoll).toHaveBeenCalledTimes(2); + expect(onDumpProgressPoll).toHaveBeenCalledTimes(1); + expect(onRestoreProgressPoll).toHaveBeenCalledTimes(1); + expect(dumpOptions).toMatchObject({ timeoutMs: 10_000, pollMs: 250 }); + expect(restoreOptions).toMatchObject({ timeoutMs: 10_000, pollMs: 250 }); + }); + + it('does not run restore when dump fails', async () => { + processRunner.run.mockRejectedValueOnce(new Error('dump failed')); + const service = new SpaceDataDbCopyService(processRunner as never); + + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsexxx'], + workDir, + strategy: 'pg_dump_restore', + }) + ).rejects.toThrow('dump failed'); + + expect(processRunner.run).toHaveBeenCalledTimes(1); + expect(processRunner.runPipeline).not.toHaveBeenCalled(); + }); + + it('writes a pgcopydb filter file and runs pgcopydb when explicitly selected', async () => { + const workDir = await mkdtemp(path.join(tmpdir(), 'teable-pgcopydb-plan-')); + processRunner.run.mockResolvedValueOnce({ + command: 'pgcopydb', + exitCode: 0, + args: [], + stderr: '', + stdout: '', + }); + const service = new SpaceDataDbCopyService(processRunner as never); + + try { + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsebbb', 'bseaaa'], + workDir, + jobs: 2, + strategy: 'pgcopydb', + processOptions: { timeoutMs: 10_000 }, + }) + ).resolves.toMatchObject({ + strategy: 'pgcopydb', + pgcopydb: { command: 'pgcopydb' }, + }); + + expect(processRunner.run).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'pgcopydb', + args: expect.arrayContaining([ + 'copy', + 'db', + '--filters', + path.join(workDir, 'pgcopydb-base-schemas.filter.ini'), + ]), + }), + { timeoutMs: 10_000 } + ); + } finally { + await rm(workDir, { recursive: true, force: true }); + } + }); + + it('rejects pgcopydb when out-of-space foreign keys need filtered restore entries', async () => { + const service = new SpaceDataDbCopyService(processRunner as never); + + await expect( + service.copyBaseSchemas({ + sourceUrl, + targetUrl, + schemaNames: ['bsexxx'], + workDir, + strategy: 'pgcopydb', + excludedForeignKeys: [ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ], + }) + ).rejects.toThrow('pgcopydb base schema copy does not support filtering'); + + expect(processRunner.run).not.toHaveBeenCalled(); + }); + + it('executes shared table COPY pipelines sequentially', async () => { + processRunner.runPipeline + .mockResolvedValueOnce({ + source: { command: 'psql', args: [], exitCode: 0, signal: null, stderr: '', stdout: '' }, + target: { + command: 'psql', + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout: 'COPY 12\n', + }, + }) + .mockResolvedValueOnce({ + source: { command: 'psql', args: [], exitCode: 0, signal: null, stderr: '', stdout: '' }, + target: { + command: 'psql', + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout: 'COPY 0\n', + }, + }); + const service = new SpaceDataDbCopyService(processRunner as never); + const onTableCopied = vi.fn(); + + await expect( + service.copySharedTables( + [ + { + table: 'record_history', + sourceSql: 'COPY source history TO STDOUT', + targetSql: 'COPY target history FROM STDIN', + source: { command: psqlCommand, args: ['history-source'] }, + target: { command: psqlCommand, args: historyTargetArgs }, + }, + { + table: 'record_trash', + sourceSql: 'COPY source trash TO STDOUT', + targetSql: 'COPY target trash FROM STDIN', + source: { command: psqlCommand, args: ['trash-source'] }, + target: { command: psqlCommand, args: trashTargetArgs }, + }, + ], + { timeoutMs: 10_000 }, + { onTableCopied } + ) + ).resolves.toEqual([ + expect.objectContaining({ table: 'record_history', copiedRows: 12 }), + expect.objectContaining({ table: 'record_trash', copiedRows: 0 }), + ]); + + expect(processRunner.runPipeline).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ table: 'record_history' }), + { timeoutMs: 10_000 } + ); + expect(processRunner.runPipeline).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ table: 'record_trash' }), + { timeoutMs: 10_000 } + ); + expect(onTableCopied).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ table: 'record_history', copiedRows: 12 }), + 0, + 2 + ); + expect(onTableCopied).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ table: 'record_trash', copiedRows: 0 }), + 1, + 2 + ); + }); + + it('executes postgres_fdw shared table inserts sequentially', async () => { + processRunner.run + .mockResolvedValueOnce({ + command: 'psql', + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout: 'INSERT 0 7\n', + }) + .mockResolvedValueOnce({ + command: 'psql', + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout: 'INSERT 0 0\n', + }); + const service = new SpaceDataDbCopyService(processRunner as never); + const onTableCopied = vi.fn(); + + await expect( + service.copySharedTablesViaPostgresFdw( + [ + { + table: 'record_history', + sql: 'fdw history', + target: { command: 'psql', args: ['history-target'] }, + }, + { + table: 'record_trash', + sql: 'fdw trash', + target: { command: 'psql', args: ['trash-target'] }, + }, + ], + { timeoutMs: 10_000 }, + { onTableCopied } + ) + ).resolves.toEqual([ + expect.objectContaining({ + strategy: 'postgres_fdw', + table: 'record_history', + copiedRows: 7, + }), + expect.objectContaining({ + strategy: 'postgres_fdw', + table: 'record_trash', + copiedRows: 0, + }), + ]); + + expect(processRunner.run).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ command: psqlCommand, args: historyTargetArgs }), + { timeoutMs: 10_000 } + ); + expect(processRunner.run).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ command: psqlCommand, args: trashTargetArgs }), + { timeoutMs: 10_000 } + ); + expect(onTableCopied).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ table: 'record_history', copiedRows: 7 }), + 0, + 2 + ); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-copy.service.ts b/apps/nestjs-backend/src/features/space/space-data-db-copy.service.ts new file mode 100644 index 0000000000..138f3bbc0b --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-copy.service.ts @@ -0,0 +1,347 @@ +import { mkdir, writeFile } from 'fs/promises'; +import path from 'path'; +import { Injectable } from '@nestjs/common'; +import { + buildBaseSchemaDumpRestorePlan, + buildBaseSchemaDumpStreamRestorePlan, + buildBaseSchemaPgcopydbPlan, + buildBaseSchemaRestoreListPlan, + buildBaseSchemaRestorePlan, + type ISpaceDataDbDumpStreamRestorePlan, + type ISharedTablePostgresFdwCopyPlan, + type ISharedTablePsqlCopyPlan, + type ISpaceDataDbDumpRestorePlan, + type ISpaceDataDbPgcopydbPlan, +} from './space-data-db-copy-plan'; +import { + SpaceDataDbProcessRunnerService, + type ISpaceDataDbProcessPipelineResult, + type ISpaceDataDbProcessRunOptions, + type ISpaceDataDbProcessRunResult, +} from './space-data-db-process-runner.service'; + +export const REQUIRED_POSTGRES_COPY_TOOLS = ['pg_dump', 'pg_restore', 'psql'] as const; +export const REQUIRED_PGCOPYDB_COPY_TOOLS = [...REQUIRED_POSTGRES_COPY_TOOLS, 'pgcopydb'] as const; +export const PG_RESTORE_LIST_STDOUT_LIMIT = 64 * 1024 * 1024; + +export type ISpaceDataDbBaseSchemaCopyStrategy = + | 'pg_dump_restore' + | 'pg_dump_stream_restore' + | 'pgcopydb'; +export type ISpaceDataDbSharedTableCopyStrategy = 'psql_copy' | 'postgres_fdw'; + +export type ISpaceDataDbExcludedForeignKey = { + schemaName: string; + tableName: string; + constraintName: string; + referencedSchemaName: string; + referencedTableName: string; +}; + +export type ISpaceDataDbFilteredRestoreList = { + restoreListFile: string; + requestedForeignKeyCount: number; + excludedEntryCount: number; + excludedForeignKeys: ISpaceDataDbExcludedForeignKey[]; +}; + +export type ISpaceDataDbBaseSchemaCopyResult = { + strategy: ISpaceDataDbBaseSchemaCopyStrategy; + plan: ISpaceDataDbDumpRestorePlan | ISpaceDataDbDumpStreamRestorePlan | ISpaceDataDbPgcopydbPlan; + dump?: ISpaceDataDbProcessRunResult; + stream?: ISpaceDataDbProcessPipelineResult; + restoreList?: ISpaceDataDbProcessRunResult; + filteredRestoreList?: ISpaceDataDbFilteredRestoreList; + restore?: ISpaceDataDbProcessRunResult; + pgcopydb?: ISpaceDataDbProcessRunResult; +}; + +export type ISpaceDataDbSharedTableCopyResult = ISpaceDataDbProcessPipelineResult & { + strategy?: ISpaceDataDbSharedTableCopyStrategy; + table: string; + copiedRows: number | null; +}; + +export type ISpaceDataDbPostgresFdwSharedTableCopyResult = { + strategy: 'postgres_fdw'; + table: string; + copiedRows: number | null; + target: ISpaceDataDbProcessRunResult; +}; + +type ISpaceDataDbSharedTableCopyHooks = { + onTableCopied?: ( + result: ISpaceDataDbSharedTableCopyResult, + index: number, + total: number + ) => void | Promise; +}; + +type ISpaceDataDbBaseSchemaCopyHooks = { + onDumpProgressPoll?: () => void | Promise; + onRestoreProgressPoll?: () => void | Promise; +}; + +const withProgressPoll = ( + options: ISpaceDataDbProcessRunOptions | undefined, + onPoll: (() => void | Promise) | undefined +): ISpaceDataDbProcessRunOptions | undefined => { + if (!onPoll) { + return options; + } + return { + ...options, + onPoll: async () => { + await options?.onPoll?.(); + await onPoll(); + }, + }; +}; + +const withMinimumStdoutLimit = ( + options: ISpaceDataDbProcessRunOptions | undefined, + stdoutLimit: number +): ISpaceDataDbProcessRunOptions => ({ + ...options, + stdoutLimit: Math.max(options?.stdoutLimit ?? 0, stdoutLimit), +}); + +export const parsePsqlCopyRowCount = (output: string): number | null => { + const matches = [...output.matchAll(/^COPY\s+(\d+)\s*$/gim)]; + const last = matches.at(-1)?.[1]; + if (last == null) { + return null; + } + const value = Number(last); + return Number.isSafeInteger(value) && value >= 0 ? value : null; +}; + +export const parsePsqlInsertRowCount = (output: string): number | null => { + const matches = [...output.matchAll(/^INSERT\s+\d+\s+(\d+)\s*$/gim)]; + const last = matches.at(-1)?.[1]; + if (last == null) { + return null; + } + const value = Number(last); + return Number.isSafeInteger(value) && value >= 0 ? value : null; +}; + +export const postgresCopyToolsForStrategy = (strategy: ISpaceDataDbBaseSchemaCopyStrategy) => + strategy === 'pgcopydb' ? [...REQUIRED_PGCOPYDB_COPY_TOOLS] : [...REQUIRED_POSTGRES_COPY_TOOLS]; + +export const pgRestoreListEntryMatchesForeignKey = ( + line: string, + foreignKey: ISpaceDataDbExcludedForeignKey +) => { + if (!line.includes(' FK CONSTRAINT ')) { + return false; + } + + const tokens = line.trim().split(/\s+/); + return ( + tokens.includes(foreignKey.schemaName) && + tokens.includes(foreignKey.tableName) && + tokens.includes(foreignKey.constraintName) + ); +}; + +export const filterPgRestoreListForForeignKeys = ( + content: string, + foreignKeys: ISpaceDataDbExcludedForeignKey[] +) => { + if (!foreignKeys.length) { + return { content, excludedEntryCount: 0 }; + } + + let excludedEntryCount = 0; + const filteredLines = content.split(/\r?\n/).filter((line) => { + const shouldExclude = foreignKeys.some((foreignKey) => + pgRestoreListEntryMatchesForeignKey(line, foreignKey) + ); + if (shouldExclude) { + excludedEntryCount += 1; + } + return !shouldExclude; + }); + + return { + content: filteredLines.join('\n'), + excludedEntryCount, + }; +}; + +@Injectable() +export class SpaceDataDbCopyService { + constructor(private readonly processRunner: SpaceDataDbProcessRunnerService) {} + + async assertPostgresToolsAvailable( + strategy: ISpaceDataDbBaseSchemaCopyStrategy = 'pg_dump_restore', + processOptions?: ISpaceDataDbProcessRunOptions + ): Promise { + const results: ISpaceDataDbProcessRunResult[] = []; + for (const command of postgresCopyToolsForStrategy(strategy)) { + results.push(await this.processRunner.run({ command, args: ['--version'] }, processOptions)); + } + return results; + } + + async copyBaseSchemas(input: { + sourceUrl: string; + targetUrl: string; + schemaNames: string[]; + workDir: string; + jobs?: number; + strategy?: ISpaceDataDbBaseSchemaCopyStrategy; + excludedForeignKeys?: ISpaceDataDbExcludedForeignKey[]; + processOptions?: ISpaceDataDbProcessRunOptions; + hooks?: ISpaceDataDbBaseSchemaCopyHooks; + }): Promise { + const excludedForeignKeys = input.excludedForeignKeys ?? []; + const requestedStrategy = input.strategy ?? 'pg_dump_stream_restore'; + const strategy = + requestedStrategy === 'pg_dump_stream_restore' && excludedForeignKeys.length + ? 'pg_dump_restore' + : requestedStrategy; + if (strategy === 'pgcopydb') { + if (excludedForeignKeys.length) { + throw new Error( + 'pgcopydb base schema copy does not support filtering out-of-space foreign keys; use pg_dump_restore for this migration' + ); + } + const plan = buildBaseSchemaPgcopydbPlan(input); + await mkdir(input.workDir, { recursive: true }); + await writeFile(plan.filterFile, plan.filterFileContent, 'utf8'); + const pgcopydb = await this.processRunner.run(plan.copy, input.processOptions); + return { + strategy, + plan, + pgcopydb, + }; + } + + if (strategy === 'pg_dump_stream_restore' && !excludedForeignKeys.length) { + const plan = buildBaseSchemaDumpStreamRestorePlan(input); + const onPoll = + input.hooks?.onDumpProgressPoll || input.hooks?.onRestoreProgressPoll + ? async () => { + await input.hooks?.onDumpProgressPoll?.(); + await input.hooks?.onRestoreProgressPoll?.(); + } + : undefined; + const stream = await this.processRunner.runPipeline( + plan, + withProgressPoll(input.processOptions, onPoll) + ); + return { + strategy, + plan, + stream, + }; + } + + const plan = buildBaseSchemaDumpRestorePlan(input); + const dump = await this.processRunner.run( + plan.dump, + withProgressPoll(input.processOptions, input.hooks?.onDumpProgressPoll) + ); + let restoreList: ISpaceDataDbProcessRunResult | undefined; + let filteredRestoreList: ISpaceDataDbFilteredRestoreList | undefined; + if (excludedForeignKeys.length) { + plan.restoreList = buildBaseSchemaRestoreListPlan(plan.dumpFile); + restoreList = await this.processRunner.run( + plan.restoreList, + withMinimumStdoutLimit(input.processOptions, PG_RESTORE_LIST_STDOUT_LIMIT) + ); + const restoreListFile = path.join(input.workDir, 'base-schemas.restore.list'); + const filtered = filterPgRestoreListForForeignKeys(restoreList.stdout, excludedForeignKeys); + await mkdir(input.workDir, { recursive: true }); + await writeFile(restoreListFile, filtered.content, 'utf8'); + plan.restoreListFile = restoreListFile; + plan.restore = buildBaseSchemaRestorePlan({ + targetUrl: input.targetUrl, + dumpFile: plan.dumpFile, + jobs: input.jobs, + restoreListFile, + }); + filteredRestoreList = { + restoreListFile, + requestedForeignKeyCount: excludedForeignKeys.length, + excludedEntryCount: filtered.excludedEntryCount, + excludedForeignKeys, + }; + } + const restore = await this.processRunner.run( + plan.restore, + withProgressPoll(input.processOptions, input.hooks?.onRestoreProgressPoll) + ); + + return { + strategy, + plan, + dump, + restoreList, + filteredRestoreList, + restore, + }; + } + + async copySharedTable( + plan: ISharedTablePsqlCopyPlan, + processOptions?: ISpaceDataDbProcessRunOptions + ): Promise { + const result = await this.processRunner.runPipeline(plan, processOptions); + return { + strategy: 'psql_copy', + table: plan.table, + copiedRows: parsePsqlCopyRowCount(`${result.target.stdout}\n${result.target.stderr}`), + ...result, + }; + } + + async copySharedTables( + plans: ISharedTablePsqlCopyPlan[], + processOptions?: ISpaceDataDbProcessRunOptions, + hooks: ISpaceDataDbSharedTableCopyHooks = {} + ): Promise { + const results: ISpaceDataDbSharedTableCopyResult[] = []; + for (const [index, plan] of plans.entries()) { + const result = await this.copySharedTable(plan, processOptions); + results.push(result); + await hooks.onTableCopied?.(result, index, plans.length); + } + return results; + } + + async copySharedTableViaPostgresFdw( + plan: ISharedTablePostgresFdwCopyPlan, + processOptions?: ISpaceDataDbProcessRunOptions + ): Promise { + const target = await this.processRunner.run(plan.target, processOptions); + return { + strategy: 'postgres_fdw', + table: plan.table, + copiedRows: parsePsqlInsertRowCount(`${target.stdout}\n${target.stderr}`), + target, + }; + } + + async copySharedTablesViaPostgresFdw( + plans: ISharedTablePostgresFdwCopyPlan[], + processOptions?: ISpaceDataDbProcessRunOptions, + hooks: { + onTableCopied?: ( + result: ISpaceDataDbPostgresFdwSharedTableCopyResult, + index: number, + total: number + ) => void | Promise; + } = {} + ): Promise { + const results: ISpaceDataDbPostgresFdwSharedTableCopyResult[] = []; + for (const [index, plan] of plans.entries()) { + const result = await this.copySharedTableViaPostgresFdw(plan, processOptions); + results.push(result); + await hooks.onTableCopied?.(result, index, plans.length); + } + return results; + } +} diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration-guard.service.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration-guard.service.spec.ts new file mode 100644 index 0000000000..51d63ee9cc --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration-guard.service.spec.ts @@ -0,0 +1,211 @@ +import { HttpErrorCode } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SpaceDataDbMigrationGuardService } from './space-data-db-migration-guard.service'; + +describe('SpaceDataDbMigrationGuardService', () => { + const prismaService = { + spaceDataDbMigrationJob: { + findFirst: vi.fn(), + findMany: vi.fn(), + }, + base: { + findUnique: vi.fn(), + }, + tableMeta: { + findUnique: vi.fn(), + }, + }; + + beforeEach(() => { + prismaService.spaceDataDbMigrationJob.findFirst.mockReset(); + prismaService.spaceDataDbMigrationJob.findMany.mockReset().mockResolvedValue([]); + prismaService.base.findUnique.mockReset(); + prismaService.tableMeta.findUnique.mockReset(); + }); + + it('allows writes when no active migration job exists for the space', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue(null); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await expect(service.assertSpaceWritable('spcxxx')).resolves.toBeUndefined(); + + expect(prismaService.spaceDataDbMigrationJob.findFirst).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ spaceId: 'spcxxx' }), + }) + ); + }); + + it('allows writes when the migration job table has not been deployed yet', async () => { + const missingTableError = Object.assign( + new Error( + 'The table `public.space_data_db_migration_job` does not exist in the current database.' + ), + { code: 'P2021' } + ); + prismaService.spaceDataDbMigrationJob.findFirst.mockRejectedValue(missingTableError); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await expect(service.assertSpaceWritable('spcxxx')).resolves.toBeUndefined(); + + expect(prismaService.spaceDataDbMigrationJob.findMany).not.toHaveBeenCalled(); + }); + + it('rejects writes for a space that has an active migration job', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjxxx', + state: 'freezing_writes', + }); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await expect(service.assertSpaceWritable('spcxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + }), + }); + }); + + it('does not treat test-only migration jobs as source write blockers', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockImplementation(async (args) => { + expect(args).toMatchObject({ + where: { + spaceId: 'spcxxx', + switchOnCompletion: true, + }, + }); + return null; + }); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await expect(service.assertSpaceWritable('spcxxx')).resolves.toBeUndefined(); + }); + + it('allows writes when the migration job table has not been migrated yet', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockRejectedValue( + Object.assign(new Error('The table `public.space_data_db_migration_job` does not exist'), { + code: 'P2021', + }) + ); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await expect(service.assertSpaceWritable('spcxxx')).resolves.toBeUndefined(); + }); + + it('rejects writes for a related space included in a grouped active migration job', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue(null); + prismaService.spaceDataDbMigrationJob.findMany.mockResolvedValue([ + { + id: 'sdmjgroup', + state: 'copying', + spaceId: 'spcprimary', + inventory: { + spaceIds: ['spcprimary', 'spcrelated'], + }, + }, + ]); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await expect(service.assertSpaceWritable('spcrelated')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjgroup', + }), + }); + }); + + it('resolves base and table ids to space ids before checking the freeze', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue(null); + prismaService.base.findUnique.mockResolvedValue({ spaceId: 'spcxxx' }); + prismaService.tableMeta.findUnique.mockResolvedValue({ base: { spaceId: 'spcxxx' } }); + const service = new SpaceDataDbMigrationGuardService(prismaService as never); + + await service.assertBaseWritable('bsexxx'); + await service.assertTableWritable('tblxxx'); + + expect(prismaService.base.findUnique).toHaveBeenCalledWith({ + where: { id: 'bsexxx' }, + select: { spaceId: true }, + }); + expect(prismaService.tableMeta.findUnique).toHaveBeenCalledWith({ + where: { id: 'tblxxx' }, + select: { base: { select: { spaceId: true } } }, + }); + expect(prismaService.spaceDataDbMigrationJob.findFirst).toHaveBeenCalledTimes(2); + }); + + it('uses the active transaction client for id resolution but root client for migration jobs', async () => { + const txClient = { + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + }, + base: { + findUnique: vi.fn().mockResolvedValue({ spaceId: 'spctx' }), + }, + tableMeta: { + findUnique: vi.fn().mockResolvedValue({ base: { spaceId: 'spctx' } }), + }, + }; + const rootPrisma = { + ...prismaService, + txClient: vi.fn().mockReturnValue(txClient), + }; + rootPrisma.spaceDataDbMigrationJob.findFirst.mockResolvedValue(null); + const service = new SpaceDataDbMigrationGuardService(rootPrisma as never); + + await service.assertBaseWritable('bsetx'); + await service.assertTableWritable('tbltx'); + + expect(txClient.base.findUnique).toHaveBeenCalledWith({ + where: { id: 'bsetx' }, + select: { spaceId: true }, + }); + expect(txClient.tableMeta.findUnique).toHaveBeenCalledWith({ + where: { id: 'tbltx' }, + select: { base: { select: { spaceId: true } } }, + }); + expect(rootPrisma.spaceDataDbMigrationJob.findFirst).toHaveBeenCalledTimes(2); + expect(txClient.spaceDataDbMigrationJob.findFirst).not.toHaveBeenCalled(); + expect(prismaService.base.findUnique).not.toHaveBeenCalled(); + expect(prismaService.tableMeta.findUnique).not.toHaveBeenCalled(); + }); + + it('does not poison the active transaction when the migration job table is missing', async () => { + const missingTableError = Object.assign( + new Error( + 'The table `public.space_data_db_migration_job` does not exist in the current database.' + ), + { code: 'P2021' } + ); + const txClient = { + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockRejectedValue(new Error('should not use transaction client')), + findMany: vi.fn(), + }, + base: { + findUnique: vi.fn().mockResolvedValue({ spaceId: 'spctx' }), + }, + tableMeta: { + findUnique: vi.fn().mockResolvedValue({ base: { spaceId: 'spctx' } }), + }, + }; + const rootPrisma = { + ...prismaService, + txClient: vi.fn().mockReturnValue(txClient), + }; + rootPrisma.spaceDataDbMigrationJob.findFirst.mockRejectedValue(missingTableError); + const service = new SpaceDataDbMigrationGuardService(rootPrisma as never); + + await expect(service.assertTableWritable('tbltx')).resolves.toBeUndefined(); + + expect(txClient.tableMeta.findUnique).toHaveBeenCalledWith({ + where: { id: 'tbltx' }, + select: { base: { select: { spaceId: true } } }, + }); + expect(rootPrisma.spaceDataDbMigrationJob.findFirst).toHaveBeenCalledTimes(1); + expect(txClient.spaceDataDbMigrationJob.findFirst).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration-guard.service.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration-guard.service.ts new file mode 100644 index 0000000000..8217a2b866 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration-guard.service.ts @@ -0,0 +1,140 @@ +import { Injectable } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { CustomHttpException } from '../../custom.exception'; +import { + activeSpaceDataDbMigrationStates, + spaceDataDbMigratingErrorCode, +} from './space-data-db-migration.constants'; + +type IMigrationJobClient = { + spaceDataDbMigrationJob: { + findFirst(args: unknown): Promise<{ id: string; state: string } | null>; + findMany( + args: unknown + ): Promise<{ id: string; state: string; spaceId: string; inventory: unknown }[]>; + }; + base: PrismaService['base']; + tableMeta: PrismaService['tableMeta']; + txClient?: () => IMigrationJobClient; +}; + +type IMigrationJobReader = Pick; + +@Injectable() +export class SpaceDataDbMigrationGuardService { + constructor(private readonly prismaService: PrismaService) {} + + async assertSpaceWritable(spaceId: string): Promise { + const activeJob = await this.findActiveMigrationForSpace(spaceId); + + if (!activeJob) { + return; + } + + throw new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbMigratingErrorCode, + migrationJobId: activeJob.id, + migrationState: activeJob.state, + spaceId, + } + ); + } + + private async findActiveMigrationForSpace(spaceId: string) { + try { + const directJob = await this.migrationJobClient.spaceDataDbMigrationJob.findFirst({ + where: { + spaceId, + switchOnCompletion: true, + state: { in: [...activeSpaceDataDbMigrationStates] }, + }, + select: { id: true, state: true }, + }); + if (directJob) { + return directJob; + } + + const activeJobs = await this.migrationJobClient.spaceDataDbMigrationJob.findMany({ + where: { + switchOnCompletion: true, + state: { in: [...activeSpaceDataDbMigrationStates] }, + }, + select: { id: true, state: true, spaceId: true, inventory: true }, + }); + return activeJobs.find((job) => this.inventoryContainsSpace(job.inventory, spaceId)) ?? null; + } catch (error) { + if (this.isMissingMigrationJobTableError(error)) { + return null; + } + throw error; + } + } + + private inventoryContainsSpace(inventory: unknown, spaceId: string) { + if (!inventory || typeof inventory !== 'object') { + return false; + } + const candidate = inventory as { spaceIds?: unknown; relatedSpaces?: { spaces?: unknown } }; + if (Array.isArray(candidate.spaceIds) && candidate.spaceIds.includes(spaceId)) { + return true; + } + return ( + Array.isArray(candidate.relatedSpaces?.spaces) && + candidate.relatedSpaces.spaces.some( + (space) => + space && typeof space === 'object' && (space as { spaceId?: unknown }).spaceId === spaceId + ) + ); + } + + private isMissingMigrationJobTableError(error: unknown) { + const code = + typeof error === 'object' && error !== null && 'code' in error + ? String((error as { code?: unknown }).code) + : ''; + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('space_data_db_migration_job') && + (code === 'P2021' || + code === 'P2022' || + code === '42P01' || + message.includes('does not exist') || + message.includes('relation')) + ); + } + + async assertBaseWritable(baseId: string): Promise { + const base = await this.prismaClient.base.findUnique({ + where: { id: baseId }, + select: { spaceId: true }, + }); + if (!base) { + throw new CustomHttpException(`Base ${baseId} not found`, HttpErrorCode.NOT_FOUND); + } + await this.assertSpaceWritable(base.spaceId); + } + + async assertTableWritable(tableId: string): Promise { + const table = await this.prismaClient.tableMeta.findUnique({ + where: { id: tableId }, + select: { base: { select: { spaceId: true } } }, + }); + if (!table) { + throw new CustomHttpException(`Table ${tableId} not found`, HttpErrorCode.NOT_FOUND); + } + await this.assertSpaceWritable(table.base.spaceId); + } + + private get prismaClient(): IMigrationJobClient { + const client = this.prismaService as unknown as IMigrationJobClient; + return client.txClient?.() ?? client; + } + + private get migrationJobClient(): IMigrationJobReader { + return this.prismaService as unknown as IMigrationJobReader; + } +} diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration-worker.service.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration-worker.service.spec.ts new file mode 100644 index 0000000000..4d1300c814 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration-worker.service.spec.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SpaceDataDbMigrationWorkerService } from './space-data-db-migration-worker.service'; + +describe('SpaceDataDbMigrationWorkerService', () => { + const workerId = 'worker-test'; + const migrationService = { + recoverStaleActiveMigrationJobs: vi.fn(), + claimNextPendingMigrationJob: vi.fn(), + runMigrationJob: vi.fn(), + }; + + beforeEach(() => { + vi.stubEnv('BYODB_SPACE_DATA_DB_MIGRATION_WORKER_ID', workerId); + migrationService.recoverStaleActiveMigrationJobs.mockReset().mockResolvedValue([]); + migrationService.claimNextPendingMigrationJob.mockReset().mockResolvedValue(null); + migrationService.runMigrationJob.mockReset().mockResolvedValue({ state: 'succeeded' }); + }); + + it('returns null when there is no pending job', async () => { + const service = new SpaceDataDbMigrationWorkerService(migrationService as never); + + await expect(service.runOnce()).resolves.toBeNull(); + + expect(migrationService.recoverStaleActiveMigrationJobs).toHaveBeenCalledWith(workerId); + expect(migrationService.claimNextPendingMigrationJob).toHaveBeenCalledWith(workerId); + expect(migrationService.runMigrationJob).not.toHaveBeenCalled(); + }); + + it('runs a claimed migration job', async () => { + migrationService.claimNextPendingMigrationJob.mockResolvedValue({ jobId: 'sdmjxxx' }); + const service = new SpaceDataDbMigrationWorkerService(migrationService as never); + + await expect(service.runOnce()).resolves.toEqual({ + jobId: 'sdmjxxx', + status: 'succeeded', + }); + + expect(migrationService.runMigrationJob).toHaveBeenCalledWith('sdmjxxx'); + }); + + it('logs and reports a failed claimed migration job without throwing', async () => { + migrationService.claimNextPendingMigrationJob.mockResolvedValue({ jobId: 'sdmjxxx' }); + migrationService.runMigrationJob.mockRejectedValue(new Error('copy failed')); + const service = new SpaceDataDbMigrationWorkerService(migrationService as never); + + await expect(service.runOnce()).resolves.toEqual({ + jobId: 'sdmjxxx', + status: 'failed', + error: 'copy failed', + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration-worker.service.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration-worker.service.ts new file mode 100644 index 0000000000..43cb129111 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration-worker.service.ts @@ -0,0 +1,101 @@ +import { hostname } from 'os'; +import { Injectable, Logger } from '@nestjs/common'; +import { SpaceDataDbMigrationService } from './space-data-db-migration.service'; + +type ISpaceDataDbMigrationWorkerRunResult = { + jobId: string; + status: 'succeeded' | 'failed'; + error?: string; +}; + +const defaultPollMs = 5000; +const defaultErrorBackoffMs = 10000; + +const readPositiveIntegerEnv = (key: string, fallback: number) => { + const value = Number.parseInt(process.env[key] ?? '', 10); + return Number.isFinite(value) && value > 0 ? value : fallback; +}; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +@Injectable() +export class SpaceDataDbMigrationWorkerService { + private readonly logger = new Logger(SpaceDataDbMigrationWorkerService.name); + private stopped = false; + + constructor(private readonly migrationService: SpaceDataDbMigrationService) {} + + stop() { + this.stopped = true; + } + + async runOnce(): Promise { + const workerId = this.getWorkerId(); + const recoveredJobs = await this.migrationService.recoverStaleActiveMigrationJobs(workerId); + for (const job of recoveredJobs) { + this.logger.warn( + `Recovered stale BYODB space data DB migration job ${job.jobId} from ${job.state}: ${job.lastError}` + ); + } + const claimedJob = await this.migrationService.claimNextPendingMigrationJob(workerId); + + if (!claimedJob) { + return null; + } + + try { + this.logger.log(`Running BYODB space data DB migration job ${claimedJob.jobId}`); + await this.migrationService.runMigrationJob(claimedJob.jobId); + this.logger.log(`Completed BYODB space data DB migration job ${claimedJob.jobId}`); + return { jobId: claimedJob.jobId, status: 'succeeded' }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error( + `Failed BYODB space data DB migration job ${claimedJob.jobId}: ${message}`, + error instanceof Error ? error.stack : undefined + ); + return { jobId: claimedJob.jobId, status: 'failed', error: message }; + } + } + + async runForever(options: { pollMs?: number; errorBackoffMs?: number } = {}) { + this.stopped = false; + const pollMs = + options.pollMs ?? + readPositiveIntegerEnv('BYODB_SPACE_DATA_DB_MIGRATION_WORKER_POLL_MS', defaultPollMs); + const errorBackoffMs = + options.errorBackoffMs ?? + readPositiveIntegerEnv( + 'BYODB_SPACE_DATA_DB_MIGRATION_WORKER_ERROR_BACKOFF_MS', + defaultErrorBackoffMs + ); + + this.logger.log( + `BYODB space data DB migration worker ${this.getWorkerId()} started; pollMs=${pollMs}` + ); + + while (!this.stopped) { + try { + const result = await this.runOnce(); + if (!result) { + await delay(pollMs); + } else if (result.status === 'failed') { + await delay(errorBackoffMs); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error( + `BYODB space data DB migration worker loop failed: ${message}`, + error instanceof Error ? error.stack : undefined + ); + await delay(errorBackoffMs); + } + } + + this.logger.log(`BYODB space data DB migration worker ${this.getWorkerId()} stopped`); + } + + private getWorkerId() { + return process.env.BYODB_SPACE_DATA_DB_MIGRATION_WORKER_ID ?? `${hostname()}:${process.pid}`; + } +} diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration.constants.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration.constants.ts new file mode 100644 index 0000000000..7a93d10fab --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration.constants.ts @@ -0,0 +1,51 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +export const migrateSpaceTargetMode = 'migrate-space'; +export const spaceDataDbAdminOnlyErrorCode = 'SPACE_DATA_DB_ADMIN_ONLY'; +export const spaceDataDbAdminOnlyMessage = + 'Space data database migration is only available from the admin panel'; + +export const activeSpaceDataDbMigrationStates = [ + 'pending', + 'waiting_worker', + 'preflight', + 'freezing_writes', + 'copying', + 'validating', + 'switching', +] as const; + +export const preCopyCancelableSpaceDataDbMigrationStates = [ + 'pending', + 'waiting_worker', + 'preflight', + 'freezing_writes', +] as const; +export const cancelableSpaceDataDbMigrationStates = [ + ...preCopyCancelableSpaceDataDbMigrationStates, + 'copying', +] as const; + +export const spaceDataDbMigratingErrorCode = 'SPACE_DATA_DB_MIGRATING'; +export const spaceDataDbMigrationActiveErrorCode = 'SPACE_DATA_DB_MIGRATION_ACTIVE'; +export const spaceDataDbMigrationCanceledErrorCode = 'SPACE_DATA_DB_MIGRATION_CANCELED'; +export const spaceDataDbMigrationCancelConflictErrorCode = + 'SPACE_DATA_DB_MIGRATION_CANCEL_CONFLICT'; +export const spaceDataDbTargetConflictErrorCode = 'SPACE_DATA_DB_TARGET_CONFLICT'; +export const spaceDataDbTargetCleanupFailedErrorCode = 'SPACE_DATA_DB_TARGET_CLEANUP_FAILED'; +export const spaceDataDbTargetExtensionMissingErrorCode = 'SPACE_DATA_DB_TARGET_EXTENSION_MISSING'; +export const spaceDataDbLargeMigrationConfirmationRequiredErrorCode = + 'SPACE_DATA_DB_LARGE_MIGRATION_CONFIRMATION_REQUIRED'; +export const spaceDataDbRollbackUnsafeErrorCode = 'SPACE_DATA_DB_ROLLBACK_UNSAFE'; +export const spaceDataDbValidationMismatchErrorCode = 'SPACE_DATA_DB_VALIDATION_MISMATCH'; +export const spaceDataDbInventoryChangedErrorCode = 'SPACE_DATA_DB_INVENTORY_CHANGED'; +export const spaceDataDbRelatedSpacesRequiredErrorCode = 'SPACE_DATA_DB_RELATED_SPACES_REQUIRED'; +export const spaceDataDbTempDiskInsufficientErrorCode = 'SPACE_DATA_DB_TEMP_DISK_INSUFFICIENT'; +export const spaceDataDbTargetDiskInsufficientErrorCode = 'SPACE_DATA_DB_TARGET_DISK_INSUFFICIENT'; +export const spaceDataDbPostgresToolUnavailableErrorCode = + 'SPACE_DATA_DB_POSTGRES_TOOL_UNAVAILABLE'; +export const spaceDataDbComputedDrainTimeoutErrorCode = 'SPACE_DATA_DB_COMPUTED_DRAIN_TIMEOUT'; +export const spaceDataDbSchemaOperationDrainTimeoutErrorCode = + 'SPACE_DATA_DB_SCHEMA_OPERATION_DRAIN_TIMEOUT'; +export const spaceDataDbBackgroundWriterDrainTimeoutErrorCode = + 'SPACE_DATA_DB_BACKGROUND_WRITER_DRAIN_TIMEOUT'; +export const spaceDataDbStaleActiveJobErrorCode = 'SPACE_DATA_DB_STALE_ACTIVE_JOB'; diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration.service.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration.service.spec.ts new file mode 100644 index 0000000000..429680cacb --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration.service.spec.ts @@ -0,0 +1,5911 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { FieldType, HttpErrorCode } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { fingerprintDatabaseUrl, fingerprintDataDbConnection } from './data-db-preflight.service'; +import { encryptDataDbUrl } from './data-db-url-secret'; +import { SpaceDataDbMigrationService } from './space-data-db-migration.service'; +import { + SpaceDataDbProcessCanceledError, + SpaceDataDbProcessError, + SpaceDataDbProcessPipelineError, +} from './space-data-db-process-runner.service'; + +const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; +const internalSchema = 'teable_meta_test'; +const schemaVersion = '20260421000000_init_data_db_baseline'; +const capabilities = { + createSchema: true, + createTable: true, + createFunction: true, + createTrigger: true, + createRole: false, + grantPrivileges: true, + inspectActivity: true, +}; + +const processResult = (command: string, stdout = '') => ({ + command, + args: [], + exitCode: 0, + signal: null, + stderr: '', + stdout, + startedAt: '2026-05-06T00:00:00.000Z', + completedAt: '2026-05-06T00:00:01.000Z', + durationMs: 1000, +}); + +const columnSignatureRows = ( + columns: { + ordinalPosition?: number; + columnName: string; + formattedType: string; + notNull?: boolean; + defaultExpression?: string | null; + }[] = [ + { columnName: '__id', formattedType: 'text', notNull: true }, + { columnName: 'fldName', formattedType: 'text' }, + ] +) => + columns.map((column, index) => ({ + ordinalPosition: column.ordinalPosition ?? index + 1, + columnName: column.columnName, + formattedType: column.formattedType, + notNull: column.notNull ?? false, + defaultExpression: column.defaultExpression ?? null, + identity: '', + generated: '', + collation: 'default', + })); + +const indexSignatureRows = ( + indexes: { + indexName: string; + isPrimary?: boolean; + isUnique?: boolean; + definition: string; + }[] = [] +) => + indexes.map((index) => ({ + indexName: index.indexName, + isPrimary: index.isPrimary ?? false, + isUnique: index.isUnique ?? false, + isValid: true, + definition: index.definition, + })); + +const constraintSignatureRows = ( + constraints: { + constraintName: string; + constraintType: string; + definition: string; + }[] = [] +) => constraints; + +const triggerSignatureRows = ( + triggers: { + triggerName: string; + enabled?: string; + definition: string; + }[] = [] +) => + triggers.map((trigger) => ({ + triggerName: trigger.triggerName, + enabled: trigger.enabled ?? 'O', + definition: trigger.definition, + })); + +describe('SpaceDataDbMigrationService', () => { + const txClient = { + dataDbConnection: { + upsert: vi.fn(), + update: vi.fn(), + }, + spaceDataDbBinding: { + upsert: vi.fn(), + }, + spaceDataDbMigrationJob: { + create: vi.fn(), + update: vi.fn(), + }, + }; + const prismaService = { + $tx: vi.fn(async (fn: (client: typeof txClient) => Promise) => fn(txClient)), + spaceDataDbBinding: { + findUnique: vi.fn(), + findMany: vi.fn(), + }, + spaceDataDbMigrationJob: { + findFirst: vi.fn(), + findMany: vi.fn(), + findUnique: vi.fn(), + update: vi.fn(), + updateMany: vi.fn(), + }, + schemaOperation: { + count: vi.fn(), + findMany: vi.fn(), + }, + base: { + count: vi.fn(), + findMany: vi.fn(), + }, + space: { + findMany: vi.fn(), + }, + tableMeta: { + count: vi.fn(), + findMany: vi.fn(), + }, + field: { + count: vi.fn(), + findMany: vi.fn(), + }, + }; + const preflightService = { + preflight: vi.fn(), + }; + const baselineService = { + initialize: vi.fn(), + getLatestSchemaVersion: vi.fn(), + }; + const sourceDataPrisma = { + $queryRawUnsafe: vi.fn(), + }; + const dataDbClientManager = { + dataPrismaForSpace: vi.fn(), + getDataDatabaseForSpace: vi.fn(), + invalidateConnection: vi.fn(), + }; + const copyService = { + assertPostgresToolsAvailable: vi.fn(), + copyBaseSchemas: vi.fn(), + copySharedTables: vi.fn(), + copySharedTablesViaPostgresFdw: vi.fn(), + }; + const targetClient = { + raw: vi.fn(), + destroy: vi.fn(), + }; + const sourceClient = { + raw: vi.fn(), + destroy: vi.fn(), + }; + + beforeEach(() => { + vi.stubEnv('PRISMA_DATABASE_URL', 'postgresql://source.example/teable'); + txClient.dataDbConnection.upsert.mockReset().mockResolvedValue({ id: 'dcnxxx' }); + txClient.dataDbConnection.update.mockReset().mockResolvedValue(undefined); + txClient.spaceDataDbBinding.upsert.mockReset().mockResolvedValue(undefined); + txClient.spaceDataDbMigrationJob.create.mockReset().mockResolvedValue({ id: 'sdmjxxx' }); + txClient.spaceDataDbMigrationJob.update.mockReset().mockResolvedValue(undefined); + prismaService.$tx.mockClear(); + prismaService.spaceDataDbBinding.findUnique.mockReset().mockResolvedValue(null); + prismaService.spaceDataDbMigrationJob.findFirst.mockReset().mockResolvedValue(null); + prismaService.spaceDataDbMigrationJob.findMany.mockReset().mockResolvedValue([]); + prismaService.spaceDataDbMigrationJob.findUnique.mockReset().mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + prismaService.spaceDataDbMigrationJob.update.mockReset().mockResolvedValue(undefined); + prismaService.spaceDataDbMigrationJob.updateMany.mockReset().mockResolvedValue({ count: 0 }); + prismaService.schemaOperation.count.mockReset().mockResolvedValue(0); + prismaService.schemaOperation.findMany.mockReset().mockResolvedValue([]); + prismaService.base.count.mockReset().mockResolvedValue(0); + prismaService.base.findMany.mockReset().mockResolvedValue([ + { + id: 'bsexxx', + tables: [{ id: 'tblxxx', dbTableName: 'bsexxx.sheet1' }], + }, + ]); + prismaService.space.findMany.mockReset().mockResolvedValue([ + { + id: 'spcxxx', + name: 'Space', + baseGroup: [ + { + id: 'bsexxx', + tables: [{ id: 'tblxxx' }], + }, + ], + }, + ]); + prismaService.tableMeta.count.mockReset().mockResolvedValue(0); + prismaService.tableMeta.findMany.mockReset().mockImplementation((args?: unknown) => { + const where = (args as { where?: { baseId?: { in?: string[] }; id?: { in?: string[] } } }) + ?.where; + if (where?.baseId?.in) { + return Promise.resolve( + where.baseId.in.flatMap((baseId) => (baseId === 'bsexxx' ? [{ id: 'tblxxx' }] : [])) + ); + } + if (where?.id?.in?.includes('tblxxx')) { + return Promise.resolve([{ id: 'tblxxx', dbTableName: 'bsexxx.sheet1' }]); + } + return Promise.resolve([ + { + id: 'tblxxx', + base: { spaceId: 'spcxxx' }, + dbTableName: 'bsexxx.sheet1', + }, + ]); + }); + prismaService.field.count.mockReset().mockResolvedValue(0); + prismaService.field.findMany.mockReset().mockResolvedValue([]); + prismaService.spaceDataDbBinding.findMany.mockReset().mockResolvedValue([]); + preflightService.preflight.mockReset().mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'empty', + capabilities, + errors: [], + }); + baselineService.initialize.mockReset().mockResolvedValue(schemaVersion); + baselineService.getLatestSchemaVersion.mockReset().mockReturnValue(schemaVersion); + sourceDataPrisma.$queryRawUnsafe.mockReset().mockResolvedValue([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]); + dataDbClientManager.dataPrismaForSpace.mockReset().mockResolvedValue(sourceDataPrisma); + dataDbClientManager.getDataDatabaseForSpace.mockReset().mockImplementation((_, options) => { + if ('sourceConnectionId' in (options ?? {})) { + if (options.sourceConnectionId === 'dcnsource') { + return Promise.resolve({ + cacheKey: 'dcnsource', + connectionId: 'dcnsource', + internalSchema: 'source_schema', + isMetaFallback: false, + url: 'postgresql://source.example/byodb?schema=source_schema&options=-c+search_path%3Dsource_schema', + }); + } + return Promise.resolve({ + cacheKey: 'meta-fallback', + connectionId: undefined, + internalSchema: undefined, + isMetaFallback: true, + url: 'postgresql://source.example/teable', + }); + } + if (options?.previewBinding) { + const previewSchema = options.previewBinding.internalSchema; + return Promise.resolve({ + cacheKey: 'dcnxxx', + connectionId: 'dcnxxx', + internalSchema: previewSchema, + isMetaFallback: false, + url: `${dataUrl}?schema=${previewSchema}&options=-c+search_path%3D${previewSchema}`, + }); + } + return Promise.resolve({ + cacheKey: 'meta-fallback', + connectionId: undefined, + internalSchema: undefined, + isMetaFallback: true, + url: 'postgresql://source.example/teable', + }); + }); + dataDbClientManager.invalidateConnection.mockReset(); + copyService.assertPostgresToolsAvailable + .mockReset() + .mockResolvedValue([ + processResult('pg_dump'), + processResult('pg_restore'), + processResult('psql'), + ]); + copyService.copyBaseSchemas.mockReset().mockImplementation((input?: { strategy?: string }) => { + if (input?.strategy === 'pg_dump_restore') { + return Promise.resolve({ + strategy: 'pg_dump_restore', + dump: processResult('pg_dump'), + restore: processResult('pg_restore'), + }); + } + return Promise.resolve({ + strategy: 'pg_dump_stream_restore', + stream: { + source: processResult('pg_dump'), + target: processResult('pg_restore'), + }, + }); + }); + copyService.copySharedTables.mockReset().mockResolvedValue([ + { + strategy: 'psql_copy', + table: 'record_history', + copiedRows: 5, + source: processResult('psql'), + target: processResult('psql', 'COPY 5\n'), + }, + ]); + copyService.copySharedTablesViaPostgresFdw.mockReset().mockResolvedValue([ + { + strategy: 'postgres_fdw', + table: 'record_history', + copiedRows: 5, + target: processResult('psql', 'INSERT 0 5\n'), + }, + ]); + targetClient.raw.mockReset().mockResolvedValue({ rows: [] }); + targetClient.destroy.mockReset().mockResolvedValue(undefined); + sourceClient.raw.mockReset().mockResolvedValue({ rows: [] }); + sourceClient.destroy.mockReset().mockResolvedValue(undefined); + }); + + const createService = ( + statfs?: never, + queues?: { + baseImportCsvQueue?: unknown; + baseImportJunctionCsvQueue?: unknown; + tableImportCsvChunkQueue?: unknown; + tableImportCsvQueue?: unknown; + } + ) => + new SpaceDataDbMigrationService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + copyService as never, + (url) => (url.includes('source.example') ? sourceClient : targetClient), + statfs, + queues?.baseImportCsvQueue as never, + queues?.baseImportJunctionCsvQueue as never, + queues?.tableImportCsvChunkQueue as never, + queues?.tableImportCsvQueue as never + ); + + const mockValidationClient = ( + client: typeof sourceClient, + rowCount: number, + signatures: { + columns?: ReturnType; + indexes?: ReturnType; + constraints?: ReturnType; + triggers?: ReturnType; + } = {} + ) => { + const columns = signatures.columns ?? columnSignatureRows(); + const indexes = signatures.indexes ?? indexSignatureRows(); + const constraints = signatures.constraints ?? constraintSignatureRows(); + const triggers = signatures.triggers ?? triggerSignatureRows(); + client.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + }, + ], + }; + } + if (sql.includes('FROM pg_attribute a')) { + return { rows: columns }; + } + if (sql.includes('FROM pg_index i')) { + return { rows: indexes }; + } + if (sql.includes('FROM pg_constraint con')) { + return { rows: constraints }; + } + if (sql.includes('FROM pg_trigger tg')) { + return { rows: triggers }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: String(rowCount) }] }; + } + if (sql.includes('to_regprocedure')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('__teable_data_schema_migrations')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('COUNT(*)')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [] }; + }); + }; + + it('rejects concurrent active migration for the same space', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjactive', + state: 'copying', + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_MIGRATION_ACTIVE', + migrationJobId: 'sdmjactive', + }), + }); + expect(preflightService.preflight).not.toHaveBeenCalled(); + expect(targetClient.raw).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('returns preflight errors when the migration target URL cannot be parsed', async () => { + preflightService.preflight.mockResolvedValue({ + ok: false, + provider: 'postgres', + classification: 'non-empty-unknown', + capabilities, + errors: [{ code: 'INVALID_DATABASE_URL', message: 'Invalid URL' }], + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: 'not-a-postgres-url', + targetMode: 'migrate-space', + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + preflight: expect.objectContaining({ + errors: [expect.objectContaining({ code: 'INVALID_DATABASE_URL' })], + }), + }), + }); + expect(targetClient.raw).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('persists source physical schema inventory for DB-to-DB copy planning', async () => { + sourceDataPrisma.$queryRawUnsafe.mockResolvedValue([ + { + schemaName: 'bsexxx', + relationName: 'junction_tblxxx_fldxxx', + relationKind: 'table', + totalBytes: '2048', + estimatedRows: '3', + }, + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + { + schemaName: 'bsexxx', + relationName: 'sheet1___auto_number_seq', + relationKind: 'sequence', + totalBytes: '512', + estimatedRows: null, + }, + ]); + const service = createService(); + + await service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }); + + expect(dataDbClientManager.dataPrismaForSpace).toHaveBeenCalledWith('spcxxx'); + expect(sourceDataPrisma.$queryRawUnsafe).toHaveBeenCalledWith( + expect.stringContaining('FROM pg_class c'), + ['bsexxx'] + ); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + inventory: expect.objectContaining({ + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { + internalSchema, + }, + physicalSchemas: [ + { + schemaName: 'bsexxx', + totalBytes: 3584, + estimatedRows: 13, + relations: [ + { + schemaName: 'bsexxx', + relationName: 'junction_tblxxx_fldxxx', + relationKind: 'table', + totalBytes: 2048, + estimatedRows: 3, + }, + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + { + schemaName: 'bsexxx', + relationName: 'sheet1___auto_number_seq', + relationKind: 'sequence', + totalBytes: 512, + estimatedRows: null, + }, + ], + }, + ], + postgresExtensionDependencies: [], + outOfScopeForeignKeys: [], + estimatedTotalBytes: 3584, + estimatedTotalRows: 13, + }), + }), + select: { id: true }, + }); + }); + + it('persists out-of-space foreign keys for filtered per-space restore planning', async () => { + sourceDataPrisma.$queryRawUnsafe + .mockResolvedValueOnce([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ]); + const service = createService(); + + await service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }); + + expect(sourceDataPrisma.$queryRawUnsafe).toHaveBeenCalledWith( + expect.stringContaining('referenced_ns.nspname <> ALL($1::text[])'), + ['bsexxx'] + ); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + inventory: expect.objectContaining({ + outOfScopeForeignKeys: [ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ], + }), + }), + select: { id: true }, + }); + }); + + it('requires explicit confirmation before starting a large estimated migration', async () => { + const largeBytes = 51 * 1024 * 1024 * 1024; + sourceDataPrisma.$queryRawUnsafe.mockResolvedValue([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: String(largeBytes), + estimatedRows: '10', + }, + ]); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_LARGE_MIGRATION_CONFIRMATION_REQUIRED', + confirmationField: 'confirmLargeMigration', + estimatedTotalBytes: largeBytes, + estimatedTotalRows: 10, + thresholds: expect.objectContaining({ + bytes: 50 * 1024 * 1024 * 1024, + }), + }), + }); + expect(targetClient.raw).not.toHaveBeenCalled(); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('starts a large estimated migration when operator confirmation is present', async () => { + const largeBytes = 51 * 1024 * 1024 * 1024; + sourceDataPrisma.$queryRawUnsafe.mockResolvedValue([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: String(largeBytes), + estimatedRows: '10', + }, + ]); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + confirmLargeMigration: true, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + inventory: expect.objectContaining({ + estimatedTotalBytes: largeBytes, + estimatedTotalRows: 10, + }), + }), + select: { id: true }, + }); + }); + + it('rejects migrate-space when target disk capacity is known to be insufficient', async () => { + const estimatedBytes = 1024; + targetClient.raw.mockImplementation((query: string) => { + if (query.includes("current_setting('data_directory')")) { + return Promise.resolve({ rows: [{ setting: '/home/postgres/pgdata/pgroot/data' }] }); + } + if (query.includes('SELECT line FROM __teable_target_disk_capacity')) { + return Promise.resolve({ rows: [{ line: '/home/postgres/pgdata|4096|1024|3072' }] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_DISK_INSUFFICIENT', + estimatedTotalBytes: estimatedBytes, + requiredBytes: 1024 * 1024 * 1024, + availableBytes: 3072, + targetDisk: expect.objectContaining({ + checked: true, + mountPath: '/home/postgres/pgdata', + }), + }), + }); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + expect(targetClient.destroy).toHaveBeenCalled(); + }); + + it('does not block migrate-space when target disk capacity cannot be inspected', async () => { + targetClient.raw.mockImplementation((query: string) => { + if (query.includes("current_setting('data_directory')")) { + return Promise.resolve({ rows: [{ setting: '/var/lib/postgresql/data' }] }); + } + if (query.includes('COPY __teable_target_disk_capacity FROM PROGRAM')) { + return Promise.reject( + new Error('must be superuser to COPY to or from an external program') + ); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalled(); + }); + + it('cleans retryable same-job target artifacts before starting a new migration', async () => { + const matchingInventory = { + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { + internalSchema, + }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + totalBytes: 1024, + estimatedRows: 10, + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }; + prismaService.spaceDataDbMigrationJob.findFirst + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ + id: 'sdmjold', + state: 'failed', + targetConnectionId: 'dcnxxx', + inventory: matchingInventory, + }); + targetClient.raw.mockResolvedValue({ rows: [] }); + const service = createService(); + const cleanup = vi.spyOn(service, 'cleanupTargetArtifactsForJob').mockResolvedValue({ + reason: 'retry_before_start', + baseSchemas: [], + sharedTables: [], + startedAt: new Date().toISOString(), + completedAt: new Date().toISOString(), + }); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(cleanup).toHaveBeenCalledWith('sdmjold', 'retry_before_start', { + truncateSharedTables: true, + }); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + }); + + it('cleans previous successful dry-run target artifacts before starting a new migration', async () => { + prismaService.spaceDataDbMigrationJob.findFirst + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ + id: 'sdmjdryrun', + spaceId: 'spcxxx', + state: 'succeeded', + switchOnCompletion: false, + targetConnectionId: 'dcnxxx', + inventory: { + targetDataDb: { internalSchema }, + baseIds: ['bseold'], + tableIds: [], + dbTableNames: [], + physicalSchemas: [{ schemaName: 'bseold', relations: [] }], + }, + }); + targetClient.raw.mockResolvedValue({ rows: [] }); + const service = createService(); + const cleanup = vi.spyOn(service, 'cleanupTargetArtifactsForJob').mockResolvedValue({ + reason: 'retry_before_start', + baseSchemas: [], + sharedTables: [], + startedAt: new Date().toISOString(), + completedAt: new Date().toISOString(), + }); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(cleanup).toHaveBeenCalledWith('sdmjdryrun', 'retry_before_start', { + truncateSharedTables: true, + }); + expect(prismaService.spaceDataDbMigrationJob.findFirst).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + where: expect.objectContaining({ + OR: expect.arrayContaining([{ state: 'succeeded', switchOnCompletion: false }]), + }), + }) + ); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + switchOnCompletion: false, + state: 'waiting_worker', + }), + }) + ); + }); + + it('allows preflight to pass when conflicts belong to a previous successful dry-run', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValueOnce({ + id: 'sdmjdryrun', + spaceId: 'spcxxx', + state: 'succeeded', + switchOnCompletion: false, + targetConnectionId: 'dcnxxx', + inventory: { + targetDataDb: { internalSchema }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [] }], + }, + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM information_schema.schemata')) { + return Promise.resolve({ rows: [{ schemaName: 'bsexxx' }] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.preflightMigrationTargetForSpace('spcxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toMatchObject({ + ok: true, + internalSchema, + }); + + expect( + targetClient.raw.mock.calls.some(([sql]) => + String(sql).includes('FROM information_schema.schemata') + ) + ).toBe(false); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('does not clean retry target artifacts when the previous job inventory differs', async () => { + prismaService.spaceDataDbMigrationJob.findFirst + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ + id: 'sdmjold', + state: 'failed', + targetConnectionId: 'dcnxxx', + inventory: { + targetDataDb: { internalSchema }, + baseIds: ['bseother'], + tableIds: [], + dbTableNames: [], + physicalSchemas: [], + }, + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM information_schema.schemata')) { + return Promise.resolve({ rows: [{ schemaName: 'bsexxx' }] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + const cleanup = vi.spyOn(service, 'cleanupTargetArtifactsForJob'); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_CONFLICT', + }), + }); + + expect(cleanup).not.toHaveBeenCalled(); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('rejects target databases with conflicting base schemas before creating a job', async () => { + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM information_schema.schemata')) { + return Promise.resolve({ rows: [{ schemaName: 'bsexxx' }] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_CONFLICT', + }), + }); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + expect(targetClient.destroy).toHaveBeenCalled(); + }); + + it('rejects target databases with conflicting shared rows before creating a job', async () => { + targetClient.raw.mockImplementation((sql: string, bindings?: unknown[]) => { + if (sql.includes('FROM information_schema.schemata')) { + return Promise.resolve({ rows: [] }); + } + if (sql.includes('SELECT to_regclass')) { + const qualifiedTable = Array.isArray(bindings) ? bindings[0] : undefined; + return Promise.resolve({ + rows: [{ exists: qualifiedTable === `"${internalSchema}"."record_history"` }], + }); + } + if (sql.includes(`FROM "${internalSchema}"."record_history"`)) { + return Promise.resolve({ rows: [{ count: '2' }] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_CONFLICT', + conflicts: [{ object: `table:${internalSchema}.record_history`, count: 2 }], + }), + }); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + expect(targetClient.destroy).toHaveBeenCalled(); + }); + + it('rejects migrate-space when the source needs pg_trgm but the target cannot create it', async () => { + sourceDataPrisma.$queryRawUnsafe.mockImplementation((query: string) => { + if (query.includes('pg_get_indexdef')) { + return Promise.resolve([ + { + extensionName: 'pg_trgm', + objectType: 'operator_class', + schemaName: 'public', + objectName: 'gin_trgm_ops', + accessMethod: 'gin', + sourceSchemaName: 'bsexxx', + sourceRelationName: 'sheet1', + sourceIndexName: 'sheet1_name_trgm_idx', + }, + ]); + } + return Promise.resolve([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]); + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('CREATE EXTENSION')) { + return Promise.reject(new Error('permission denied to create extension "pg_trgm"')); + } + if (sql.includes('FROM pg_opclass opc')) { + return Promise.resolve({ rows: [] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_EXTENSION_MISSING', + errors: [ + expect.objectContaining({ + code: 'MISSING_POSTGRES_EXTENSION', + message: expect.stringContaining('permission denied to create extension "pg_trgm"'), + remediation: expect.stringContaining('CREATE EXTENSION IF NOT EXISTS pg_trgm'), + }), + ], + missingExtensions: [ + expect.objectContaining({ + extensionName: 'pg_trgm', + schemaName: 'public', + objectName: 'gin_trgm_ops', + }), + ], + }), + }); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + expect(targetClient.raw).toHaveBeenCalledWith( + 'CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "public"' + ); + expect(targetClient.destroy).toHaveBeenCalled(); + }); + + it('starts migrate-space after creating the required pg_trgm operator class on target', async () => { + sourceDataPrisma.$queryRawUnsafe.mockImplementation((query: string) => { + if (query.includes('pg_get_indexdef')) { + return Promise.resolve([ + { + extensionName: 'pg_trgm', + objectType: 'operator_class', + schemaName: 'public', + objectName: 'gin_trgm_ops', + accessMethod: 'gin', + sourceSchemaName: 'bsexxx', + sourceRelationName: 'sheet1', + sourceIndexName: 'sheet1_name_trgm_idx', + }, + ]); + } + return Promise.resolve([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]); + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('CREATE EXTENSION')) { + return { + rows: [], + }; + } + if (sql.includes('FROM pg_opclass opc')) { + return { + rows: [{ schemaName: 'public', objectName: 'gin_trgm_ops', accessMethod: 'gin' }], + }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + expect(targetClient.raw).toHaveBeenCalledWith( + 'CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "public"' + ); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + inventory: expect.objectContaining({ + postgresExtensionDependencies: [ + expect.objectContaining({ + extensionName: 'pg_trgm', + schemaName: 'public', + objectName: 'gin_trgm_ops', + sourceObjects: ['bsexxx.sheet1.sheet1_name_trgm_idx'], + }), + ], + }), + }), + select: { id: true }, + }); + }); + + it('accepts pg_trgm installed in the target extensions schema when it is search-path visible', async () => { + sourceDataPrisma.$queryRawUnsafe.mockImplementation((query: string) => { + if (query.includes('pg_get_indexdef')) { + return Promise.resolve([ + { + extensionName: 'pg_trgm', + objectType: 'operator_class', + schemaName: 'public', + objectName: 'gin_trgm_ops', + accessMethod: 'gin', + sourceSchemaName: 'bsexxx', + sourceRelationName: 'sheet1', + sourceIndexName: 'sheet1_name_trgm_idx', + }, + ]); + } + return Promise.resolve([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]); + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('CREATE EXTENSION') && sql.includes('WITH SCHEMA "public"')) { + return Promise.reject(new Error('permission denied for schema public')); + } + if (sql.includes('CREATE EXTENSION') && sql.includes('WITH SCHEMA "extensions"')) { + return Promise.resolve({ rows: [] }); + } + if (sql.includes('FROM pg_opclass opc')) { + return Promise.resolve({ + rows: [{ schemaName: 'extensions', objectName: 'gin_trgm_ops', accessMethod: 'gin' }], + }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(targetClient.raw).toHaveBeenCalledWith( + 'CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "public"' + ); + expect(targetClient.raw).toHaveBeenCalledWith( + 'CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "extensions"' + ); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + }); + + it('preflights migrate-space pg_trgm dependencies before target schema initialization', async () => { + sourceDataPrisma.$queryRawUnsafe.mockImplementation((query: string) => { + if (query.includes('pg_get_indexdef')) { + return Promise.resolve([ + { + extensionName: 'pg_trgm', + objectType: 'operator_class', + schemaName: 'public', + objectName: 'gin_trgm_ops', + accessMethod: 'gin', + sourceSchemaName: 'bsexxx', + sourceRelationName: 'sheet1', + sourceIndexName: 'sheet1_name_trgm_idx', + }, + ]); + } + return Promise.resolve([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]); + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('CREATE EXTENSION')) { + return Promise.reject(new Error('permission denied to create extension "pg_trgm"')); + } + if (sql.includes('FROM pg_opclass opc')) { + return Promise.resolve({ rows: [] }); + } + return Promise.resolve({ rows: [] }); + }); + const service = createService(); + + await expect( + service.preflightMigrationTargetForSpace('spcxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_EXTENSION_MISSING', + errors: [ + expect.objectContaining({ + code: 'MISSING_POSTGRES_EXTENSION', + message: expect.stringContaining('permission denied to create extension "pg_trgm"'), + }), + ], + }), + }); + + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }); + expect(targetClient.raw).toHaveBeenCalledWith( + 'CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "public"' + ); + expect(baselineService.initialize).not.toHaveBeenCalled(); + expect(prismaService.$tx).not.toHaveBeenCalled(); + }); + + it('creates a migration job without switching routing when preflight passes', async () => { + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }) + ).resolves.toEqual({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + }); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + expect(txClient.dataDbConnection.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + create: expect.objectContaining({ + status: 'migrating', + schemaVersion, + }), + update: expect.objectContaining({ + status: 'migrating', + schemaVersion, + }), + }) + ); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetMode: 'migrate-space', + switchOnCompletion: false, + state: 'waiting_worker', + inventory: expect.objectContaining({ + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { + internalSchema, + }, + relatedSpaces: expect.objectContaining({ + primarySpaceId: 'spcxxx', + hasCrossSpaceLinks: false, + }), + spaceIds: ['spcxxx'], + copySpaceIds: ['spcxxx'], + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + sharedTableIds: ['tblxxx'], + relatedSharedTableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + totalBytes: 1024, + estimatedRows: 10, + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + }, + ], + postgresExtensionDependencies: [], + outOfScopeForeignKeys: [], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }), + createdBy: 'usrxxx', + }), + select: { id: true }, + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + }); + + it('starts and switches a migration for cross-space linked spaces together', async () => { + prismaService.field.findMany.mockResolvedValue([ + { + id: 'fldlink', + type: FieldType.Link, + isLookup: null, + isConditionalLookup: false, + options: JSON.stringify({ foreignTableId: 'tblrelated' }), + lookupOptions: null, + tableId: 'tblxxx', + table: { + id: 'tblxxx', + base: { spaceId: 'spcxxx' }, + }, + }, + ]); + prismaService.tableMeta.findMany + .mockResolvedValueOnce([ + { + id: 'tblrelated', + base: { spaceId: 'spcrelated' }, + }, + ]) + .mockResolvedValueOnce([ + { id: 'tblxxx', dbTableName: 'bsexxx.sheet1' }, + { id: 'tblrelated', dbTableName: 'bserelated.sheet1' }, + ]); + prismaService.space.findMany.mockResolvedValue([ + { + id: 'spcxxx', + name: 'Primary', + baseGroup: [{ id: 'bsexxx', tables: [{ id: 'tblxxx' }] }], + }, + { + id: 'spcrelated', + name: 'Related', + baseGroup: [{ id: 'bserelated', tables: [{ id: 'tblrelated' }] }], + }, + ]); + sourceDataPrisma.$queryRawUnsafe.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return Promise.resolve([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + { + schemaName: 'bserelated', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '2048', + estimatedRows: '20', + }, + ]); + } + return Promise.resolve([]); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + internalSchema, + switchOnCompletion: true, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + inventory: expect.objectContaining({ + spaceIds: ['spcrelated', 'spcxxx'], + copySpaceIds: ['spcrelated', 'spcxxx'], + baseIds: ['bserelated', 'bsexxx'], + tableIds: ['tblrelated', 'tblxxx'], + dbTableNames: ['bserelated.sheet1', 'bsexxx.sheet1'], + relatedSpaces: expect.objectContaining({ + primarySpaceId: 'spcxxx', + hasCrossSpaceLinks: true, + spaces: expect.arrayContaining([ + expect.objectContaining({ spaceId: 'spcxxx', isPrimary: true }), + expect.objectContaining({ spaceId: 'spcrelated', isPrimary: false }), + ]), + links: [ + expect.objectContaining({ + fromSpaceId: 'spcxxx', + toSpaceId: 'spcrelated', + fromFieldId: 'fldlink', + }), + ], + }), + }), + }), + select: { id: true }, + }); + + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: null, + inventory: txClient.spaceDataDbMigrationJob.create.mock.calls[0][0].data.inventory, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + vi.spyOn(service, 'validateCopyForJob').mockResolvedValue({ + phase: 'validation_completed', + progress: { + phase: 'validation_completed', + totalSteps: 9, + completedSteps: 8, + percent: 88.9, + estimatedTotalBytes: 3072, + completedEstimatedBytes: 3072, + estimatedTotalRows: 30, + completedEstimatedRows: 30, + startedAt: '2026-05-06T00:00:00.000Z', + updatedAt: '2026-05-06T00:01:00.000Z', + etaMs: null, + }, + targetSchemaVersion: { latest: schemaVersion, exists: true }, + routeSmoke: { + ok: true, + connectionId: 'dcnxxx', + internalSchema, + cacheKey: 'dcnxxx', + isMetaFallback: false, + }, + baseSchemas: [], + sharedTables: [], + undoFunction: { exists: true }, + completedAt: '2026-05-06T00:01:00.000Z', + }); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + }); + + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ where: { spaceId: 'spcrelated' } }) + ); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ where: { spaceId: 'spcxxx' } }) + ); + }); + + it('starts a repair migration for default spaces linked to an existing target-bound space', async () => { + const targetInternalSchema = 'spcalready'; + prismaService.field.findMany.mockResolvedValue([ + { + id: 'fldlink', + type: FieldType.Link, + isLookup: null, + isConditionalLookup: false, + options: JSON.stringify({ foreignTableId: 'tblalready' }), + lookupOptions: null, + tableId: 'tblrelated', + table: { + id: 'tblrelated', + base: { spaceId: 'spcrelated' }, + }, + }, + ]); + prismaService.tableMeta.findMany + .mockResolvedValueOnce([ + { + id: 'tblalready', + base: { spaceId: 'spcalready' }, + }, + ]) + .mockResolvedValueOnce([{ id: 'tblrelated', dbTableName: 'bserelated.sheet1' }]) + .mockResolvedValueOnce([{ id: 'tblrelated' }, { id: 'tbldeletedrelated' }]) + .mockResolvedValueOnce([ + { id: 'tblalready' }, + { id: 'tbldeletedalready' }, + { id: 'tblrelated' }, + { id: 'tbldeletedrelated' }, + ]); + prismaService.space.findMany.mockResolvedValue([ + { + id: 'spcalready', + name: 'Already', + baseGroup: [{ id: 'bsealready', tables: [{ id: 'tblalready' }] }], + }, + { + id: 'spcrelated', + name: 'Related', + baseGroup: [{ id: 'bserelated', tables: [{ id: 'tblrelated' }] }], + }, + ]); + prismaService.spaceDataDbBinding.findMany.mockResolvedValue([ + { + spaceId: 'spcalready', + mode: 'byodb', + state: 'ready', + dataDbConnection: { + id: 'dcnalready', + encryptedUrl: encryptDataDbUrl(dataUrl), + urlFingerprint: fingerprintDataDbConnection(dataUrl, targetInternalSchema), + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema: targetInternalSchema, + }, + }, + ]); + sourceDataPrisma.$queryRawUnsafe.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return Promise.resolve([ + { + schemaName: 'bserelated', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '2048', + estimatedRows: '20', + }, + ]); + } + return Promise.resolve([]); + }); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcrelated', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + switchOnCompletion: true, + }) + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + connectionId: 'dcnxxx', + }); + + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: dataUrl, + targetMode: 'migrate-space', + internalSchema: targetInternalSchema, + }); + expect(txClient.dataDbConnection.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + where: { urlFingerprint: fingerprintDataDbConnection(dataUrl, targetInternalSchema) }, + }) + ); + expect(txClient.spaceDataDbMigrationJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + spaceId: 'spcrelated', + targetInternalSchema, + inventory: expect.objectContaining({ + spaceIds: ['spcalready', 'spcrelated'], + copySpaceIds: ['spcrelated'], + baseIds: ['bserelated'], + tableIds: ['tblrelated'], + sharedTableIds: ['tbldeletedrelated', 'tblrelated'], + relatedSharedTableIds: [ + 'tblalready', + 'tbldeletedalready', + 'tbldeletedrelated', + 'tblrelated', + ], + dbTableNames: ['bserelated.sheet1'], + relatedSpaces: expect.objectContaining({ + primarySpaceId: 'spcrelated', + hasCrossSpaceLinks: true, + spaces: expect.arrayContaining([ + expect.objectContaining({ + spaceId: 'spcalready', + dataDbMode: 'byodb', + dataDbDatabaseFingerprint: fingerprintDatabaseUrl(dataUrl), + }), + expect.objectContaining({ spaceId: 'spcrelated', dataDbMode: 'default' }), + ]), + }), + }), + }), + select: { id: true }, + }); + + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcrelated', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema, + createdBy: 'usrxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: null, + inventory: txClient.spaceDataDbMigrationJob.create.mock.calls[0][0].data.inventory, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + vi.spyOn(service, 'validateCopyForJob').mockResolvedValue({ + phase: 'validation_completed', + progress: { + phase: 'validation_completed', + totalSteps: 9, + completedSteps: 8, + percent: 88.9, + estimatedTotalBytes: 2048, + completedEstimatedBytes: 2048, + estimatedTotalRows: 20, + completedEstimatedRows: 20, + startedAt: '2026-05-06T00:00:00.000Z', + updatedAt: '2026-05-06T00:01:00.000Z', + etaMs: null, + }, + targetSchemaVersion: { latest: schemaVersion, exists: true }, + routeSmoke: { + ok: true, + connectionId: 'dcnxxx', + internalSchema: targetInternalSchema, + cacheKey: 'dcnxxx', + isMetaFallback: false, + }, + baseSchemas: [], + sharedTables: [], + undoFunction: { exists: true }, + completedAt: '2026-05-06T00:01:00.000Z', + }); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + }); + + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ where: { spaceId: 'spcalready' } }) + ); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ where: { spaceId: 'spcrelated' } }) + ); + }); + + it('rejects a repair migration when linked BYODB spaces use another physical database', async () => { + const otherUrl = 'postgresql://teable:secret@other.example:5432/teable_data'; + prismaService.field.findMany.mockResolvedValue([ + { + id: 'fldlink', + type: FieldType.Link, + isLookup: null, + isConditionalLookup: false, + options: JSON.stringify({ foreignTableId: 'tblalready' }), + lookupOptions: null, + tableId: 'tblrelated', + table: { + id: 'tblrelated', + base: { spaceId: 'spcrelated' }, + }, + }, + ]); + prismaService.tableMeta.findMany.mockResolvedValueOnce([ + { + id: 'tblalready', + base: { spaceId: 'spcalready' }, + }, + ]); + prismaService.space.findMany.mockResolvedValue([ + { + id: 'spcalready', + name: 'Already', + baseGroup: [{ id: 'bsealready', tables: [{ id: 'tblalready' }] }], + }, + { + id: 'spcrelated', + name: 'Related', + baseGroup: [{ id: 'bserelated', tables: [{ id: 'tblrelated' }] }], + }, + ]); + prismaService.spaceDataDbBinding.findMany.mockResolvedValue([ + { + spaceId: 'spcalready', + mode: 'byodb', + state: 'ready', + dataDbConnection: { + id: 'dcnalready', + encryptedUrl: encryptDataDbUrl(otherUrl), + urlFingerprint: fingerprintDataDbConnection(otherUrl, 'spcalready'), + displayHost: 'other.example:5432', + displayDatabase: 'teable_data', + internalSchema: 'spcalready', + }, + }, + ]); + const service = createService(); + + await expect( + service.startMigrationForSpace('spcrelated', 'usrxxx', { + url: dataUrl, + targetMode: 'migrate-space', + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_RELATED_SPACES_REQUIRED', + mismatchedSpaceIds: ['spcalready'], + }), + }); + + expect(txClient.spaceDataDbMigrationJob.create).not.toHaveBeenCalled(); + }); + + it('claims the oldest waiting-worker migration job for a worker', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjwaiting', + state: 'waiting_worker', + }); + prismaService.spaceDataDbMigrationJob.updateMany.mockResolvedValue({ count: 1 }); + const service = createService(); + + await expect(service.claimNextPendingMigrationJob('worker-1')).resolves.toEqual({ + jobId: 'sdmjwaiting', + }); + + expect(prismaService.spaceDataDbMigrationJob.findFirst).toHaveBeenCalledWith({ + where: { + targetMode: 'migrate-space', + state: { in: ['waiting_worker', 'pending'] }, + }, + orderBy: [{ createdTime: 'asc' }, { id: 'asc' }], + select: { id: true, state: true }, + }); + expect(prismaService.spaceDataDbMigrationJob.updateMany).toHaveBeenCalledWith({ + where: { + id: 'sdmjwaiting', + state: 'waiting_worker', + }, + data: expect.objectContaining({ + state: 'freezing_writes', + startedAt: expect.any(Date), + lastError: null, + copyStats: expect.objectContaining({ + phase: 'worker_claimed', + worker: expect.objectContaining({ + id: 'worker-1', + previousState: 'waiting_worker', + claimedAt: expect.any(String), + }), + }), + }), + }); + }); + + it('still claims legacy pending migration jobs for a worker', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjpending', + state: 'pending', + }); + prismaService.spaceDataDbMigrationJob.updateMany.mockResolvedValue({ count: 1 }); + const service = createService(); + + await expect(service.claimNextPendingMigrationJob('worker-1')).resolves.toEqual({ + jobId: 'sdmjpending', + }); + + expect(prismaService.spaceDataDbMigrationJob.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + id: 'sdmjpending', + state: 'pending', + }, + data: expect.objectContaining({ + state: 'freezing_writes', + }), + }) + ); + }); + + it('does not claim a migration job when another worker wins the claim update', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjwaiting', + state: 'waiting_worker', + }); + prismaService.spaceDataDbMigrationJob.updateMany.mockResolvedValue({ count: 0 }); + const service = createService(); + + await expect(service.claimNextPendingMigrationJob('worker-1')).resolves.toBeNull(); + }); + + it('returns null when no pending migration job exists for a worker', async () => { + const service = createService(); + + await expect(service.claimNextPendingMigrationJob('worker-1')).resolves.toBeNull(); + + expect(prismaService.spaceDataDbMigrationJob.updateMany).not.toHaveBeenCalled(); + }); + + it('recovers stale active migration jobs left behind by interrupted workers', async () => { + const now = new Date('2026-05-06T01:00:00.000Z'); + prismaService.spaceDataDbMigrationJob.findMany.mockResolvedValue([ + { + id: 'sdmjstale', + spaceId: 'spcxxx', + state: 'copying', + targetInternalSchema: internalSchema, + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: null, + createdBy: 'usrxxx', + lastModifiedTime: new Date('2026-05-06T00:20:00.000Z'), + inventory: { baseIds: ['bsexxx'], tableIds: ['tblxxx'] }, + copyStats: { phase: 'copying_base_schemas' }, + validationStats: null, + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + targetConnection: { encryptedUrl: encryptDataDbUrl(dataUrl) }, + }, + ]); + prismaService.spaceDataDbMigrationJob.updateMany.mockResolvedValue({ count: 1 }); + const service = createService(); + const resumeSource = vi.spyOn(service, 'resumeSourceComputedForJob').mockResolvedValue({ + deleted: 1, + } as never); + const cleanupTarget = vi.spyOn(service, 'cleanupTargetArtifactsForJob').mockResolvedValue({ + reason: 'stale_active_job', + baseSchemas: [], + sharedTables: [], + startedAt: now.toISOString(), + completedAt: now.toISOString(), + } as never); + + await expect( + service.recoverStaleActiveMigrationJobs('worker-1', { now, staleAfterMs: 30 * 60 * 1000 }) + ).resolves.toEqual([ + expect.objectContaining({ + jobId: 'sdmjstale', + state: 'copying', + lastError: expect.stringContaining('no worker progress since 2026-05-06T00:20:00.000Z'), + }), + ]); + + expect(prismaService.spaceDataDbMigrationJob.findMany).toHaveBeenCalledWith({ + where: { + targetMode: 'migrate-space', + state: { in: ['preflight', 'freezing_writes', 'copying', 'validating', 'switching'] }, + OR: [ + { lastModifiedTime: null }, + { lastModifiedTime: { lt: new Date('2026-05-06T00:30:00.000Z') } }, + ], + }, + include: { targetConnection: true }, + orderBy: [{ lastModifiedTime: 'asc' }, { createdTime: 'asc' }, { id: 'asc' }], + }); + expect(prismaService.spaceDataDbMigrationJob.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + id: 'sdmjstale', + state: 'copying', + }), + data: expect.objectContaining({ + state: 'failed', + completedAt: now, + lastError: expect.stringContaining( + 'Space data database migration job sdmjstale is stale' + ), + copyStats: expect.objectContaining({ + phase: 'copying_base_schemas', + staleRecovery: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_STALE_ACTIVE_JOB', + workerId: 'worker-1', + previousState: 'copying', + staleAfterMs: 30 * 60 * 1000, + }), + }), + }), + }) + ); + expect(resumeSource).toHaveBeenCalledWith('sdmjstale'); + expect(cleanupTarget).toHaveBeenCalledWith('sdmjstale', 'stale_active_job', { + truncateSharedTables: true, + }); + }); + + it('copies base schemas for an existing migration job and records copy stats', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + outOfScopeForeignKeys: [ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: '10' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect( + service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 2, + timeoutMs: 1000, + }) + ).resolves.toMatchObject({ + phase: 'base_schemas_completed', + baseSchemas: { + schemaNames: ['bsexxx'], + copiedRelationCount: 1, + totalCopiedRows: 10, + copiedRelations: [ + expect.objectContaining({ + schemaName: 'bsexxx', + relationName: 'sheet1', + copiedRows: 10, + estimatedRows: 10, + }), + ], + }, + }); + + expect(dataDbClientManager.getDataDatabaseForSpace).toHaveBeenCalledWith('spcxxx', { + sourceConnectionId: null, + }); + expect(copyService.copyBaseSchemas).toHaveBeenCalledWith( + expect.objectContaining({ + sourceUrl: 'postgresql://source.example/teable', + targetUrl: dataUrl, + schemaNames: ['bsexxx'], + workDir: '/tmp/sdmjxxx', + jobs: 2, + strategy: 'pg_dump_restore', + excludedForeignKeys: [ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ], + processOptions: expect.objectContaining({ + timeoutMs: 1000, + pollMs: 5000, + pollFailureTimeoutMs: 30000, + pollTimeoutMs: 30000, + shouldCancel: expect.any(Function), + }), + hooks: expect.objectContaining({ + onDumpProgressPoll: expect.any(Function), + onRestoreProgressPoll: expect.any(Function), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'copying', + copyStats: expect.objectContaining({ + phase: 'copying_base_schemas', + progress: expect.objectContaining({ + estimatedTotalBytes: 1024, + completedEstimatedBytes: 0, + estimatedTotalRows: 10, + completedEstimatedRows: 0, + completedSteps: 5, + totalSteps: 9, + etaMs: expect.any(Number), + }), + }), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'copying', + copyStats: expect.objectContaining({ + phase: 'base_schemas_completed', + progress: expect.objectContaining({ + completedEstimatedBytes: 1024, + completedEstimatedRows: 10, + completedSteps: 6, + }), + baseSchemas: expect.objectContaining({ + copiedRelationCount: 1, + totalCopiedRows: 10, + strategy: 'pg_dump_restore', + dump: expect.objectContaining({ + command: 'pg_dump', + startedAt: '2026-05-06T00:00:00.000Z', + completedAt: '2026-05-06T00:00:01.000Z', + durationMs: 1000, + }), + restore: expect.objectContaining({ + command: 'pg_restore', + durationMs: 1000, + }), + }), + }), + }), + }) + ); + }); + + it('uses the inventory source connection when the current binding already points at the target', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: null, + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + }, + }); + dataDbClientManager.getDataDatabaseForSpace.mockImplementation((_, options) => { + if ('sourceConnectionId' in (options ?? {})) { + return Promise.resolve({ + cacheKey: 'meta-fallback', + isMetaFallback: true, + url: 'postgresql://source.example/teable', + }); + } + return Promise.resolve({ + cacheKey: 'dcnxxx', + connectionId: 'dcnxxx', + internalSchema, + isMetaFallback: false, + url: `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}`, + }); + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: '10' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect( + service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 2, + }) + ).resolves.toMatchObject({ + phase: 'base_schemas_completed', + }); + + expect(dataDbClientManager.getDataDatabaseForSpace).toHaveBeenCalledWith('spcxxx', { + sourceConnectionId: null, + }); + expect(copyService.copyBaseSchemas).toHaveBeenCalledWith( + expect.objectContaining({ + sourceUrl: 'postgresql://source.example/teable', + targetUrl: dataUrl, + }) + ); + }); + + it('records active pg_stat_progress_copy samples while dumping and restoring base schemas', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + sourceClient.raw.mockImplementation((sql: string) => { + if (sql.includes('pg_stat_progress_copy')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + command: 'COPY TO', + copyType: 'PIPE', + bytesProcessed: '512', + bytesTotal: '1024', + tuplesProcessed: '4', + tuplesExcluded: '0', + }, + ], + }; + } + return { rows: [] }; + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('pg_stat_progress_copy')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + command: 'COPY FROM', + copyType: 'PIPE', + bytesProcessed: '768', + bytesTotal: '1024', + tuplesProcessed: '7', + tuplesExcluded: '0', + }, + ], + }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: '10' }] }; + } + return { rows: [] }; + }); + copyService.copyBaseSchemas.mockImplementation( + async (input: { + hooks?: { + onDumpProgressPoll?: () => Promise; + onRestoreProgressPoll?: () => Promise; + }; + }) => { + await input.hooks?.onDumpProgressPoll?.(); + await input.hooks?.onRestoreProgressPoll?.(); + return { + strategy: 'pg_dump_stream_restore', + stream: { + source: processResult('pg_dump'), + target: processResult('pg_restore'), + }, + }; + } + ); + const service = createService(); + + await expect( + service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + timeoutMs: 1000, + progressPollMs: 250, + }) + ).resolves.toMatchObject({ + phase: 'base_schemas_completed', + }); + + const progressUpdates = prismaService.spaceDataDbMigrationJob.update.mock.calls + .map(([args]) => args) + .filter((args) => args.data.copyStats?.baseSchemas?.activeCopy); + + expect(progressUpdates).toHaveLength(2); + expect(progressUpdates[0]).toMatchObject({ + data: { + copyStats: { + phase: 'copying_base_schemas', + baseSchemas: { + strategy: 'pg_dump_stream_restore', + activeCopy: { + phase: 'dump', + activeRelationCount: 1, + activeRelations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + command: 'COPY TO', + bytesProcessed: 512, + bytesTotal: 1024, + tuplesProcessed: 4, + estimatedRows: 10, + totalBytes: 1024, + }, + ], + }, + }, + }, + }, + }); + expect(progressUpdates[1]).toMatchObject({ + data: { + copyStats: { + baseSchemas: { + activeCopy: { + phase: 'restore', + activeRelations: [ + expect.objectContaining({ + command: 'COPY FROM', + bytesProcessed: 768, + tuplesProcessed: 7, + }), + ], + }, + }, + }, + }, + }); + }); + + it('records a base schema heartbeat before pg_stat_progress_copy inspection resolves', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + let resolveProgressQuery: (value: { rows: unknown[] }) => void = vi.fn(); + const progressQuery = new Promise<{ rows: unknown[] }>((resolve) => { + resolveProgressQuery = resolve; + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('pg_stat_progress_copy')) { + return progressQuery; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: '10' }] }; + } + return { rows: [] }; + }); + copyService.copyBaseSchemas.mockImplementation( + async (input: { + hooks?: { + onRestoreProgressPoll?: () => Promise; + }; + }) => { + const pollPromise = input.hooks?.onRestoreProgressPoll?.(); + await Promise.resolve(); + expect( + prismaService.spaceDataDbMigrationJob.update.mock.calls.some(([args]) => { + const heartbeat = args.data.copyStats?.baseSchemas?.heartbeat; + return heartbeat?.stage === 'progress_poll' && heartbeat?.phase === 'restore'; + }) + ).toBe(true); + resolveProgressQuery({ rows: [] }); + await pollPromise; + return { + strategy: 'pg_dump_stream_restore', + stream: { + source: processResult('pg_dump'), + target: processResult('pg_restore'), + }, + }; + } + ); + const service = createService(); + + await expect( + service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + progressPollMs: 250, + }) + ).resolves.toMatchObject({ + phase: 'base_schemas_completed', + }); + }); + + it('keeps base schema copy fresh while collecting post-copy row stats', async () => { + vi.useFakeTimers(); + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + let resolveCount: (value: { rows: { count: string }[] }) => void = vi.fn(); + const countRows = new Promise<{ rows: { count: string }[] }>((resolve) => { + resolveCount = resolve; + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return countRows; + } + return { rows: [] }; + }); + const service = createService(); + + const promise = service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + progressPollMs: 50, + }); + await vi.advanceTimersByTimeAsync(150); + + const heartbeatCount = prismaService.spaceDataDbMigrationJob.update.mock.calls.filter( + ([args]) => args.data.copyStats?.baseSchemas?.heartbeat?.stage === 'post_copy_stats' + ).length; + expect(heartbeatCount).toBeGreaterThanOrEqual(2); + + resolveCount({ rows: [{ count: '10' }] }); + await expect(promise).resolves.toMatchObject({ + phase: 'base_schemas_completed', + }); + vi.useRealTimers(); + }); + + it('marks the migration failed when base schema copy fails', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: [], + physicalSchemas: [], + }, + }); + copyService.copyBaseSchemas.mockRejectedValue( + new SpaceDataDbProcessError('pg_dump failed', { + ...processResult('pg_dump'), + exitCode: 1, + stderr: 'dump stderr', + }) + ); + const service = createService(); + + await expect( + service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + }) + ).rejects.toThrow('pg_dump failed'); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'pg_dump failed: pg_dump - exit 1 - dump stderr', + copyStats: expect.objectContaining({ + phase: 'base_schemas_failed', + baseSchemas: expect.objectContaining({ + error: 'pg_dump failed: pg_dump - exit 1 - dump stderr', + failure: expect.objectContaining({ + type: 'process', + result: expect.objectContaining({ + command: 'pg_dump', + exitCode: 1, + stderr: 'dump stderr', + durationMs: 1000, + }), + }), + }), + }), + }), + }) + ); + }); + + it('preserves canceled state when a base schema copy process is killed by cancel', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'copying', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: [], + physicalSchemas: [], + }, + }); + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'canceled', + }); + copyService.copyBaseSchemas.mockRejectedValue( + new SpaceDataDbProcessCanceledError({ + ...processResult('pg_dump'), + command: 'pg_dump', + args: [], + exitCode: null, + signal: 'SIGTERM', + stderr: '', + stdout: '', + }) + ); + const service = createService(); + + await expect( + service.copyBaseSchemasForJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + }) + ).rejects.toBeInstanceOf(SpaceDataDbProcessCanceledError); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledTimes(1); + expect(prismaService.spaceDataDbMigrationJob.update).not.toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + state: 'failed', + }), + }) + ); + }); + + it('copies shared rows for an existing migration job and records copy stats', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetInternalSchema: internalSchema, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { schemaName: 'bsexxx', relations: [], totalBytes: 1024, estimatedRows: 10 }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + const service = createService(); + + await expect( + service.copySharedRowsForJob('sdmjxxx', { timeoutMs: 1000 }) + ).resolves.toMatchObject({ + phase: 'shared_rows_completed', + sharedTables: { + copiedTableCount: 1, + totalCopiedRows: 5, + copiedTables: [{ table: 'record_history', copiedRows: 5 }], + }, + }); + + expect(dataDbClientManager.getDataDatabaseForSpace).toHaveBeenCalledWith('spcxxx', { + sourceConnectionId: null, + }); + expect(copyService.copySharedTables).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + table: 'record_history', + sourceSql: expect.stringContaining('FROM "public"."record_history"'), + targetSql: expect.stringContaining(`COPY "${internalSchema}"."record_history"`), + }), + expect.objectContaining({ + table: 'computed_update_outbox', + sourceSql: expect.stringContaining(`"base_id" = ANY(ARRAY['bsexxx']::text[])`), + }), + ]), + expect.objectContaining({ + timeoutMs: 1000, + onPoll: expect.any(Function), + shouldCancel: expect.any(Function), + }), + expect.objectContaining({ + onTableCopied: expect.any(Function), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'copying', + copyStats: expect.objectContaining({ phase: 'copying_shared_rows' }), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'copying', + copyStats: expect.objectContaining({ + phase: 'shared_rows_completed', + progress: expect.objectContaining({ + completedSteps: 7, + completedEstimatedBytes: 1024, + completedEstimatedRows: 10, + }), + sharedTables: expect.objectContaining({ + strategy: 'psql_copy', + totalCopiedRows: 5, + copiedTables: expect.arrayContaining([ + expect.objectContaining({ + strategy: 'psql_copy', + table: 'record_history', + copiedRows: 5, + source: expect.objectContaining({ + command: 'psql', + durationMs: 1000, + }), + target: expect.objectContaining({ + command: 'psql', + stdout: 'COPY 5\n', + durationMs: 1000, + }), + }), + ]), + }), + }), + }), + }) + ); + }); + + it('copies shared rows only for the default subset during repair migrations', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcrelated', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetInternalSchema: internalSchema, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + relatedSpaces: { + primarySpaceId: 'spcrelated', + hasCrossSpaceLinks: true, + spaces: [ + { + spaceId: 'spcalready', + name: 'Already', + isPrimary: false, + baseIds: ['bsealready'], + tableIds: ['tblalready'], + dataDbMode: 'byodb', + }, + { + spaceId: 'spcrelated', + name: 'Related', + isPrimary: true, + baseIds: ['bserelated'], + tableIds: ['tblrelated'], + dataDbMode: 'default', + }, + ], + links: [], + }, + spaceIds: ['spcalready', 'spcrelated'], + copySpaceIds: ['spcrelated'], + baseIds: ['bserelated'], + tableIds: ['tblrelated'], + sharedTableIds: ['tblrelated', 'tbldeletedrelated'], + dbTableNames: ['bserelated.sheet1'], + physicalSchemas: [ + { schemaName: 'bserelated', relations: [], totalBytes: 1024, estimatedRows: 10 }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + const service = createService(); + + await expect( + service.copySharedRowsForJob('sdmjxxx', { timeoutMs: 1000 }) + ).resolves.toMatchObject({ + phase: 'shared_rows_completed', + }); + + expect(copyService.copySharedTables).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + table: 'record_history', + sourceSql: expect.stringContaining( + `"table_id" = ANY(ARRAY['tblrelated', 'tbldeletedrelated']::text[])` + ), + }), + expect.objectContaining({ + table: 'computed_update_outbox_seed', + sourceSql: expect.stringContaining(`"table_id" = ANY(ARRAY['tblrelated']::text[])`), + }), + expect.objectContaining({ + table: 'computed_update_pause_scope', + sourceSql: expect.stringContaining(`"scope_id" = ANY(ARRAY['spcrelated']::text[])`), + }), + ]), + expect.anything(), + expect.anything() + ); + const plans = copyService.copySharedTables.mock.calls.at(-1)?.[0] as Array<{ + sourceSql: string; + }>; + expect(JSON.stringify(plans)).not.toContain('spcalready'); + expect(JSON.stringify(plans)).not.toContain('tblalready'); + expect(JSON.stringify(plans)).not.toContain('bsealready'); + expect(JSON.stringify(plans)).not.toContain('tbldeletedalready'); + }); + + it('copies shared rows through postgres_fdw when explicitly selected', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + targetInternalSchema: internalSchema, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { schemaName: 'bsexxx', relations: [], totalBytes: 1024, estimatedRows: 10 }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + const service = createService(); + + await expect( + service.copySharedRowsForJob('sdmjxxx', { timeoutMs: 1000, strategy: 'postgres_fdw' }) + ).resolves.toMatchObject({ + phase: 'shared_rows_completed', + sharedTables: { + strategy: 'postgres_fdw', + copiedTableCount: 1, + totalCopiedRows: 5, + copiedTables: [{ strategy: 'postgres_fdw', table: 'record_history', copiedRows: 5 }], + }, + }); + + expect(copyService.copySharedTables).not.toHaveBeenCalled(); + expect(copyService.copySharedTablesViaPostgresFdw).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + table: 'record_history', + sql: expect.stringContaining('CREATE EXTENSION IF NOT EXISTS postgres_fdw'), + target: expect.objectContaining({ command: 'psql' }), + }), + expect.objectContaining({ + table: 'computed_update_outbox', + sql: expect.stringContaining(`"base_id" = ANY(ARRAY['bsexxx']::text[])`), + }), + ]), + expect.objectContaining({ + timeoutMs: 1000, + shouldCancel: expect.any(Function), + }), + expect.objectContaining({ + onTableCopied: expect.any(Function), + }) + ); + }); + + it('marks the migration failed when shared row copy fails', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + sharedTableIds: ['tblxxx', 'tbldeleted'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + copyService.copySharedTables.mockImplementation( + async ( + _plans: unknown[], + _options: unknown, + hooks: { + onTableCopied?: (result: unknown, index: number, total: number) => void | Promise; + } + ) => { + await hooks.onTableCopied?.( + { + table: 'record_history', + copiedRows: 5, + source: processResult('psql'), + target: processResult('psql', 'COPY 5\n'), + }, + 0, + 8 + ); + throw new SpaceDataDbProcessPipelineError('psql copy failed', { + label: 'shared-table:record_trash', + source: { + ...processResult('psql'), + exitCode: 1, + stderr: 'source failed', + }, + target: { + ...processResult('psql'), + exitCode: null, + signal: 'SIGTERM', + }, + }); + } + ); + const service = createService(); + + await expect(service.copySharedRowsForJob('sdmjxxx', {})).rejects.toThrow('psql copy failed'); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'psql copy failed', + copyStats: expect.objectContaining({ + phase: 'shared_rows_failed', + sharedTables: expect.objectContaining({ + copiedTableCount: 1, + totalCopiedRows: 5, + copiedTables: [ + expect.objectContaining({ + table: 'record_history', + copiedRows: 5, + }), + ], + error: 'psql copy failed', + failure: expect.objectContaining({ + type: 'pipeline', + result: expect.objectContaining({ + label: 'shared-table:record_trash', + source: expect.objectContaining({ + command: 'psql', + exitCode: 1, + stderr: 'source failed', + }), + target: expect.objectContaining({ + command: 'psql', + signal: 'SIGTERM', + }), + }), + }), + }), + }), + }), + }) + ); + }); + + it('cleans target schemas and scoped shared rows idempotently before switch', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'failed', + targetInternalSchema: internalSchema, + copyStats: { phase: 'shared_rows_failed' }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + sharedTableIds: ['tblxxx', 'tbldeleted'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM information_schema.schemata')) { + return { rows: [{ schemaName: 'bsexxx' }] }; + } + if (sql.includes('to_regclass')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('WITH deleted')) { + return { rows: [{ count: '3' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect( + service.cleanupTargetArtifactsForJob('sdmjxxx', 'copy_failed') + ).resolves.toMatchObject({ + reason: 'copy_failed', + baseSchemas: [{ schemaName: 'bsexxx', dropped: true }], + sharedTables: expect.arrayContaining([ + expect.objectContaining({ table: 'record_history', deletedRows: 3 }), + ]), + }); + + expect(targetClient.raw).toHaveBeenCalledWith('DROP SCHEMA IF EXISTS "bsexxx" CASCADE'); + expect( + targetClient.raw.mock.calls.some( + ([sql, bindings]) => + typeof sql === 'string' && + sql.includes('WITH deleted') && + JSON.stringify(bindings).includes('tbldeleted') + ) + ).toBe(true); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'shared_rows_failed', + targetCleanup: expect.objectContaining({ + reason: 'copy_failed', + completedAt: expect.any(String), + }), + }), + }), + }) + ); + }); + + it('truncates target shared tables when the target connection is unbound', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'failed', + targetInternalSchema: internalSchema, + copyStats: { phase: 'shared_rows_failed' }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + sharedTableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM information_schema.schemata')) { + return { rows: [] }; + } + if (sql.includes('to_regclass')) { + return { rows: [{ exists: true }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect( + service.cleanupTargetArtifactsForJob('sdmjxxx', 'retry_before_start', { + truncateSharedTables: true, + }) + ).resolves.toMatchObject({ + reason: 'retry_before_start', + truncateSharedTables: true, + sharedTables: expect.arrayContaining([ + expect.objectContaining({ table: 'record_history', deletedRows: null, truncated: true }), + ]), + }); + + const truncateCalls = targetClient.raw.mock.calls.filter( + ([sql]) => typeof sql === 'string' && sql.startsWith('TRUNCATE TABLE ') + ); + expect(truncateCalls).toHaveLength(1); + expect(truncateCalls[0][0]).toContain(`"${internalSchema}"."record_history"`); + expect(truncateCalls[0][0]).toContain(`"${internalSchema}"."computed_update_outbox_seed"`); + expect(truncateCalls[0][0]).toContain(`"${internalSchema}"."computed_update_outbox"`); + expect( + targetClient.raw.mock.calls.some( + ([sql]) => typeof sql === 'string' && sql.includes('WITH deleted') + ) + ).toBe(false); + }); + + it('returns a clear conflict when retry target cleanup fails', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'failed', + targetInternalSchema: internalSchema, + copyStats: { phase: 'shared_rows_failed' }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + sharedTableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('to_regclass')) { + return { rows: [{ exists: true }] }; + } + if (sql.startsWith('TRUNCATE TABLE ')) { + throw new Error('cannot truncate a table referenced in a foreign key constraint'); + } + return { rows: [] }; + }); + const service = createService(); + + await expect( + service.cleanupTargetArtifactsForJob('sdmjxxx', 'retry_before_start', { + truncateSharedTables: true, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TARGET_CLEANUP_FAILED', + targetError: expect.stringContaining('foreign key constraint'), + }), + }); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + targetCleanup: expect.objectContaining({ + failedAt: expect.any(String), + error: expect.stringContaining('foreign key constraint'), + }), + }), + }), + }) + ); + }); + + it('refuses automatic target cleanup after a successful switch', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'succeeded', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + }, + }); + const service = createService(); + + await expect(service.cleanupTargetArtifactsForJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + }); + + expect(targetClient.raw).not.toHaveBeenCalled(); + }); + + it('fails validation on base table row-count mismatch and does not switch routing', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 2); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: [ + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + sourceCount: 3, + targetCount: 2, + }), + ], + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.update).not.toHaveBeenCalled(); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'Space data database migration validation failed', + validationStats: expect.objectContaining({ phase: 'validation_failed' }), + }), + }) + ); + expect(sourceClient.destroy).toHaveBeenCalled(); + expect(targetClient.destroy).toHaveBeenCalled(); + }); + + it('fails validation on base table column signature mismatch and does not switch routing', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3, { columns: columnSignatureRows() }); + mockValidationClient(targetClient, 3, { + columns: columnSignatureRows([{ columnName: '__id', formattedType: 'text', notNull: true }]), + }); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: [ + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + reason: 'column_signature_mismatch', + sourceColumns: expect.arrayContaining([ + expect.objectContaining({ columnName: 'fldName', formattedType: 'text' }), + ]), + targetColumns: [expect.objectContaining({ columnName: '__id', formattedType: 'text' })], + }), + ], + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.update).not.toHaveBeenCalled(); + }); + + it('accepts restored table columns when dropped source columns leave ordinal gaps', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: 'dcnsource', + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3, { + columns: columnSignatureRows([ + { columnName: '__id', formattedType: 'text', notNull: true }, + { ordinalPosition: 3, columnName: 'fldName', formattedType: 'text' }, + ]), + }); + mockValidationClient(targetClient, 3, { + columns: columnSignatureRows([ + { columnName: '__id', formattedType: 'text', notNull: true }, + { columnName: 'fldName', formattedType: 'text' }, + ]), + }); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + validationStats: expect.objectContaining({ + phase: 'validation_completed', + baseSchemas: [ + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + sourceCount: 3, + targetCount: 3, + }), + ], + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + where: { spaceId: 'spcxxx' }, + }) + ); + }); + + it('allows existing related target rows during repair validation', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcrelated', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + relatedSpaces: { + primarySpaceId: 'spcrelated', + hasCrossSpaceLinks: true, + spaces: [ + { + spaceId: 'spcalready', + name: 'Already', + isPrimary: false, + baseIds: ['bsealready'], + tableIds: ['tblalready'], + dataDbMode: 'byodb', + }, + { + spaceId: 'spcrelated', + name: 'Related', + isPrimary: true, + baseIds: ['bserelated'], + tableIds: ['tblrelated'], + dataDbMode: 'default', + }, + ], + links: [], + }, + spaceIds: ['spcalready', 'spcrelated'], + copySpaceIds: ['spcrelated'], + baseIds: ['bserelated'], + tableIds: ['tblrelated'], + dbTableNames: ['bserelated.sheet1'], + physicalSchemas: [{ schemaName: 'bserelated', relations: [], totalBytes: 1024 }], + }, + }); + const sourceCountSqls: string[] = []; + sourceClient.raw.mockImplementation((sql: string) => { + if (sql.includes('COUNT(*)')) { + sourceCountSqls.push(sql); + return { rows: [{ count: sql.includes('tblrelated') ? '3' : '0' }] }; + } + return { rows: [] }; + }); + const targetCountCalls: Array<{ sql: string; bindings?: unknown[] }> = []; + targetClient.raw.mockImplementation((sql: string, bindings?: unknown[]) => { + if (sql.includes('to_regprocedure') || sql.includes('__teable_data_schema_migrations')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('COUNT(*)')) { + targetCountCalls.push({ sql, bindings }); + if (sql.includes('NOT (') && sql.includes('tblalready')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [{ count: sql.includes('tblrelated') ? '3' : '0' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + }); + + expect(sourceCountSqls.some((sql) => sql.includes('tblalready'))).toBe(false); + expect( + targetCountCalls.some( + (call) => call.sql.includes('NOT (') && JSON.stringify(call.bindings).includes('tblalready') + ) + ).toBe(true); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ where: { spaceId: 'spcalready' } }) + ); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith( + expect.objectContaining({ where: { spaceId: 'spcrelated' } }) + ); + }); + + it('scopes base table foreign key validation to schemas inside the migrated space', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: 'dcnsource', + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + outOfScopeForeignKeys: [ + { + schemaName: 'bsexxx', + tableName: 'sheet1', + constraintName: 'fk_out_of_scope', + referencedSchemaName: 'bseyyy', + referencedTableName: 'sheet2', + }, + ], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 3); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + }); + + expect(sourceClient.raw).toHaveBeenCalledWith( + expect.stringContaining('referenced_ns.nspname = ANY(?::text[])'), + ['bsexxx', 'sheet1', ['bsexxx']] + ); + expect(targetClient.raw).toHaveBeenCalledWith( + expect.stringContaining('referenced_ns.nspname = ANY(?::text[])'), + ['bsexxx', 'sheet1', ['bsexxx']] + ); + }); + + it('fails validation on base table index constraint or trigger signature mismatch', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3, { + indexes: indexSignatureRows([ + { + indexName: 'sheet1_name_idx', + definition: 'CREATE INDEX sheet1_name_idx ON bsexxx.sheet1 USING btree ("fldName")', + }, + ]), + constraints: constraintSignatureRows([ + { + constraintName: 'sheet1_name_check', + constraintType: 'c', + definition: 'CHECK (length("fldName") > 0)', + }, + ]), + triggers: triggerSignatureRows([ + { + triggerName: 'sheet1_update_trigger', + definition: 'CREATE TRIGGER sheet1_update_trigger BEFORE UPDATE ON bsexxx.sheet1', + }, + ]), + }); + mockValidationClient(targetClient, 3); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: expect.arrayContaining([ + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + reason: 'index_signature_mismatch', + sourceIndexes: expect.arrayContaining([ + expect.objectContaining({ indexName: 'sheet1_name_idx' }), + ]), + targetIndexes: [], + }), + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + reason: 'constraint_signature_mismatch', + sourceConstraints: expect.arrayContaining([ + expect.objectContaining({ constraintName: 'sheet1_name_check' }), + ]), + targetConstraints: [], + }), + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + reason: 'trigger_signature_mismatch', + sourceTriggers: expect.arrayContaining([ + expect.objectContaining({ triggerName: 'sheet1_update_trigger' }), + ]), + targetTriggers: [], + }), + ]), + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.update).not.toHaveBeenCalled(); + }); + + it('fails validation when the target data DB baseline schema version is not current', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 3); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + }, + ], + }; + } + if (sql.includes('FROM pg_attribute a')) { + return { rows: columnSignatureRows() }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: '3' }] }; + } + if (sql.includes('to_regprocedure')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('__teable_data_schema_migrations')) { + return { rows: [{ exists: false }] }; + } + if (sql.includes('COUNT(*)')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: [ + expect.objectContaining({ + object: `schema:${internalSchema}.__teable_data_schema_migrations`, + reason: 'target_schema_version_mismatch', + }), + ], + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(targetClient.raw).toHaveBeenCalledWith( + expect.stringContaining(`"${internalSchema}"."__teable_data_schema_migrations"`), + [schemaVersion] + ); + }); + + it('fails validation when dry-run routing does not resolve to the target BYODB connection', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 3); + dataDbClientManager.getDataDatabaseForSpace.mockImplementation((_, options) => { + if (options?.previewBinding) { + return Promise.resolve({ + cacheKey: 'meta-fallback', + isMetaFallback: true, + url: 'postgresql://source.example/teable', + }); + } + return Promise.resolve({ url: 'postgresql://source.example/teable' }); + }); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: [ + expect.objectContaining({ + object: 'route:spcxxx', + reason: 'target_route_smoke_failed', + }), + ], + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(dataDbClientManager.getDataDatabaseForSpace).toHaveBeenCalledWith('spcxxx', { + previewBinding: expect.objectContaining({ + spaceId: 'spcxxx', + connectionId: 'dcnxxx', + internalSchema, + }), + }); + }); + + it('fails validation when copied target shared tables contain out-of-scope rows', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 3); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + }, + ], + }; + } + if (sql.includes('FROM pg_attribute a')) { + return { rows: columnSignatureRows() }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: '3' }] }; + } + if (sql.includes('to_regprocedure')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('__teable_data_schema_migrations')) { + return { rows: [{ exists: true }] }; + } + if ( + sql.includes(`FROM "${internalSchema}"."record_history"`) && + sql.includes('NOT ("table_id" = ANY') + ) { + return { rows: [{ count: '1' }] }; + } + if (sql.includes('COUNT(*)')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_VALIDATION_MISMATCH', + mismatches: [ + expect.objectContaining({ + object: 'shared:record_history', + reason: 'out_of_scope_target_rows', + targetCount: 1, + }), + ], + }), + }); + + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + }); + + it('validates copied data without switching the space binding by default', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: 'dcnsource', + targetConnectionId: 'dcnxxx', + switchOnCompletion: false, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 3); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + validationStats: expect.objectContaining({ + phase: 'validation_completed', + switchOnCompletion: false, + switched: false, + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).not.toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ state: 'switching' }), + }) + ); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(sourceClient.raw).toHaveBeenCalledWith( + expect.stringContaining(`DELETE FROM "public"."computed_update_pause_scope"`), + ['space', ['spcxxx'], 'space-data-db-migration:sdmjxxx'] + ); + expect(txClient.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + status: 'ready', + lastError: null, + }), + }); + expect(txClient.spaceDataDbMigrationJob.update).toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'succeeded', + validationStats: expect.objectContaining({ + switchOnCompletion: false, + switched: false, + }), + lastError: null, + }), + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + expect(dataDbClientManager.invalidateConnection).not.toHaveBeenCalledWith('dcnsource'); + }); + + it('validates copied data and switches the space binding to the target BYODB connection', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: 'dcnsource', + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + mockValidationClient(targetClient, 3); + const service = createService(); + + await expect(service.validateAndSwitchJob('sdmjxxx')).resolves.toMatchObject({ + state: 'succeeded', + validationStats: expect.objectContaining({ + phase: 'validation_completed', + switchOnCompletion: true, + switched: true, + switchedAt: expect.any(String), + routeSmoke: expect.objectContaining({ + ok: true, + connectionId: 'dcnxxx', + internalSchema, + }), + baseSchemas: [ + expect.objectContaining({ + object: 'base:bsexxx.sheet1', + sourceCount: 3, + targetCount: 3, + }), + ], + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'validating', + validationStats: expect.objectContaining({ phase: 'validating_copy' }), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'validating', + validationStats: expect.objectContaining({ phase: 'validation_completed' }), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ state: 'switching' }), + }) + ); + expect(txClient.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + status: 'ready', + lastError: null, + }), + }); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith({ + where: { spaceId: 'spcxxx' }, + create: { + spaceId: 'spcxxx', + dataDbConnectionId: 'dcnxxx', + mode: 'byodb', + state: 'ready', + createdBy: 'usrxxx', + }, + update: { + dataDbConnectionId: 'dcnxxx', + mode: 'byodb', + state: 'ready', + }, + }); + expect(txClient.spaceDataDbMigrationJob.update).toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'succeeded', + lastError: null, + }), + }); + expect(targetClient.raw).toHaveBeenCalledWith( + expect.stringContaining(`DELETE FROM "${internalSchema}"."computed_update_pause_scope"`), + ['space', ['spcxxx'], 'space-data-db-migration:sdmjxxx'] + ); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnsource'); + }); + + it('keeps validation fresh while row counts are running', async () => { + vi.useFakeTimers(); + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: 'dcnsource', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + }); + mockValidationClient(sourceClient, 3); + let resolveTargetCount: (value: { rows: { count: string }[] }) => void = vi.fn(); + const targetCountRows = new Promise<{ rows: { count: string }[] }>((resolve) => { + resolveTargetCount = resolve; + }); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + }, + ], + }; + } + if (sql.includes('FROM pg_attribute a')) { + return { rows: columnSignatureRows() }; + } + if (sql.includes('FROM pg_index i')) { + return { rows: [] }; + } + if (sql.includes('FROM pg_constraint con')) { + return { rows: [] }; + } + if (sql.includes('FROM pg_trigger tg')) { + return { rows: [] }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return targetCountRows; + } + if (sql.includes('to_regprocedure')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('__teable_data_schema_migrations')) { + return { rows: [{ exists: true }] }; + } + if (sql.includes('COUNT(*)')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [] }; + }); + const service = createService(); + + const promise = service.validateAndSwitchJob('sdmjxxx'); + await vi.advanceTimersByTimeAsync(15_000); + + const heartbeatCount = prismaService.spaceDataDbMigrationJob.update.mock.calls.filter( + ([args]) => args.data.validationStats?.heartbeat?.stage === 'validating_copy' + ).length; + expect(heartbeatCount).toBeGreaterThanOrEqual(2); + + resolveTargetCount({ rows: [{ count: '3' }] }); + await expect(promise).resolves.toMatchObject({ + state: 'succeeded', + }); + vi.useRealTimers(); + }); + + it('waits for active source computed tasks to finish or become reclaimable', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + sourceClient.raw + .mockResolvedValueOnce({ + rows: [ + { + activeCount: '1', + reclaimableCount: '0', + oldestActiveLockedAt: '2026-05-06T00:00:00.000Z', + }, + ], + }) + .mockResolvedValueOnce({ + rows: [ + { + activeCount: '0', + reclaimableCount: '1', + oldestActiveLockedAt: null, + }, + ], + }); + const service = createService(); + + await expect( + service.waitForSourceComputedDrainForJob('sdmjxxx', { + timeoutMs: 50, + pollMs: 1, + processingLeaseMs: 120_000, + }) + ).resolves.toMatchObject({ + activeCount: 0, + reclaimableCount: 1, + }); + + expect(sourceClient.raw).toHaveBeenCalledTimes(2); + expect(sourceClient.raw).toHaveBeenCalledWith( + expect.stringContaining(`FROM "public"."computed_update_outbox"`), + [expect.any(Date), expect.any(Date), expect.any(Date), ['bsexxx']] + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'computed_drained', + computedDrain: expect.objectContaining({ + activeCount: 0, + reclaimableCount: 1, + }), + }), + }), + }) + ); + expect(sourceClient.destroy).toHaveBeenCalled(); + }); + + it('fails the migration when source computed tasks stay active past the drain timeout', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + sourceClient.raw.mockResolvedValue({ + rows: [ + { + activeCount: '2', + reclaimableCount: '0', + oldestActiveLockedAt: '2026-05-06T00:00:00.000Z', + }, + ], + }); + const service = createService(); + + await expect( + service.waitForSourceComputedDrainForJob('sdmjxxx', { + timeoutMs: 0, + pollMs: 1, + processingLeaseMs: 120_000, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_COMPUTED_DRAIN_TIMEOUT', + activeCount: 2, + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: expect.stringContaining('Timed out waiting for source computed tasks'), + copyStats: expect.objectContaining({ + phase: 'computed_drain_timeout', + }), + }), + }) + ); + expect(sourceClient.destroy).toHaveBeenCalled(); + }); + + it('marks the migration failed when source computed drain inspection fails', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + sourceClient.raw.mockRejectedValue(new Error('source query failed')); + const service = createService(); + + await expect( + service.waitForSourceComputedDrainForJob('sdmjxxx', { + timeoutMs: 50, + pollMs: 1, + processingLeaseMs: 120_000, + }) + ).rejects.toThrow('source query failed'); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'source query failed', + copyStats: expect.objectContaining({ + phase: 'computed_drain_failed', + }), + }), + }) + ); + expect(sourceClient.destroy).toHaveBeenCalled(); + }); + + it('waits for open schema operations for the space to finish before copy', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + prismaService.schemaOperation.count.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + prismaService.schemaOperation.findMany.mockResolvedValueOnce([ + { + id: 'sgoxxx', + status: 'running', + phase: 'alter_table', + baseId: 'bsexxx', + tableId: 'tblxxx', + lockedAt: new Date('2026-05-06T00:00:00.000Z'), + lockedBy: 'worker-1', + lastModifiedTime: new Date('2026-05-06T00:00:00.000Z'), + }, + ]); + const service = createService(); + + await expect( + service.waitForSchemaOperationsForJob('sdmjxxx', { + timeoutMs: 50, + pollMs: 1, + }) + ).resolves.toMatchObject({ + openCount: 0, + }); + + expect(prismaService.schemaOperation.count).toHaveBeenCalledWith({ + where: expect.objectContaining({ + status: { in: ['pending', 'running', 'error'] }, + }), + }); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'schema_operations_drained', + schemaOperations: expect.objectContaining({ openCount: 0 }), + }), + }), + }) + ); + }); + + it('fails the migration when schema operations remain open past the drain timeout', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + prismaService.schemaOperation.count.mockResolvedValue(2); + prismaService.schemaOperation.findMany.mockResolvedValue([ + { + id: 'sgoxxx', + status: 'running', + phase: 'alter_table', + baseId: 'bsexxx', + tableId: 'tblxxx', + lockedAt: new Date('2026-05-06T00:00:00.000Z'), + lockedBy: 'worker-1', + lastModifiedTime: new Date('2026-05-06T00:00:00.000Z'), + }, + ]); + const service = createService(); + + await expect( + service.waitForSchemaOperationsForJob('sdmjxxx', { + timeoutMs: 0, + pollMs: 1, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_SCHEMA_OPERATION_DRAIN_TIMEOUT', + openCount: 2, + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: expect.stringContaining('Timed out waiting for schema operations'), + copyStats: expect.objectContaining({ + phase: 'schema_operation_drain_timeout', + }), + }), + }) + ); + }); + + it('waits for provisioning resources and import queue jobs before copy', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + prismaService.base.count.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + prismaService.base.findMany + .mockResolvedValueOnce([ + { + id: 'bsexxx', + provisionState: 'pending', + lastModifiedTime: new Date('2026-05-06T00:00:00.000Z'), + }, + ]) + .mockResolvedValueOnce([]); + const importJob = { + id: 'import-table-csv-chunk:tblxxx:abc123', + data: { + baseId: 'bsexxx', + table: { id: 'tblxxx', name: 'Sheet 1' }, + }, + timestamp: new Date('2026-05-06T00:00:00.000Z').getTime(), + getState: vi.fn().mockResolvedValue('active'), + }; + const tableImportCsvChunkQueue = { + getJobs: vi.fn().mockResolvedValueOnce([importJob]).mockResolvedValueOnce([]), + }; + const service = createService(undefined, { tableImportCsvChunkQueue }); + + await expect( + service.waitForBackgroundWritersForJob('sdmjxxx', { + timeoutMs: 50, + pollMs: 1, + }) + ).resolves.toMatchObject({ + openCount: 0, + }); + + expect(prismaService.base.count).toHaveBeenCalledWith({ + where: expect.objectContaining({ + provisionState: { in: ['pending', 'deleting', 'error'] }, + }), + }); + expect(tableImportCsvChunkQueue.getJobs).toHaveBeenCalledWith( + ['waiting', 'active', 'delayed', 'prioritized', 'waiting-children'], + 0, + 99, + false + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'background_writers_drained', + backgroundWriters: expect.objectContaining({ openCount: 0 }), + }), + }), + }) + ); + }); + + it('fails the migration when background writer queue inspection hangs', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + const baseImportCsvQueue = { + getJobs: vi.fn().mockImplementation(() => new Promise(() => undefined)), + }; + const service = createService(undefined, { baseImportCsvQueue }); + + await expect( + service.waitForBackgroundWritersForJob('sdmjxxx', { + timeoutMs: 1000, + pollMs: 1, + probeTimeoutMs: 1, + }) + ).rejects.toThrow('Timed out inspecting import queues after 1ms'); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'Timed out inspecting import queues after 1ms', + copyStats: expect.objectContaining({ + phase: 'background_writer_drain_failed', + }), + }), + }) + ); + }); + + it('scans open queue jobs in bounded batches while draining background writers', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + const unrelatedJob = { + id: 'import-table-csv-chunk:tblother:abc123', + data: { + baseId: 'bseother', + table: { id: 'tblother', name: 'Other' }, + }, + getState: vi.fn().mockResolvedValue('waiting'), + }; + const scopedJob = { + id: 'import-table-csv-chunk:tblxxx:def456', + data: { + baseId: 'bsexxx', + table: { id: 'tblxxx', name: 'Sheet 1' }, + }, + getState: vi.fn().mockResolvedValue('active'), + }; + const tableImportCsvChunkQueue = { + getJobs: vi.fn().mockResolvedValueOnce([unrelatedJob]).mockResolvedValueOnce([scopedJob]), + }; + const service = createService(undefined, { tableImportCsvChunkQueue }); + + await expect( + service.waitForBackgroundWritersForJob('sdmjxxx', { + timeoutMs: 0, + pollMs: 1, + queueScanBatchSize: 1, + queueScanLimit: 2, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_BACKGROUND_WRITER_DRAIN_TIMEOUT', + openCount: 1, + queueJobCount: 1, + }), + }); + + expect(tableImportCsvChunkQueue.getJobs).toHaveBeenNthCalledWith( + 1, + ['waiting', 'active', 'delayed', 'prioritized', 'waiting-children'], + 0, + 0, + false + ); + expect(tableImportCsvChunkQueue.getJobs).toHaveBeenNthCalledWith( + 2, + ['waiting', 'active', 'delayed', 'prioritized', 'waiting-children'], + 1, + 1, + false + ); + }); + + it('fails the migration when background writers remain open past the drain timeout', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + }); + const importJob = { + id: 'base_import_csv_job', + data: { + baseId: 'bsexxx', + tableIdMap: { + oldTableId: 'tblxxx', + }, + }, + getState: vi.fn().mockResolvedValue('waiting'), + }; + const baseImportCsvQueue = { + getJobs: vi.fn().mockResolvedValue([importJob]), + }; + const service = createService(undefined, { baseImportCsvQueue }); + + await expect( + service.waitForBackgroundWritersForJob('sdmjxxx', { + timeoutMs: 0, + pollMs: 1, + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_BACKGROUND_WRITER_DRAIN_TIMEOUT', + openCount: 1, + queueJobCount: 1, + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: expect.stringContaining('Timed out waiting for background writers'), + copyStats: expect.objectContaining({ + phase: 'background_writer_drain_timeout', + }), + }), + }) + ); + }); + + it('fails before copy when the source inventory changed after freeze', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { + internalSchema, + }, + baseIds: ['bsexxx'], + tableIds: ['tblold'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + }, + }); + prismaService.space.findMany.mockResolvedValue([ + { + id: 'spcxxx', + name: 'Space', + baseGroup: [ + { + id: 'bsexxx', + tables: [{ id: 'tblnew' }], + }, + ], + }, + ]); + prismaService.tableMeta.findMany.mockResolvedValue([ + { id: 'tblnew', dbTableName: 'bsexxx.sheet2' }, + ]); + const service = createService(); + + await expect(service.assertSourceInventoryUnchangedForJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_INVENTORY_CHANGED', + mismatches: expect.arrayContaining([ + expect.objectContaining({ + object: 'tableIds', + reason: 'inventory_changed', + added: ['tblnew'], + removed: ['tblold'], + }), + expect.objectContaining({ + object: 'dbTableNames', + added: ['bsexxx.sheet2'], + removed: ['bsexxx.sheet1'], + }), + ]), + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: expect.stringContaining('Source inventory changed'), + copyStats: expect.objectContaining({ + phase: 'source_inventory_changed', + }), + }), + }) + ); + expect(copyService.copyBaseSchemas).not.toHaveBeenCalled(); + }); + + it('fails before copy when deleted-table shared scope changed after freeze', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { + internalSchema, + }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + sharedTableIds: ['tblxxx'], + relatedSharedTableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 10, + }, + ], + totalBytes: 1024, + estimatedRows: 10, + }, + ], + }, + }); + prismaService.tableMeta.findMany.mockImplementation((args?: unknown) => { + const where = (args as { where?: { baseId?: { in?: string[] }; id?: { in?: string[] } } }) + ?.where; + if (where?.baseId?.in) { + return Promise.resolve([{ id: 'tblxxx' }, { id: 'tbldeleted' }]); + } + return Promise.resolve([{ id: 'tblxxx', dbTableName: 'bsexxx.sheet1' }]); + }); + const service = createService(); + + await expect(service.assertSourceInventoryUnchangedForJob('sdmjxxx')).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_INVENTORY_CHANGED', + mismatches: expect.arrayContaining([ + expect.objectContaining({ + object: 'sharedTableIds', + added: ['tbldeleted'], + }), + ]), + }), + }); + }); + + it('does not treat equivalent extension dependency objects as changed when JSON key order differs', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + sourceDataDb: { + isMetaFallback: true, + internalSchema: null, + connectionId: null, + cacheKey: 'meta-fallback', + mode: 'default', + }, + targetDataDb: { + internalSchema, + }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + estimatedRows: 10, + totalBytes: 1024, + relations: [ + { + estimatedRows: 10, + totalBytes: 1024, + relationKind: 'table', + relationName: 'sheet1', + schemaName: 'bsexxx', + }, + ], + schemaName: 'bsexxx', + }, + ], + postgresExtensionDependencies: [ + { + sourceObjects: ['bsexxx.sheet1.idx_trgm_sheet1_fldName'], + accessMethod: 'gin', + objectName: 'gin_trgm_ops', + schemaName: 'public', + objectType: 'operator_class', + extensionName: 'pg_trgm', + }, + ], + estimatedTotalBytes: 1024, + estimatedTotalRows: 10, + }, + }); + sourceDataPrisma.$queryRawUnsafe + .mockResolvedValueOnce([ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: '1024', + estimatedRows: '10', + }, + ]) + .mockResolvedValueOnce([ + { + extensionName: 'pg_trgm', + objectType: 'operator_class', + schemaName: 'public', + objectName: 'gin_trgm_ops', + accessMethod: 'gin', + sourceSchemaName: 'bsexxx', + sourceRelationName: 'sheet1', + sourceIndexName: 'idx_trgm_sheet1_fldName', + }, + ]); + const service = createService(); + + await expect(service.assertSourceInventoryUnchangedForJob('sdmjxxx')).resolves.toBeUndefined(); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + lastError: null, + copyStats: expect.objectContaining({ + phase: 'source_inventory_verified', + }), + }), + }) + ); + }); + + it('fails before copy when the temp work directory has insufficient free space', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + relations: [], + totalBytes: 2048, + estimatedRows: 10, + }, + ], + estimatedTotalBytes: 2048, + estimatedTotalRows: 10, + }, + }); + const statfs = vi.fn().mockResolvedValue({ + bavail: 3, + bfree: 3, + bsize: 1024, + }); + const service = createService(statfs as never); + + await expect( + service.assertTempWorkDirCapacityForJob('sdmjxxx', '/tmp/sdmjxxx', { + multiplier: 2, + minFreeBytes: 0, + baseSchemaCopyStrategy: 'pg_dump_restore', + }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_TEMP_DISK_INSUFFICIENT', + requiredBytes: 4096, + availableBytes: 3072, + }), + }); + + expect(statfs).toHaveBeenCalledWith('/tmp/sdmjxxx'); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: expect.stringContaining('Insufficient temp disk space'), + copyStats: expect.objectContaining({ + phase: 'temp_disk_insufficient', + }), + }), + }) + ); + expect(copyService.copyBaseSchemas).not.toHaveBeenCalled(); + }); + + it('does not require full dump space for streaming base-schema copy', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + estimatedTotalBytes: 2048, + estimatedTotalRows: 10, + }, + }); + const statfs = vi.fn().mockResolvedValue({ + bavail: 3, + bfree: 3, + bsize: 1024, + }); + const service = createService(statfs as never); + + await expect( + service.assertTempWorkDirCapacityForJob('sdmjxxx', '/tmp/sdmjxxx', { + multiplier: 2, + minFreeBytes: 1024, + baseSchemaCopyStrategy: 'pg_dump_stream_restore', + }) + ).resolves.toBeUndefined(); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenLastCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'temp_disk_checked', + tempDisk: expect.objectContaining({ + strategy: 'pg_dump_stream_restore', + requiresFullDumpSpace: false, + requiredBytes: 1024, + availableBytes: 3072, + }), + }), + }), + }) + ); + }); + + it('runs a migration job through copy, validation, and switch phases in order', async () => { + const service = createService(); + const pauseSource = vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ + created: true, + } as never); + const waitForSourceComputedDrain = vi + .spyOn(service, 'waitForSourceComputedDrainForJob') + .mockResolvedValue({ + activeCount: 0, + reclaimableCount: 0, + } as never); + const waitForSchemaOperations = vi + .spyOn(service, 'waitForSchemaOperationsForJob') + .mockResolvedValue({ + openCount: 0, + } as never); + const waitForBackgroundWriters = vi + .spyOn(service, 'waitForBackgroundWritersForJob') + .mockResolvedValue({ + openCount: 0, + } as never); + const assertSourceInventoryUnchanged = vi + .spyOn(service, 'assertSourceInventoryUnchangedForJob') + .mockResolvedValue(undefined); + const assertTempWorkDirCapacity = vi + .spyOn(service, 'assertTempWorkDirCapacityForJob') + .mockResolvedValue(undefined); + const resumeSource = vi.spyOn(service, 'resumeSourceComputedForJob').mockResolvedValue({ + deleted: 0, + } as never); + const copyBaseSchemas = vi.spyOn(service, 'copyBaseSchemasForJob').mockResolvedValue({ + phase: 'base_schemas_completed', + } as never); + const copySharedRows = vi.spyOn(service, 'copySharedRowsForJob').mockResolvedValue({ + phase: 'shared_rows_completed', + } as never); + const validateAndSwitch = vi.spyOn(service, 'validateAndSwitchJob').mockResolvedValue({ + state: 'succeeded', + } as never); + + await expect( + service.runMigrationJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 2, + timeoutMs: 1000, + }) + ).resolves.toEqual({ state: 'succeeded' }); + + expect(copyService.assertPostgresToolsAvailable).toHaveBeenCalledWith( + 'pg_dump_stream_restore', + expect.objectContaining({ + timeoutMs: 1000, + shouldCancel: expect.any(Function), + }) + ); + expect(pauseSource).toHaveBeenCalledWith('sdmjxxx'); + expect(waitForSourceComputedDrain).toHaveBeenCalledWith('sdmjxxx', { + timeoutMs: 600_000, + pollMs: 5_000, + processingLeaseMs: 120_000, + }); + expect(waitForSchemaOperations).toHaveBeenCalledWith('sdmjxxx', { + timeoutMs: 600_000, + pollMs: 5_000, + }); + expect(waitForBackgroundWriters).toHaveBeenCalledWith('sdmjxxx', { + timeoutMs: 600_000, + pollMs: 5_000, + probeTimeoutMs: 30_000, + queueScanBatchSize: 100, + queueScanLimit: 1000, + }); + expect(assertSourceInventoryUnchanged).toHaveBeenCalledWith('sdmjxxx'); + expect(assertTempWorkDirCapacity).toHaveBeenCalledWith('sdmjxxx', '/tmp/sdmjxxx', { + multiplier: 2, + minFreeBytes: 536_870_912, + baseSchemaCopyStrategy: 'pg_dump_stream_restore', + }); + expect(resumeSource).not.toHaveBeenCalled(); + expect(copyBaseSchemas).toHaveBeenCalledWith('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 2, + timeoutMs: 1000, + strategy: 'pg_dump_stream_restore', + }); + expect(copySharedRows).toHaveBeenCalledWith('sdmjxxx', { + timeoutMs: 1000, + strategy: 'psql_copy', + }); + expect(validateAndSwitch).toHaveBeenCalledWith('sdmjxxx'); + expect(copyService.assertPostgresToolsAvailable.mock.invocationCallOrder[0]).toBeLessThan( + pauseSource.mock.invocationCallOrder[0] + ); + expect(copyBaseSchemas.mock.invocationCallOrder[0]).toBeLessThan( + copySharedRows.mock.invocationCallOrder[0] + ); + expect(pauseSource.mock.invocationCallOrder[0]).toBeLessThan( + waitForSourceComputedDrain.mock.invocationCallOrder[0] + ); + expect(waitForSourceComputedDrain.mock.invocationCallOrder[0]).toBeLessThan( + waitForSchemaOperations.mock.invocationCallOrder[0] + ); + expect(waitForSchemaOperations.mock.invocationCallOrder[0]).toBeLessThan( + waitForBackgroundWriters.mock.invocationCallOrder[0] + ); + expect(waitForBackgroundWriters.mock.invocationCallOrder[0]).toBeLessThan( + assertSourceInventoryUnchanged.mock.invocationCallOrder[0] + ); + expect(assertSourceInventoryUnchanged.mock.invocationCallOrder[0]).toBeLessThan( + assertTempWorkDirCapacity.mock.invocationCallOrder[0] + ); + expect(assertTempWorkDirCapacity.mock.invocationCallOrder[0]).toBeLessThan( + copyBaseSchemas.mock.invocationCallOrder[0] + ); + expect(copySharedRows.mock.invocationCallOrder[0]).toBeLessThan( + validateAndSwitch.mock.invocationCallOrder[0] + ); + }); + + it('runs a test-only migration job without freezing source writes', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: false, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [{ schemaName: 'bsexxx', relations: [], totalBytes: 1024 }], + }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + const service = createService(); + const pauseSource = vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ + created: true, + } as never); + const waitForSourceComputedDrain = vi + .spyOn(service, 'waitForSourceComputedDrainForJob') + .mockResolvedValue({ + activeCount: 0, + reclaimableCount: 0, + } as never); + const waitForSchemaOperations = vi + .spyOn(service, 'waitForSchemaOperationsForJob') + .mockResolvedValue({ + openCount: 0, + } as never); + const waitForBackgroundWriters = vi + .spyOn(service, 'waitForBackgroundWritersForJob') + .mockResolvedValue({ + openCount: 0, + } as never); + const assertSourceInventoryUnchanged = vi + .spyOn(service, 'assertSourceInventoryUnchangedForJob') + .mockResolvedValue(undefined); + const assertTempWorkDirCapacity = vi + .spyOn(service, 'assertTempWorkDirCapacityForJob') + .mockResolvedValue(undefined); + const copyBaseSchemas = vi.spyOn(service, 'copyBaseSchemasForJob').mockResolvedValue({ + phase: 'base_schemas_completed', + } as never); + const copySharedRows = vi.spyOn(service, 'copySharedRowsForJob').mockResolvedValue({ + phase: 'shared_rows_completed', + } as never); + const validateAndSwitch = vi.spyOn(service, 'validateAndSwitchJob').mockResolvedValue({ + state: 'succeeded', + } as never); + + await expect(service.runMigrationJob('sdmjxxx', { workDir: '/tmp/sdmjxxx' })).resolves.toEqual({ + state: 'succeeded', + }); + + expect(pauseSource).not.toHaveBeenCalled(); + expect(waitForSourceComputedDrain).not.toHaveBeenCalled(); + expect(waitForSchemaOperations).not.toHaveBeenCalled(); + expect(waitForBackgroundWriters).not.toHaveBeenCalled(); + expect(assertSourceInventoryUnchanged).not.toHaveBeenCalled(); + expect(assertTempWorkDirCapacity).toHaveBeenCalledWith('sdmjxxx', '/tmp/sdmjxxx', { + multiplier: 2, + minFreeBytes: 536_870_912, + baseSchemaCopyStrategy: 'pg_dump_stream_restore', + }); + expect(copyBaseSchemas).toHaveBeenCalledWith('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 1, + timeoutMs: 86_400_000, + strategy: 'pg_dump_stream_restore', + }); + expect(copySharedRows).toHaveBeenCalledWith('sdmjxxx', { + timeoutMs: 86_400_000, + strategy: 'psql_copy', + }); + expect(validateAndSwitch).toHaveBeenCalledWith('sdmjxxx'); + }); + + it('caps copy jobs before passing them to pg_dump and pg_restore', async () => { + const service = createService(); + vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ created: false } as never); + vi.spyOn(service, 'waitForSourceComputedDrainForJob').mockResolvedValue({ + activeCount: 0, + reclaimableCount: 0, + } as never); + vi.spyOn(service, 'waitForSchemaOperationsForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'waitForBackgroundWritersForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'assertSourceInventoryUnchangedForJob').mockResolvedValue(undefined); + vi.spyOn(service, 'assertTempWorkDirCapacityForJob').mockResolvedValue(undefined); + const copyBaseSchemas = vi.spyOn(service, 'copyBaseSchemasForJob').mockResolvedValue({ + phase: 'base_schemas_completed', + } as never); + vi.spyOn(service, 'copySharedRowsForJob').mockResolvedValue({ + phase: 'shared_rows_completed', + } as never); + vi.spyOn(service, 'validateAndSwitchJob').mockResolvedValue({ + state: 'succeeded', + } as never); + + await expect( + service.runMigrationJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 20, + maxJobs: 3, + timeoutMs: 1000, + }) + ).resolves.toEqual({ state: 'succeeded' }); + + expect(copyBaseSchemas).toHaveBeenCalledWith('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 3, + timeoutMs: 1000, + strategy: 'pg_dump_stream_restore', + }); + }); + + it('checks pgcopydb and passes the selected base-schema strategy when explicitly requested', async () => { + copyService.assertPostgresToolsAvailable.mockResolvedValueOnce([ + processResult('pg_dump'), + processResult('pg_restore'), + processResult('psql'), + processResult('pgcopydb'), + ]); + const service = createService(); + vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ created: false } as never); + vi.spyOn(service, 'waitForSourceComputedDrainForJob').mockResolvedValue({ + activeCount: 0, + reclaimableCount: 0, + } as never); + vi.spyOn(service, 'waitForSchemaOperationsForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'waitForBackgroundWritersForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'assertSourceInventoryUnchangedForJob').mockResolvedValue(undefined); + vi.spyOn(service, 'assertTempWorkDirCapacityForJob').mockResolvedValue(undefined); + const copyBaseSchemas = vi.spyOn(service, 'copyBaseSchemasForJob').mockResolvedValue({ + phase: 'base_schemas_completed', + } as never); + const copySharedRows = vi.spyOn(service, 'copySharedRowsForJob').mockResolvedValue({ + phase: 'shared_rows_completed', + } as never); + vi.spyOn(service, 'validateAndSwitchJob').mockResolvedValue({ + state: 'succeeded', + } as never); + + await expect( + service.runMigrationJob('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 4, + baseSchemaCopyStrategy: 'pgcopydb', + sharedTableCopyStrategy: 'postgres_fdw', + timeoutMs: 1000, + }) + ).resolves.toEqual({ state: 'succeeded' }); + + expect(copyService.assertPostgresToolsAvailable).toHaveBeenCalledWith( + 'pgcopydb', + expect.objectContaining({ timeoutMs: 1000, shouldCancel: expect.any(Function) }) + ); + expect(copyBaseSchemas).toHaveBeenCalledWith('sdmjxxx', { + workDir: '/tmp/sdmjxxx', + jobs: 4, + timeoutMs: 1000, + strategy: 'pgcopydb', + }); + expect(copySharedRows).toHaveBeenCalledWith('sdmjxxx', { + timeoutMs: 1000, + strategy: 'postgres_fdw', + }); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'postgres_tools_checked', + postgresTools: expect.objectContaining({ + requiredTools: ['pg_dump', 'pg_restore', 'psql', 'pgcopydb'], + }), + }), + }), + }) + ); + }); + + it('fails fast before pausing source computed work when PostgreSQL client tools are unavailable', async () => { + prismaService.spaceDataDbMigrationJob.findFirst + .mockResolvedValueOnce({ + id: 'sdmjxxx', + state: 'freezing_writes', + }) + .mockResolvedValueOnce({ + id: 'sdmjxxx', + state: 'freezing_writes', + }) + .mockResolvedValueOnce({ + id: 'sdmjxxx', + state: 'failed', + }); + copyService.assertPostgresToolsAvailable.mockRejectedValueOnce( + new Error('spawn pg_dump ENOENT') + ); + const service = createService(); + const pauseSource = vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ + created: true, + } as never); + + await expect( + service.runMigrationJob('sdmjxxx', { workDir: '/tmp/sdmjxxx', timeoutMs: 60_000 }) + ).rejects.toMatchObject({ + code: HttpErrorCode.VALIDATION_ERROR, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_POSTGRES_TOOL_UNAVAILABLE', + requiredTools: ['pg_dump', 'pg_restore', 'psql'], + cause: 'spawn pg_dump ENOENT', + }), + }); + + expect(pauseSource).not.toHaveBeenCalled(); + expect(copyService.copyBaseSchemas).not.toHaveBeenCalled(); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + copyStats: expect.objectContaining({ + phase: 'postgres_tools_checking', + }), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'failed', + lastError: 'spawn pg_dump ENOENT', + copyStats: expect.objectContaining({ + phase: 'postgres_tools_unavailable', + postgresTools: expect.objectContaining({ + requiredTools: ['pg_dump', 'pg_restore', 'psql'], + error: 'spawn pg_dump ENOENT', + }), + }), + }), + }) + ); + expect(prismaService.spaceDataDbMigrationJob.updateMany).toHaveBeenCalledWith({ + where: { id: 'sdmjxxx', state: 'failed' }, + data: expect.objectContaining({ + state: 'failed', + completedAt: expect.any(Date), + lastError: 'Required PostgreSQL client tools are unavailable for space data DB migration', + }), + }); + }); + + it('cancels a pre-copy migration and resumes the migration-owned source pause', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'freezing_writes', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetMode: 'migrate-space', + state: 'canceled', + targetInternalSchema: internalSchema, + inventory: { baseIds: ['bsexxx'] }, + copyStats: { phase: 'canceled_before_copy' }, + validationStats: null, + lastError: 'Space data database migration canceled by usrxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: new Date('2026-05-06T00:01:00.000Z'), + createdTime: new Date('2026-05-06T00:00:00.000Z'), + lastModifiedTime: new Date('2026-05-06T00:01:00.000Z'), + targetConnection: null, + }); + sourceClient.raw.mockResolvedValue({ rows: [{ id: 'sdmp_sdmjxxx' }] }); + const service = createService(); + + await expect( + service.cancelMigrationForSpace('spcxxx', 'sdmjxxx', 'usrxxx') + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + state: 'canceled', + }); + + expect(sourceClient.raw).toHaveBeenCalledWith( + expect.stringContaining('DELETE FROM "public"."computed_update_pause_scope"'), + ['space', ['spcxxx'], 'space-data-db-migration:sdmjxxx'] + ); + expect(txClient.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + status: 'error', + lastError: expect.stringContaining('canceled by usrxxx'), + }), + }); + expect(txClient.spaceDataDbMigrationJob.update).toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'canceled', + lastError: expect.stringContaining('canceled by usrxxx'), + copyStats: expect.objectContaining({ + phase: 'canceled_before_copy', + canceled: expect.objectContaining({ canceledBy: 'usrxxx' }), + }), + }), + }); + }); + + it('rejects canceling a migration that has entered validation', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'validating', + targetConnectionId: 'dcnxxx', + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + inventory: { baseIds: ['bsexxx'] }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + const service = createService(); + + await expect( + service.cancelMigrationForSpace('spcxxx', 'sdmjxxx', 'usrxxx') + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_MIGRATION_CANCEL_CONFLICT', + migrationState: 'validating', + }), + }); + + expect(sourceClient.raw).not.toHaveBeenCalled(); + expect(txClient.spaceDataDbMigrationJob.update).not.toHaveBeenCalled(); + }); + + it('rolls back a completed migration after a clean post-switch proof', async () => { + const completedAt = new Date('2026-05-06T00:10:00.000Z'); + const job = { + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'succeeded', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt, + inventory: { + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { internalSchema }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + validationStats: { + phase: 'validation_completed', + baseSchemas: [], + sharedTables: [], + }, + copyStats: null, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }; + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue(job); + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + ...job, + targetMode: 'migrate-space', + state: 'rolled_back', + lastError: null, + createdTime: new Date('2026-05-06T00:00:00.000Z'), + lastModifiedTime: new Date('2026-05-06T00:11:00.000Z'), + targetConnection: null, + }); + const service = createService(); + vi.spyOn( + service as unknown as { + inspectPostSwitchRollbackProof: (job: unknown) => Promise; + }, + 'inspectPostSwitchRollbackProof' + ).mockResolvedValue({ + eligible: true, + switchedAt: completedAt.toISOString(), + checkedAt: '2026-05-06T00:11:00.000Z', + findings: [], + }); + const resumeSource = vi + .spyOn( + service as unknown as { + resumeOriginalSourceComputedPause: ( + job: unknown, + sourceDataDb: unknown + ) => Promise<{ deleted: number }>; + }, + 'resumeOriginalSourceComputedPause' + ) + .mockResolvedValue({ deleted: 1 }); + + await expect( + service.rollbackMigrationForSpace('spcxxx', 'sdmjxxx', 'usrrollback') + ).resolves.toMatchObject({ + jobId: 'sdmjxxx', + state: 'rolled_back', + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: { state: 'switching', lastError: null }, + }); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith({ + where: { spaceId: 'spcxxx' }, + create: expect.objectContaining({ + spaceId: 'spcxxx', + dataDbConnectionId: null, + mode: 'default', + state: 'ready', + createdBy: 'usrrollback', + }), + update: { + dataDbConnectionId: null, + mode: 'default', + state: 'ready', + }, + }); + expect(txClient.spaceDataDbMigrationJob.update).toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'rolled_back', + lastError: null, + validationStats: expect.objectContaining({ + rollback: expect.objectContaining({ + eligible: true, + rolledBackBy: 'usrrollback', + }), + }), + }), + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + expect(resumeSource).toHaveBeenCalledWith( + job, + expect.objectContaining({ cacheKey: 'meta-fallback', isMetaFallback: true }) + ); + }); + + it('rejects post-switch rollback when target writes are detected', async () => { + const completedAt = new Date('2026-05-06T00:10:00.000Z'); + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'succeeded', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt, + inventory: { + sourceDataDb: { + mode: 'default', + cacheKey: 'meta-fallback', + connectionId: null, + internalSchema: null, + isMetaFallback: true, + }, + targetDataDb: { internalSchema }, + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [], + }, + validationStats: { + phase: 'validation_completed', + baseSchemas: [], + sharedTables: [], + }, + copyStats: null, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + const service = createService(); + vi.spyOn( + service as unknown as { + inspectPostSwitchRollbackProof: (job: unknown) => Promise; + }, + 'inspectPostSwitchRollbackProof' + ).mockResolvedValue({ + eligible: false, + switchedAt: completedAt.toISOString(), + checkedAt: '2026-05-06T00:11:00.000Z', + findings: [{ object: 'base:bsexxx.sheet1', reason: 'row_count_changed' }], + }); + + await expect( + service.rollbackMigrationForSpace('spcxxx', 'sdmjxxx', 'usrrollback') + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_ROLLBACK_UNSAFE', + rollback: expect.objectContaining({ + eligible: false, + }), + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenNthCalledWith(1, { + where: { id: 'sdmjxxx' }, + data: { state: 'switching', lastError: null }, + }); + expect(prismaService.spaceDataDbMigrationJob.update).toHaveBeenNthCalledWith(2, { + where: { id: 'sdmjxxx' }, + data: expect.objectContaining({ + state: 'succeeded', + lastError: expect.stringContaining('rollback is unsafe'), + validationStats: expect.objectContaining({ + rollback: expect.objectContaining({ eligible: false }), + }), + }), + }); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(dataDbClientManager.invalidateConnection).not.toHaveBeenCalled(); + }); + + it('rejects rollback for a successful dry-run migration that did not switch routing', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'succeeded', + sourceConnectionId: null, + targetConnectionId: 'dcnxxx', + switchOnCompletion: false, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + completedAt: new Date('2026-05-06T00:10:00.000Z'), + inventory: { + baseIds: ['bsexxx'], + tableIds: ['tblxxx'], + }, + validationStats: { + phase: 'validation_completed', + switchOnCompletion: false, + switched: false, + }, + copyStats: null, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + const service = createService(); + + await expect( + service.rollbackMigrationForSpace('spcxxx', 'sdmjxxx', 'usrrollback') + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_ROLLBACK_UNSAFE', + }), + }); + + expect(prismaService.spaceDataDbMigrationJob.update).not.toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: { state: 'switching', lastError: null }, + }); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + }); + + it('rejects rollback from a BYODB source before entering the active switching state', async () => { + prismaService.spaceDataDbMigrationJob.findUnique.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'succeeded', + sourceConnectionId: 'dcnsource', + targetConnectionId: 'dcntarget', + switchOnCompletion: true, + targetInternalSchema: internalSchema, + createdBy: 'usrxxx', + completedAt: new Date('2026-05-06T00:10:00.000Z'), + inventory: { + sourceDataDb: { + mode: 'byodb', + cacheKey: 'dcnsource', + connectionId: 'dcnsource', + internalSchema, + isMetaFallback: false, + }, + targetDataDb: { internalSchema }, + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: [], + physicalSchemas: [], + }, + validationStats: null, + copyStats: null, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + const service = createService(); + + await expect( + service.rollbackMigrationForSpace('spcxxx', 'sdmjxxx', 'usrrollback') + ).rejects.toMatchObject({ + code: HttpErrorCode.VALIDATION_ERROR, + }); + + expect(prismaService.spaceDataDbMigrationJob.update).not.toHaveBeenCalledWith({ + where: { id: 'sdmjxxx' }, + data: { state: 'switching', lastError: null }, + }); + }); + + it('proves a post-switch rollback is clean when target inventory, counts, and timestamps match', async () => { + const service = createService(); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + }, + ], + }; + } + if (sql.includes('FROM information_schema.columns')) { + return { rows: [{ columnName: '__last_modified_time' }] }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: sql.includes('> ?::timestamp') ? '0' : '5' }] }; + } + if (sql.includes('COUNT(*)')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [] }; + }); + + await expect( + ( + service as unknown as { + inspectPostSwitchRollbackProof: (job: unknown) => Promise<{ + eligible: boolean; + findings: unknown[]; + }>; + } + ).inspectPostSwitchRollbackProof({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + completedAt: new Date('2026-05-06T00:10:00.000Z'), + targetInternalSchema: internalSchema, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + totalBytes: 1024, + estimatedRows: 5, + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 5, + }, + ], + }, + ], + }, + validationStats: { + baseSchemas: [{ object: 'base:bsexxx.sheet1', sourceCount: 5, targetCount: 5 }], + sharedTables: [ + { object: 'shared:computed_update_outbox', sourceCount: 0, targetCount: 0 }, + { object: 'shared:computed_update_dead_letter', sourceCount: 0, targetCount: 0 }, + { object: 'shared:computed_update_pause_scope', sourceCount: 0, targetCount: 0 }, + { object: 'shared:__undo_log', sourceCount: 0, targetCount: 0 }, + ], + }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }) + ).resolves.toMatchObject({ + eligible: true, + findings: [], + }); + }); + + it('flags post-switch rollback as unsafe when target row counts or timestamps changed', async () => { + const service = createService(); + targetClient.raw.mockImplementation((sql: string) => { + if (sql.includes('FROM pg_class c')) { + return { + rows: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + }, + ], + }; + } + if (sql.includes('FROM information_schema.columns')) { + return { rows: [{ columnName: '__last_modified_time' }] }; + } + if (sql.includes('FROM "bsexxx"."sheet1"')) { + return { rows: [{ count: sql.includes('> ?::timestamp') ? '2' : '6' }] }; + } + if (sql.includes('COUNT(*)')) { + return { rows: [{ count: '0' }] }; + } + return { rows: [] }; + }); + + const proof = await ( + service as unknown as { + inspectPostSwitchRollbackProof: (job: unknown) => Promise<{ + eligible: boolean; + findings: { reason: string }[]; + }>; + } + ).inspectPostSwitchRollbackProof({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + completedAt: new Date('2026-05-06T00:10:00.000Z'), + targetInternalSchema: internalSchema, + inventory: { + baseIds: ['bsexxx'], + tableIds: [], + dbTableNames: ['bsexxx.sheet1'], + physicalSchemas: [ + { + schemaName: 'bsexxx', + totalBytes: 1024, + estimatedRows: 5, + relations: [ + { + schemaName: 'bsexxx', + relationName: 'sheet1', + relationKind: 'table', + totalBytes: 1024, + estimatedRows: 5, + }, + ], + }, + ], + }, + validationStats: { + baseSchemas: [{ object: 'base:bsexxx.sheet1', sourceCount: 5, targetCount: 5 }], + sharedTables: [ + { object: 'shared:computed_update_outbox', sourceCount: 0, targetCount: 0 }, + { object: 'shared:computed_update_dead_letter', sourceCount: 0, targetCount: 0 }, + { object: 'shared:computed_update_pause_scope', sourceCount: 0, targetCount: 0 }, + { object: 'shared:__undo_log', sourceCount: 0, targetCount: 0 }, + ], + }, + targetConnection: { + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }); + + expect(proof.eligible).toBe(false); + expect(proof.findings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ reason: 'row_count_changed' }), + expect.objectContaining({ reason: 'post_switch_timestamp_rows' }), + ]) + ); + }); + + it('stops the runner before physical copy when cancellation is observed at a checkpoint', async () => { + prismaService.spaceDataDbMigrationJob.findFirst + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'canceled', + }); + const service = createService(); + vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ created: true } as never); + vi.spyOn(service, 'waitForSourceComputedDrainForJob').mockResolvedValue({ + activeCount: 0, + reclaimableCount: 0, + } as never); + vi.spyOn(service, 'waitForSchemaOperationsForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'waitForBackgroundWritersForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'assertSourceInventoryUnchangedForJob').mockResolvedValue(undefined); + vi.spyOn(service, 'assertTempWorkDirCapacityForJob').mockResolvedValue(undefined); + const copyBaseSchemas = vi.spyOn(service, 'copyBaseSchemasForJob').mockResolvedValue({ + phase: 'base_schemas_completed', + } as never); + const resumeSource = vi.spyOn(service, 'resumeSourceComputedForJob').mockResolvedValue({ + deleted: 1, + } as never); + + await expect( + service.runMigrationJob('sdmjxxx', { workDir: '/tmp/sdmjxxx' }) + ).rejects.toMatchObject({ + code: HttpErrorCode.CONFLICT, + data: expect.objectContaining({ + errorCode: 'SPACE_DATA_DB_MIGRATION_CANCELED', + }), + }); + + expect(copyBaseSchemas).not.toHaveBeenCalled(); + expect(resumeSource).toHaveBeenCalledWith('sdmjxxx'); + }); + + it('resumes a migration-created source computed pause when copy fails before switch', async () => { + const service = createService(); + vi.spyOn(service, 'pauseSourceComputedForJob').mockResolvedValue({ created: true } as never); + vi.spyOn(service, 'waitForSourceComputedDrainForJob').mockResolvedValue({ + activeCount: 0, + reclaimableCount: 0, + } as never); + vi.spyOn(service, 'waitForSchemaOperationsForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'waitForBackgroundWritersForJob').mockResolvedValue({ + openCount: 0, + } as never); + vi.spyOn(service, 'assertSourceInventoryUnchangedForJob').mockResolvedValue(undefined); + vi.spyOn(service, 'assertTempWorkDirCapacityForJob').mockResolvedValue(undefined); + vi.spyOn(service, 'copyBaseSchemasForJob').mockRejectedValue(new Error('copy failed')); + const copySharedRows = vi.spyOn(service, 'copySharedRowsForJob'); + const validateAndSwitch = vi.spyOn(service, 'validateAndSwitchJob'); + const resumeSource = vi.spyOn(service, 'resumeSourceComputedForJob').mockResolvedValue({ + deleted: 1, + } as never); + const cleanupTargetArtifacts = vi + .spyOn(service, 'cleanupTargetArtifactsForJob') + .mockResolvedValue({ + reason: 'pre_switch_failure', + baseSchemas: [], + sharedTables: [], + startedAt: '2026-05-06T00:00:00.000Z', + completedAt: '2026-05-06T00:00:01.000Z', + } as never); + + await expect(service.runMigrationJob('sdmjxxx', { workDir: '/tmp/sdmjxxx' })).rejects.toThrow( + 'copy failed' + ); + expect(resumeSource).toHaveBeenCalledWith('sdmjxxx'); + expect(cleanupTargetArtifacts).toHaveBeenCalledWith('sdmjxxx', 'pre_switch_failure'); + expect(copySharedRows).not.toHaveBeenCalled(); + expect(validateAndSwitch).not.toHaveBeenCalled(); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.update).not.toHaveBeenCalled(); + }); + + it('returns a sanitized migration job status', async () => { + prismaService.spaceDataDbMigrationJob.findFirst.mockResolvedValue({ + id: 'sdmjxxx', + spaceId: 'spcxxx', + targetMode: 'migrate-space', + switchOnCompletion: false, + state: 'copying', + targetInternalSchema: internalSchema, + inventory: { baseIds: ['bsexxx'] }, + copyStats: { phase: 'copying_shared_rows' }, + validationStats: null, + lastError: null, + startedAt: new Date('2026-05-06T00:00:00.000Z'), + completedAt: null, + createdTime: new Date('2026-05-06T00:00:00.000Z'), + lastModifiedTime: new Date('2026-05-06T00:01:00.000Z'), + targetConnection: { + provider: 'postgres', + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema, + schemaVersion, + lastValidatedAt: new Date('2026-05-06T00:00:00.000Z'), + lastError: null, + encryptedUrl: 'encrypted-secret', + capabilities, + }, + }); + const service = createService(); + + await expect(service.getMigrationJobStatus('spcxxx', 'sdmjxxx')).resolves.toMatchObject({ + jobId: 'sdmjxxx', + spaceId: 'spcxxx', + state: 'copying', + switchOnCompletion: false, + targetConnection: { + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + }, + copyStats: { + phase: 'copying_shared_rows', + }, + }); + const status = await service.getMigrationJobStatus('spcxxx', 'sdmjxxx'); + expect(JSON.stringify(status)).not.toContain('encrypted-secret'); + expect(prismaService.spaceDataDbMigrationJob.findFirst).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sdmjxxx' }, + }) + ); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts new file mode 100644 index 0000000000..9d4b244cc6 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts @@ -0,0 +1,6804 @@ +import { mkdir, statfs as nodeStatfs } from 'fs/promises'; +import { tmpdir } from 'os'; +import path from 'path'; +import { InjectQueue } from '@nestjs/bullmq'; +import { Inject, Injectable, Optional } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { getMetaDatabaseUrl } from '@teable/db-data-prisma'; +import { PrismaService, ProvisionState, type Prisma } from '@teable/db-main-prisma'; +import type { + IDataDbMigrationJobStatusVo, + IDataDbPreflightRo, + IDataDbPreflightVo, +} from '@teable/openapi'; +import { Queue } from 'bullmq'; +import { CustomHttpException } from '../../custom.exception'; +import { + DataDbClientManager, + type IResolvedDataDatabase, +} from '../../global/data-db-client-manager.service'; +import { BASE_IMPORT_CSV_QUEUE } from '../base/base-import-processor/base-import-csv.processor'; +import { BASE_IMPORT_JUNCTION_CSV_QUEUE } from '../base/base-import-processor/base-import-junction.processor'; +import { TABLE_IMPORT_CSV_CHUNK_QUEUE } from '../import/open-api/import-csv-chunk.processor'; +import { TABLE_IMPORT_CSV_QUEUE } from '../import/open-api/import-csv.processor'; +import { DataDbBaselineService } from './data-db-baseline.service'; +import { resolveDataDbInternalSchema } from './data-db-internal-schema'; +import { + DATA_DB_PREFLIGHT_CLIENT_FACTORY, + dataDbKnexClientFactory, + fingerprintDatabaseUrl, + fingerprintDataDbConnection, + getDatabaseUrlDisplayParts, + type IDataDbPreflightClient, + type IDataDbPreflightClientFactory, + DataDbPreflightService, +} from './data-db-preflight.service'; +import { decryptDataDbUrl, encryptDataDbUrl } from './data-db-url-secret'; +import { + buildMigrationSharedTablePostgresFdwCopyPlans, + buildMigrationSharedTablePsqlCopyPlans, +} from './space-data-db-copy-plan'; +import { + postgresCopyToolsForStrategy, + SpaceDataDbCopyService, + type ISpaceDataDbBaseSchemaCopyResult, + type ISpaceDataDbBaseSchemaCopyStrategy, + type ISpaceDataDbExcludedForeignKey, + type ISpaceDataDbPostgresFdwSharedTableCopyResult, + type ISpaceDataDbSharedTableCopyStrategy, + type ISpaceDataDbSharedTableCopyResult, +} from './space-data-db-copy.service'; +import { + activeSpaceDataDbMigrationStates, + cancelableSpaceDataDbMigrationStates, + migrateSpaceTargetMode, + spaceDataDbBackgroundWriterDrainTimeoutErrorCode, + spaceDataDbComputedDrainTimeoutErrorCode, + spaceDataDbInventoryChangedErrorCode, + spaceDataDbLargeMigrationConfirmationRequiredErrorCode, + spaceDataDbMigrationCanceledErrorCode, + spaceDataDbMigrationActiveErrorCode, + spaceDataDbMigrationCancelConflictErrorCode, + spaceDataDbPostgresToolUnavailableErrorCode, + spaceDataDbRelatedSpacesRequiredErrorCode, + spaceDataDbRollbackUnsafeErrorCode, + spaceDataDbSchemaOperationDrainTimeoutErrorCode, + spaceDataDbStaleActiveJobErrorCode, + spaceDataDbTargetCleanupFailedErrorCode, + spaceDataDbTargetDiskInsufficientErrorCode, + spaceDataDbTargetExtensionMissingErrorCode, + spaceDataDbTargetConflictErrorCode, + spaceDataDbTempDiskInsufficientErrorCode, + spaceDataDbValidationMismatchErrorCode, +} from './space-data-db-migration.constants'; +import { + SpaceDataDbProcessCanceledError, + SpaceDataDbProcessError, + SpaceDataDbProcessPipelineError, + SpaceDataDbProcessPipelineCanceledError, + type ISpaceDataDbProcessRunOptions, +} from './space-data-db-process-runner.service'; +import { + resolveSpaceDataDbRelatedSpaces, + type ISpaceDataDbRelatedSpaces, +} from './space-data-db-related-spaces'; + +export const spaceDataDbStatfsToken = Symbol('SPACE_DATA_DB_STATFS'); + +export type ISpaceDataDbStatfs = typeof nodeStatfs; + +type IPreparedMigrationTarget = { + encryptedUrl: string; + urlFingerprint: string; + displayHost: string; + displayDatabase: string; + internalSchema: string; + schemaVersion: string | null; + capabilities: IDataDbPreflightVo['capabilities']; +}; + +type ISpaceDataDbPhysicalRelation = { + schemaName: string; + relationName: string; + relationKind: string; + totalBytes: number; + estimatedRows: number | null; +}; + +type ISpaceDataDbPhysicalSchema = { + schemaName: string; + relations: ISpaceDataDbPhysicalRelation[]; + totalBytes: number; + estimatedRows: number; +}; + +type ISpaceDataDbPostgresExtensionDependency = { + extensionName: string; + objectType: 'operator_class'; + schemaName: string; + objectName: string; + accessMethod: string; + sourceObjects: string[]; +}; + +type ISpaceDataDbProcessFailureResult = { + command: string; + exitCode?: number | null; + signal?: NodeJS.Signals | null; + stderr?: string; + stdout?: string; +}; + +type ISpaceDataDbProcessFailureStats = + | { + type: 'pipeline'; + message: string; + result: { + source: ISpaceDataDbProcessFailureResult; + target: ISpaceDataDbProcessFailureResult; + }; + } + | { + type: 'process'; + message: string; + result: ISpaceDataDbProcessFailureResult; + } + | { + type: 'unknown'; + message: string; + }; + +type ISpaceDataDbInventorySourceDataDb = { + mode: 'default' | 'byodb'; + cacheKey: string; + connectionId: string | null; + internalSchema: string | null; + isMetaFallback: boolean; +}; + +type ISpaceDataDbInventoryTargetDataDb = { + internalSchema: string; +}; + +type ISpaceDataDbInventoryRelatedSpaces = ISpaceDataDbRelatedSpaces; + +type ISpaceDataDbInventory = { + sourceDataDb: ISpaceDataDbInventorySourceDataDb; + targetDataDb: ISpaceDataDbInventoryTargetDataDb; + relatedSpaces: ISpaceDataDbInventoryRelatedSpaces; + spaceIds: string[]; + copySpaceIds: string[]; + baseIds: string[]; + tableIds: string[]; + sharedTableIds: string[]; + relatedSharedTableIds: string[]; + dbTableNames: string[]; + physicalSchemas: ISpaceDataDbPhysicalSchema[]; + postgresExtensionDependencies: ISpaceDataDbPostgresExtensionDependency[]; + outOfScopeForeignKeys: ISpaceDataDbExcludedForeignKey[]; + estimatedTotalBytes: number; + estimatedTotalRows: number; +}; + +type ITargetConflict = { + object: string; + count?: number; +}; + +type ITargetDiskCapacity = { + checked: boolean; + reason?: string; + dataDirectory?: string; + mountPath?: string; + availableBytes?: number; + totalBytes?: number; + usedBytes?: number; + checkedAt: string; +}; + +type IValidationMismatch = { + object: string; + reason: string; + sourceCount?: number; + targetCount?: number; + sourceKind?: string; + targetKind?: string | null; + sourceColumns?: IBaseRelationColumnSignature[]; + targetColumns?: IBaseRelationColumnSignature[]; + sourceIndexes?: IBaseRelationIndexSignature[]; + targetIndexes?: IBaseRelationIndexSignature[]; + sourceConstraints?: IBaseRelationConstraintSignature[]; + targetConstraints?: IBaseRelationConstraintSignature[]; + sourceTriggers?: IBaseRelationTriggerSignature[]; + targetTriggers?: IBaseRelationTriggerSignature[]; +}; + +type IInventoryChangedMismatch = { + object: string; + reason: 'inventory_changed'; + expectedCount?: number; + actualCount?: number; + added?: string[]; + removed?: string[]; + expected?: unknown; + actual?: unknown; +}; + +type IRowCountValidation = { + object: string; + sourceCount: number; + targetCount: number; +}; + +type IValidationStats = { + phase: 'validation_completed'; + progress: IMigrationProgressStats; + targetSchemaVersion: { latest: string | null; exists: boolean }; + routeSmoke: IRouteSmokeStats; + baseSchemas: IRowCountValidation[]; + sharedTables: IRowCountValidation[]; + undoFunction: { exists: boolean }; + switchOnCompletion?: boolean; + switched?: boolean; + switchedAt?: string; + completedAt: string; +}; + +type IComputedDrainStats = { + activeCount: number; + reclaimableCount: number; + oldestActiveLockedAt: string | null; + checkedAt: string; +}; + +type IRouteSmokeStats = { + ok: boolean; + connectionId: string | null; + internalSchema: string | null; + cacheKey: string | null; + isMetaFallback: boolean; + error?: string; +}; + +type ISharedTableCopySummary = { + strategy?: ISpaceDataDbSharedTableCopyStrategy; + table: string; + copiedRows: number | null; + source?: ISpaceDataDbSharedTableCopyResult['source']; + target: ISpaceDataDbSharedTableCopyResult['target']; +}; + +type ISharedTableCopyHeartbeat = { + stage: 'copying_shared_rows'; + tableNames: string[]; + copiedTableCount: number; + totalTables: number; + totalCopiedRows: number | null; + strategy: ISpaceDataDbSharedTableCopyStrategy; + updatedAt: string; +}; + +type IBaseSchemaRelationCopySummary = { + schemaName: string; + relationName: string; + relationKind: string; + copiedRows: number; + estimatedRows: number | null; + totalBytes: number; +}; + +type IBaseSchemaCopyProgressPhase = 'dump' | 'restore'; +type IBaseSchemaCopyHeartbeatStage = 'progress_poll' | 'post_copy_stats'; +type IValidationHeartbeatStage = 'validating_copy'; + +type IBaseSchemaActiveCopyRelation = { + schemaName: string; + relationName: string; + command: string | null; + copyType: string | null; + bytesProcessed: number | null; + bytesTotal: number | null; + tuplesProcessed: number | null; + tuplesExcluded: number | null; + estimatedRows: number | null; + totalBytes: number | null; +}; + +type IBaseSchemaActiveCopyProgress = { + phase: IBaseSchemaCopyProgressPhase; + sampledAt: string; + activeRelationCount: number; + activeRelations: IBaseSchemaActiveCopyRelation[]; + error?: string; +}; + +type IBaseSchemaCopyHeartbeat = { + stage: IBaseSchemaCopyHeartbeatStage; + phase?: IBaseSchemaCopyProgressPhase; + updatedAt: string; +}; + +type IValidationHeartbeat = { + stage: IValidationHeartbeatStage; + updatedAt: string; +}; + +type IBaseRelationColumnSignature = { + ordinalPosition: number; + columnName: string; + formattedType: string; + notNull: boolean; + defaultExpression: string | null; + identity: string; + generated: string; + collation: string | null; +}; + +type IBaseRelationIndexSignature = { + indexName: string; + isPrimary: boolean; + isUnique: boolean; + isValid: boolean; + definition: string; +}; + +type IBaseRelationConstraintSignature = { + constraintName: string; + constraintType: string; + definition: string; +}; + +type IBaseRelationTriggerSignature = { + triggerName: string; + enabled: string; + definition: string; +}; + +type ITargetArtifactCleanupStats = { + reason: string; + baseSchemas: { + schemaName: string; + dropped: boolean; + }[]; + sharedTables: { + table: string; + deletedRows: number | null; + truncated?: boolean; + }[]; + truncateSharedTables?: boolean; + startedAt: string; + completedAt?: string; + failedAt?: string; + error?: string; +}; + +type IPostSwitchWriteFinding = { + object: string; + reason: string; + count?: number; + expectedCount?: number; + actualCount?: number; + maxChangedAt?: string | null; +}; + +type IPostSwitchRollbackProof = { + eligible: boolean; + switchedAt: string; + checkedAt: string; + findings: IPostSwitchWriteFinding[]; +}; + +type IMigrationProgressStats = { + phase: string; + totalSteps: number; + completedSteps: number; + percent: number; + estimatedTotalBytes: number; + completedEstimatedBytes: number; + estimatedTotalRows: number; + completedEstimatedRows: number; + startedAt: string | null; + updatedAt: string; + etaMs: number | null; +}; + +type ISchemaOperationDrainStats = { + openCount: number; + sample: { + id: string; + status: string; + phase: string; + baseId: string | null; + tableId: string | null; + lockedAt: string | null; + lockedBy: string | null; + lastModifiedTime: string | null; + }[]; + checkedAt: string; +}; + +type IBackgroundWriterDrainStats = { + openCount: number; + provisionResourceCount: number; + queueJobCount: number; + sample: { + kind: 'provision_resource' | 'queue_job'; + resourceType?: 'base' | 'table' | 'field'; + queueName?: string; + id: string; + state: string; + baseId: string | null; + tableId: string | null; + lastModifiedTime?: string | null; + timestamp?: string | null; + }[]; + checkedAt: string; +}; + +type IMigrationJobRecord = { + id: string; + spaceId: string; + state: string; + sourceConnectionId: string | null; + targetConnectionId: string | null; + switchOnCompletion?: boolean | null; + targetInternalSchema: string; + createdBy: string; + startedAt: Date | null; + completedAt: Date | null; + inventory: unknown; + copyStats?: unknown; + validationStats?: unknown; + targetConnection: { encryptedUrl: string } | null; +}; + +type IRunMigrationJobOptions = { + workDir?: string; + jobs?: number; + maxJobs?: number; + baseSchemaCopyStrategy?: ISpaceDataDbBaseSchemaCopyStrategy; + sharedTableCopyStrategy?: ISpaceDataDbSharedTableCopyStrategy; + timeoutMs?: number; + computedDrainTimeoutMs?: number; + computedDrainPollMs?: number; + computedProcessingLeaseMs?: number; + schemaOperationDrainTimeoutMs?: number; + schemaOperationDrainPollMs?: number; + backgroundWriterDrainTimeoutMs?: number; + backgroundWriterDrainPollMs?: number; + backgroundWriterDrainProbeTimeoutMs?: number; + backgroundWriterQueueScanBatchSize?: number; + backgroundWriterQueueScanLimit?: number; + tempDiskMultiplier?: number; + tempDiskMinFreeBytes?: number; +}; + +type IResolvedRunMigrationJobOptions = Required; + +type IMigrationJobStatusRecord = { + id: string; + spaceId: string; + targetMode: string; + switchOnCompletion?: boolean | null; + state: string; + targetInternalSchema: string; + inventory: unknown; + copyStats: unknown; + validationStats: unknown; + lastError: string | null; + startedAt: Date | null; + completedAt: Date | null; + createdTime: Date; + lastModifiedTime: Date | null; + targetConnection: { + provider: 'postgres'; + displayHost: string | null; + displayDatabase: string | null; + internalSchema: string; + schemaVersion: string | null; + lastValidatedAt: Date | null; + lastError: string | null; + capabilities: unknown; + } | null; +}; + +type IRetryableMigrationJobRecord = { + id: string; + spaceId: string; + state: string; + switchOnCompletion?: boolean | null; + targetConnectionId: string | null; + inventory: unknown; +}; + +type IClaimableMigrationJobRecord = { + id: string; + state: string; +}; + +type IBaseSchemaCopyContext = { + inventory: ISpaceDataDbInventory; + schemaNames: string[]; + strategy: ISpaceDataDbBaseSchemaCopyStrategy; + progressPollMs: number; + startedAt: string; +}; + +type IBaseSchemaProgressClients = { + dump: IDataDbPreflightClient; + restore: IDataDbPreflightClient; +}; + +type IStaleMigrationJobRecord = IMigrationJobRecord & { + lastModifiedTime: Date | null; +}; + +type IMigrationJobClient = { + spaceDataDbMigrationJob: { + findFirst( + args: unknown + ): Promise<({ id: string; state: string } | IMigrationJobStatusRecord) | null>; + findMany(args: unknown): Promise; + findUnique(args: unknown): Promise; + create(args: unknown): Promise<{ id: string }>; + update(args: unknown): Promise; + updateMany(args: unknown): Promise<{ count: number }>; + }; +}; + +type IPrismaTransactionClient = { + dataDbConnection: { + upsert(args: unknown): Promise<{ id: string }>; + update(args: unknown): Promise; + }; + spaceDataDbBinding: { + upsert(args: unknown): Promise; + }; + spaceDataDbMigrationJob: { + create(args: unknown): Promise<{ id: string }>; + update(args: unknown): Promise; + }; +}; + +type IDataPrismaIntrospectionClient = { + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +const dataDbUrlRequiredError = 'Data database URL is required'; +const dataDbMigrationTable = '__teable_data_schema_migrations'; +const sharedTables = { + recordHistory: 'record_history', + tableTrash: 'table_trash', + recordTrash: 'record_trash', + computedUpdateOutbox: 'computed_update_outbox', + computedUpdateDeadLetter: 'computed_update_dead_letter', + computedUpdateOutboxSeed: 'computed_update_outbox_seed', + computedUpdatePauseScope: 'computed_update_pause_scope', + undoLog: '__undo_log', +}; +const relationKindsWithRows = new Set(['table', 'partitioned_table', 'foreign_table']); +const relationKindsWithColumns = new Set([ + 'table', + 'partitioned_table', + 'foreign_table', + 'view', + 'materialized_view', +]); +const relationKindsWithIndexSignatures = new Set([ + 'table', + 'partitioned_table', + 'foreign_table', + 'materialized_view', +]); +const relationKindsWithTableDependencySignatures = new Set([ + 'table', + 'partitioned_table', + 'foreign_table', +]); +const validationFailedMessage = 'Space data database migration validation failed'; +const metaFallbackDataDbCacheKey = 'meta-fallback'; +const defaultMigrationCopyTimeoutMs = 24 * 60 * 60 * 1000; +const defaultMigrationCopyJobs = 1; +const defaultMigrationCopyMaxJobs = 4; +const defaultComputedDrainTimeoutMs = 10 * 60 * 1000; +const defaultComputedDrainPollMs = 5 * 1000; +const defaultComputedProcessingLeaseMs = 2 * 60 * 1000; +const defaultSchemaOperationDrainTimeoutMs = 10 * 60 * 1000; +const defaultSchemaOperationDrainPollMs = 5 * 1000; +const defaultBackgroundWriterDrainTimeoutMs = 10 * 60 * 1000; +const defaultBackgroundWriterDrainPollMs = 5 * 1000; +const defaultBackgroundWriterDrainProbeTimeoutMs = 30 * 1000; +const defaultBackgroundWriterQueueScanBatchSize = 100; +const defaultBackgroundWriterQueueScanLimit = 1000; +const defaultTempDiskMultiplier = 2; +const defaultTempDiskMinFreeBytes = 512 * 1024 * 1024; +const defaultTargetDiskMultiplier = 2; +const defaultTargetDiskMinFreeBytes = 1024 * 1024 * 1024; +const defaultLargeMigrationByteThreshold = 50 * 1024 * 1024 * 1024; +const defaultLargeMigrationRowThreshold = 10_000_000; +const defaultPostgresToolCheckTimeoutMs = 5_000; +const defaultCopyProgressPollMs = 5_000; +const defaultCopyProgressPollTimeoutMs = 30_000; +const defaultStaleActiveJobTimeoutMs = 5 * 60 * 1000; +const openSchemaOperationStatuses = ['pending', 'running', 'error'] as const; +const openProvisionStates = [ + ProvisionState.pending, + ProvisionState.deleting, + ProvisionState.error, +] as const; +const openQueueJobStates = [ + 'waiting', + 'active', + 'delayed', + 'prioritized', + 'waiting-children', +] as const; +const migrationProgressTotalSteps = 9; +const migrationProgressCompletedSteps: Record = { + postgres_tools_checking: 0, + postgres_tools_checked: 0, + postgres_tools_unavailable: 0, + computed_paused: 1, + computed_draining: 1, + computed_drained: 2, + computed_drain_timeout: 1, + computed_drain_failed: 1, + schema_operations_draining: 2, + schema_operations_drained: 3, + schema_operation_drain_timeout: 2, + schema_operation_drain_failed: 2, + background_writers_draining: 3, + background_writers_drained: 4, + background_writer_drain_timeout: 3, + background_writer_drain_failed: 3, + source_inventory_verified: 4, + source_inventory_changed: 3, + temp_disk_checked: 5, + temp_disk_insufficient: 4, + copying_base_schemas: 5, + base_schemas_completed: 6, + base_schemas_failed: 5, + copying_shared_rows: 6, + shared_rows_completed: 7, + shared_rows_failed: 6, + validating_copy: 7, + validation_completed: 8, + validation_failed: 7, + switching: 8, + succeeded: 9, + canceled_before_copy: 1, +}; + +const migrationPauseReason = (jobId: string) => `space-data-db-migration:${jobId}`; + +const readPositiveIntEnv = (key: string, fallback: number) => { + const value = Number(process.env[key]); + return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback; +}; + +const readPositiveNumberEnv = (key: string, fallback: number) => { + const value = Number(process.env[key]); + return Number.isFinite(value) && value > 0 ? value : fallback; +}; + +const readNonNegativeIntEnv = (key: string, fallback: number) => { + const value = Number(process.env[key]); + return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback; +}; + +const readBaseSchemaCopyStrategyEnv = ( + key: string, + fallback: ISpaceDataDbBaseSchemaCopyStrategy +): ISpaceDataDbBaseSchemaCopyStrategy => { + const value = process.env[key]; + return value === 'pgcopydb' || value === 'pg_dump_restore' || value === 'pg_dump_stream_restore' + ? value + : fallback; +}; + +const readSharedTableCopyStrategyEnv = ( + key: string, + fallback: ISpaceDataDbSharedTableCopyStrategy +): ISpaceDataDbSharedTableCopyStrategy => { + const value = process.env[key]; + return value === 'postgres_fdw' || value === 'psql_copy' ? value : fallback; +}; + +const optionOrPositiveIntEnv = (optionValue: number | undefined, key: string, fallback: number) => + optionValue ?? readPositiveIntEnv(key, fallback); + +const optionOrPositiveNumberEnv = ( + optionValue: number | undefined, + key: string, + fallback: number +) => optionValue ?? readPositiveNumberEnv(key, fallback); + +const normalizeRawRows = (result: { rows?: T[] } | T[]): T[] => { + if (Array.isArray(result)) { + return result; + } + return result.rows ?? []; +}; + +const sortJsonValue = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map(sortJsonValue); + } + + if (!value || typeof value !== 'object') { + return value; + } + + return Object.keys(value as Record) + .sort() + .reduce>((acc, key) => { + acc[key] = sortJsonValue((value as Record)[key]); + return acc; + }, {}); +}; + +const stableJsonStringify = (value: unknown): string => JSON.stringify(sortJsonValue(value)); + +const buildPostgresExtensionDependencyKey = ( + dependency: Pick< + ISpaceDataDbPostgresExtensionDependency, + 'extensionName' | 'objectType' | 'schemaName' | 'objectName' | 'accessMethod' + > +) => + [ + dependency.extensionName, + dependency.objectType, + dependency.schemaName, + dependency.objectName, + dependency.accessMethod, + ].join(':'); + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const withTimeout = async (promise: Promise, timeoutMs: number, message: string) => { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return promise; + } + + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + timeout.unref?.(); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +}; + +const quoteIdent = (identifier: string) => `"${identifier.replace(/"/g, '""')}"`; + +const qualify = (schema: string, table: string) => `${quoteIdent(schema)}.${quoteIdent(table)}`; + +const shellQuote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + +const buildPreflightErrorMessage = (preflight: IDataDbPreflightVo) => { + const errorCodes = preflight.errors.map((error) => error.code).join(', '); + return errorCodes + ? `Data database preflight failed: ${errorCodes}` + : `Data database preflight failed: ${preflight.classification}`; +}; + +@Injectable() +export class SpaceDataDbMigrationService { + private readonly clientFactory: IDataDbPreflightClientFactory; + + constructor( + private readonly prismaService: PrismaService, + private readonly preflightService: DataDbPreflightService, + private readonly baselineService: DataDbBaselineService, + private readonly dataDbClientManager: DataDbClientManager, + private readonly copyService: SpaceDataDbCopyService, + @Optional() + @Inject(DATA_DB_PREFLIGHT_CLIENT_FACTORY) + clientFactory?: IDataDbPreflightClientFactory, + @Optional() + @Inject(spaceDataDbStatfsToken) + private readonly statfs: ISpaceDataDbStatfs = nodeStatfs, + @Optional() + @InjectQueue(BASE_IMPORT_CSV_QUEUE) + private readonly baseImportCsvQueue?: Queue, + @Optional() + @InjectQueue(BASE_IMPORT_JUNCTION_CSV_QUEUE) + private readonly baseImportJunctionCsvQueue?: Queue, + @Optional() + @InjectQueue(TABLE_IMPORT_CSV_CHUNK_QUEUE) + private readonly tableImportCsvChunkQueue?: Queue, + @Optional() + @InjectQueue(TABLE_IMPORT_CSV_QUEUE) + private readonly tableImportCsvQueue?: Queue + ) { + this.clientFactory = clientFactory ?? dataDbKnexClientFactory; + } + + async startMigrationForSpace(spaceId: string, createdBy: string, dataDb: IDataDbPreflightRo) { + if (dataDb.targetMode !== migrateSpaceTargetMode) { + throw new CustomHttpException( + 'Only migrate-space BYODB target mode can start a space data DB migration', + HttpErrorCode.VALIDATION_ERROR + ); + } + if (!dataDb.url) { + throw new CustomHttpException(dataDbUrlRequiredError, HttpErrorCode.VALIDATION_ERROR); + } + + const relatedSpaces = await resolveSpaceDataDbRelatedSpaces(this.prismaService, spaceId); + this.assertRelatedSpacesCanMigrateTogether(spaceId, relatedSpaces); + const relatedSpaceIds = this.getRelatedSpaceIds(relatedSpaces); + await this.assertNoActiveMigrationForSpaces(relatedSpaceIds, spaceId); + + const targetDatabaseFingerprint = fingerprintDatabaseUrl(dataDb.url); + this.assertSpacesCanJoinTargetDataDb(relatedSpaces, targetDatabaseFingerprint, spaceId); + const internalSchema = await this.resolveMigrationTargetInternalSchemaOrThrow( + dataDb, + relatedSpaces, + targetDatabaseFingerprint, + spaceId + ); + const targetUrlFingerprint = fingerprintDataDbConnection(dataDb.url, internalSchema); + this.assertRelatedSpacesShareSameTarget(relatedSpaces, targetUrlFingerprint, spaceId); + const copySpaceIds = this.getRelatedSpaceIdsForCopy(relatedSpaces, targetUrlFingerprint); + this.assertRequestedSpaceIsCopySource(spaceId, copySpaceIds); + const inventory = await this.buildInventory(spaceId, internalSchema, relatedSpaces, { + copySpaceIds, + }); + this.assertLargeMigrationConfirmed(spaceId, inventory, dataDb.confirmLargeMigration === true); + await this.runTargetPreflightOperation( + dataDb, + internalSchema, + spaceId, + 'check_target_extension_dependencies', + () => this.assertTargetSupportsSourceDependencies(dataDb.url, inventory, spaceId) + ); + await this.assertTargetDiskCapacity(dataDb.url, inventory, spaceId); + await this.cleanupRetryableTargetArtifacts({ + spaceId, + targetUrlFingerprint, + targetInternalSchema: internalSchema, + inventory, + }); + await this.runTargetPreflightOperation( + dataDb, + internalSchema, + spaceId, + 'check_target_conflicts', + () => this.assertTargetHasNoSpaceConflicts(dataDb.url, internalSchema, inventory, spaceId) + ); + const prepared = await this.prepareMigrationTarget(dataDb.url, internalSchema); + + let connectionId: string | undefined; + const runTransaction = this.prismaService.$tx.bind(this.prismaService) as unknown as ( + fn: (prisma: IPrismaTransactionClient) => Promise + ) => Promise; + const job = await runTransaction(async (prisma) => { + const connection = await prisma.dataDbConnection.upsert({ + where: { urlFingerprint: prepared.urlFingerprint }, + create: { + provider: 'postgres', + encryptedUrl: prepared.encryptedUrl, + urlFingerprint: prepared.urlFingerprint, + displayHost: prepared.displayHost, + displayDatabase: prepared.displayDatabase, + internalSchema: prepared.internalSchema, + status: 'migrating', + schemaVersion: prepared.schemaVersion, + capabilities: prepared.capabilities, + lastValidatedAt: new Date(), + createdBy, + }, + update: { + encryptedUrl: prepared.encryptedUrl, + displayHost: prepared.displayHost, + displayDatabase: prepared.displayDatabase, + internalSchema: prepared.internalSchema, + status: 'migrating', + schemaVersion: prepared.schemaVersion, + capabilities: prepared.capabilities, + lastValidatedAt: new Date(), + lastError: null, + }, + select: { id: true }, + }); + connectionId = connection.id; + + return await prisma.spaceDataDbMigrationJob.create({ + data: { + spaceId, + sourceConnectionId: null, + targetConnectionId: connection.id, + targetMode: migrateSpaceTargetMode, + switchOnCompletion: dataDb.switchOnCompletion === true, + state: 'waiting_worker', + targetUrlFingerprint: prepared.urlFingerprint, + targetInternalSchema: prepared.internalSchema, + inventory, + createdBy, + }, + select: { id: true }, + }); + }); + + if (connectionId) { + await this.dataDbClientManager.invalidateConnection(connectionId); + } + + return { + jobId: job.id, + connectionId, + }; + } + + async preflightMigrationTargetForSpace(spaceId: string, dataDb: IDataDbPreflightRo) { + if (!dataDb.url) { + throw new CustomHttpException(dataDbUrlRequiredError, HttpErrorCode.VALIDATION_ERROR); + } + + const relatedSpaces = await resolveSpaceDataDbRelatedSpaces(this.prismaService, spaceId); + this.assertRelatedSpacesCanMigrateTogether(spaceId, relatedSpaces); + const targetDatabaseFingerprint = fingerprintDatabaseUrl(dataDb.url); + this.assertSpacesCanJoinTargetDataDb(relatedSpaces, targetDatabaseFingerprint, spaceId); + const internalSchema = await this.resolveMigrationTargetInternalSchemaOrThrow( + dataDb, + relatedSpaces, + targetDatabaseFingerprint, + spaceId + ); + const targetUrlFingerprint = fingerprintDataDbConnection(dataDb.url, internalSchema); + this.assertRelatedSpacesShareSameTarget(relatedSpaces, targetUrlFingerprint, spaceId); + const copySpaceIds = this.getRelatedSpaceIdsForCopy(relatedSpaces, targetUrlFingerprint); + this.assertRequestedSpaceIsCopySource(spaceId, copySpaceIds); + const inventory = await this.buildInventory(spaceId, internalSchema, relatedSpaces, { + copySpaceIds, + }); + + const preflight = await this.preflightService.preflight({ + ...dataDb, + targetMode: migrateSpaceTargetMode, + internalSchema, + }); + if (!preflight.ok) { + return preflight; + } + + await this.runTargetPreflightOperation( + dataDb, + internalSchema, + spaceId, + 'check_target_extension_dependencies', + () => this.assertTargetSupportsSourceDependencies(dataDb.url, inventory, spaceId) + ); + await this.assertTargetDiskCapacity(dataDb.url, inventory, spaceId); + const restartableTargetArtifactJob = await this.findRestartableTargetArtifactJob({ + spaceId, + targetUrlFingerprint, + targetInternalSchema: internalSchema, + }); + if ( + !restartableTargetArtifactJob || + !this.isSuccessfulDryRunJob(restartableTargetArtifactJob) + ) { + await this.runTargetPreflightOperation( + dataDb, + internalSchema, + spaceId, + 'check_target_conflicts', + () => this.assertTargetHasNoSpaceConflicts(dataDb.url, internalSchema, inventory, spaceId) + ); + } + + return { + ...preflight, + internalSchema, + }; + } + + async claimNextPendingMigrationJob(workerId: string) { + const claimableJob = (await this.migrationJobClient.spaceDataDbMigrationJob.findFirst({ + where: { + targetMode: migrateSpaceTargetMode, + state: { in: ['waiting_worker', 'pending'] }, + }, + orderBy: [{ createdTime: 'asc' }, { id: 'asc' }], + select: { id: true, state: true }, + })) as unknown as IClaimableMigrationJobRecord | null; + + if (!claimableJob) { + return null; + } + + const claimedAt = new Date(); + const claimed = await this.migrationJobClient.spaceDataDbMigrationJob.updateMany({ + where: { + id: claimableJob.id, + state: claimableJob.state, + }, + data: { + state: 'freezing_writes', + startedAt: claimedAt, + lastError: null, + copyStats: { + phase: 'worker_claimed', + worker: { + id: workerId, + previousState: claimableJob.state, + claimedAt: claimedAt.toISOString(), + }, + }, + }, + }); + + if (claimed.count !== 1) { + return null; + } + + return { jobId: claimableJob.id }; + } + + async recoverStaleActiveMigrationJobs( + workerId: string, + options: { staleAfterMs?: number; now?: Date } = {} + ) { + const staleAfterMs = Math.max( + 1, + Math.floor( + options.staleAfterMs ?? + readPositiveIntEnv( + 'BYODB_SPACE_DATA_DB_STALE_ACTIVE_JOB_TIMEOUT_MS', + defaultStaleActiveJobTimeoutMs + ) + ) + ); + const now = options.now ?? new Date(); + const staleBefore = new Date(now.getTime() - staleAfterMs); + const states = activeSpaceDataDbMigrationStates.filter( + (state) => state !== 'pending' && state !== 'waiting_worker' + ); + const jobs = (await this.migrationJobClient.spaceDataDbMigrationJob.findMany({ + where: { + targetMode: migrateSpaceTargetMode, + state: { in: states }, + OR: [{ lastModifiedTime: null }, { lastModifiedTime: { lt: staleBefore } }], + }, + include: { targetConnection: true }, + orderBy: [{ lastModifiedTime: 'asc' }, { createdTime: 'asc' }, { id: 'asc' }], + })) as unknown as IStaleMigrationJobRecord[]; + + const recovered: { jobId: string; state: string; lastError: string }[] = []; + for (const job of jobs) { + const lastError = `Space data database migration job ${job.id} is stale; no worker progress since ${ + job.lastModifiedTime?.toISOString() ?? 'never' + }`; + const marked = await this.migrationJobClient.spaceDataDbMigrationJob.updateMany({ + where: { + id: job.id, + state: job.state, + OR: [{ lastModifiedTime: null }, { lastModifiedTime: { lt: staleBefore } }], + }, + data: { + state: 'failed', + completedAt: now, + lastError, + copyStats: { + ...(this.asRecord(job.copyStats) ?? {}), + staleRecovery: { + errorCode: spaceDataDbStaleActiveJobErrorCode, + workerId, + previousState: job.state, + staleAfterMs, + staleBefore: staleBefore.toISOString(), + recoveredAt: now.toISOString(), + }, + }, + }, + }); + if (marked.count !== 1) { + continue; + } + + await this.resumeSourceComputedForJob(job.id).catch(() => undefined); + if (['copying', 'validating'].includes(job.state)) { + await this.cleanupTargetArtifactsForJob(job.id, 'stale_active_job', { + truncateSharedTables: await this.canTruncateTargetSharedTables(job.targetConnectionId), + }).catch(() => undefined); + } + + recovered.push({ jobId: job.id, state: job.state, lastError }); + } + + return recovered; + } + + private async cleanupRetryableTargetArtifacts(input: { + spaceId: string; + targetUrlFingerprint: string; + targetInternalSchema: string; + inventory: ISpaceDataDbInventory; + }) { + const retryableJob = await this.findRestartableTargetArtifactJob(input); + + if (!retryableJob) { + return; + } + + if (!this.isSuccessfulDryRunJob(retryableJob)) { + const previousInventory = this.normalizeInventory( + retryableJob.inventory, + retryableJob.spaceId ?? input.spaceId + ); + const mismatches = this.compareInventoryForCopy(previousInventory, input.inventory, { + compareSharedScope: false, + }); + if (mismatches.length) { + return; + } + } + + await this.cleanupTargetArtifactsForJob(retryableJob.id, 'retry_before_start', { + truncateSharedTables: await this.canTruncateTargetSharedTables( + retryableJob.targetConnectionId + ), + }); + } + + private async findRestartableTargetArtifactJob(input: { + spaceId: string; + targetUrlFingerprint: string; + targetInternalSchema: string; + }): Promise { + return (await this.migrationJobClient.spaceDataDbMigrationJob.findFirst({ + where: { + spaceId: input.spaceId, + targetMode: migrateSpaceTargetMode, + targetUrlFingerprint: input.targetUrlFingerprint, + targetInternalSchema: input.targetInternalSchema, + OR: [ + { state: { in: ['failed', 'canceled'] } }, + { state: 'succeeded', switchOnCompletion: false }, + ], + }, + orderBy: { createdTime: 'desc' }, + select: { + id: true, + spaceId: true, + state: true, + switchOnCompletion: true, + targetConnectionId: true, + inventory: true, + }, + })) as IRetryableMigrationJobRecord | null; + } + + private isSuccessfulDryRunJob( + job: Pick + ) { + return job.state === 'succeeded' && job.switchOnCompletion !== true; + } + + private async canTruncateTargetSharedTables(targetConnectionId: string | null) { + if (!targetConnectionId) { + return false; + } + const bindings = await this.prismaService.spaceDataDbBinding.findMany({ + where: { dataDbConnectionId: targetConnectionId }, + select: { spaceId: true }, + take: 1, + }); + return bindings.length === 0; + } + + private assertLargeMigrationConfirmed( + spaceId: string, + inventory: ISpaceDataDbInventory, + confirmed: boolean + ) { + const byteThreshold = readNonNegativeIntEnv( + 'BYODB_SPACE_DATA_DB_LARGE_MIGRATION_BYTES', + defaultLargeMigrationByteThreshold + ); + const rowThreshold = readNonNegativeIntEnv( + 'BYODB_SPACE_DATA_DB_LARGE_MIGRATION_ROWS', + defaultLargeMigrationRowThreshold + ); + const reasons = [ + byteThreshold > 0 && inventory.estimatedTotalBytes > byteThreshold + ? { + metric: 'estimatedTotalBytes', + estimated: inventory.estimatedTotalBytes, + threshold: byteThreshold, + } + : null, + rowThreshold > 0 && inventory.estimatedTotalRows > rowThreshold + ? { + metric: 'estimatedTotalRows', + estimated: inventory.estimatedTotalRows, + threshold: rowThreshold, + } + : null, + ].filter(Boolean); + + if (!reasons.length || confirmed) { + return; + } + + throw new CustomHttpException( + 'Space data database migration is above the large-space confirmation threshold', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbLargeMigrationConfirmationRequiredErrorCode, + confirmationField: 'confirmLargeMigration', + spaceId, + estimatedTotalBytes: inventory.estimatedTotalBytes, + estimatedTotalRows: inventory.estimatedTotalRows, + baseCount: inventory.baseIds.length, + tableCount: inventory.tableIds.length, + physicalSchemaCount: inventory.physicalSchemas.length, + thresholds: { + bytes: byteThreshold, + rows: rowThreshold, + }, + reasons, + } + ); + } + + async assertTargetDiskCapacity( + targetUrl: string, + inventory: ISpaceDataDbInventory, + spaceId: string + ) { + const multiplier = readPositiveNumberEnv( + 'BYODB_SPACE_DATA_DB_TARGET_DISK_MULTIPLIER', + defaultTargetDiskMultiplier + ); + const minFreeBytes = readNonNegativeIntEnv( + 'BYODB_SPACE_DATA_DB_TARGET_DISK_MIN_FREE_BYTES', + defaultTargetDiskMinFreeBytes + ); + const requiredBytes = Math.max( + Math.ceil(inventory.estimatedTotalBytes * multiplier), + minFreeBytes + ); + const capacity = await this.inspectTargetDiskCapacity(targetUrl); + + if ( + !capacity.checked || + capacity.availableBytes == null || + capacity.availableBytes >= requiredBytes + ) { + return; + } + + throw new CustomHttpException( + 'Target data database does not have enough disk space for migration', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbTargetDiskInsufficientErrorCode, + spaceId, + estimatedTotalBytes: inventory.estimatedTotalBytes, + estimatedTotalRows: inventory.estimatedTotalRows, + requiredBytes, + availableBytes: capacity.availableBytes, + multiplier, + minFreeBytes, + targetDisk: capacity, + } + ); + } + + private async inspectTargetDiskCapacity(targetUrl: string): Promise { + const checkedAt = new Date().toISOString(); + const client = this.clientFactory(targetUrl); + try { + const dataDirectoryRows = normalizeRawRows<{ setting: string }>( + await client.raw(`SELECT current_setting('data_directory') AS setting`) + ); + const dataDirectory = dataDirectoryRows[0]?.setting; + if (!dataDirectory) { + return { checked: false, reason: 'data_directory_unavailable', checkedAt }; + } + + const command = [ + 'bash', + '-lc', + [ + 'set -euo pipefail', + `target=${shellQuote(dataDirectory)}`, + 'mount=$(df -P "$target" | awk \'NR==2 {print $6}\')', + 'df -B1 -P "$mount" | awk -v mount="$mount" \'NR==2 {print mount "|" $2 "|" $3 "|" $4}\'', + ].join('; '), + ] + .map(shellQuote) + .join(' '); + await client.raw(`CREATE TEMP TABLE IF NOT EXISTS __teable_target_disk_capacity(line text)`); + await client.raw(`TRUNCATE __teable_target_disk_capacity`); + await client.raw( + `COPY __teable_target_disk_capacity FROM PROGRAM '${command.replace(/'/g, "''")}'` + ); + const rows = normalizeRawRows<{ line: string }>( + await client.raw(`SELECT line FROM __teable_target_disk_capacity LIMIT 1`) + ); + const [mountPath, totalBytes, usedBytes, availableBytes] = String(rows[0]?.line ?? '').split( + '|' + ); + const available = Number(availableBytes); + const total = Number(totalBytes); + const used = Number(usedBytes); + if (!mountPath || !Number.isFinite(available)) { + return { checked: false, reason: 'df_output_unavailable', dataDirectory, checkedAt }; + } + return { + checked: true, + dataDirectory, + mountPath, + availableBytes: available, + totalBytes: Number.isFinite(total) ? total : undefined, + usedBytes: Number.isFinite(used) ? used : undefined, + checkedAt, + }; + } catch (error) { + return { + checked: false, + reason: + error instanceof Error ? this.sanitizeTargetDiskError(error.message) : String(error), + checkedAt, + }; + } finally { + await client.destroy().catch(() => undefined); + } + } + + private sanitizeTargetDiskError(message: string) { + return message.replace(/postgresql:\/\/[^@\s]+@/g, 'postgresql://***@').slice(0, 500); + } + + async copyBaseSchemasForJob( + jobId: string, + options: { + workDir: string; + jobs?: number; + timeoutMs?: number; + strategy?: ISpaceDataDbBaseSchemaCopyStrategy; + progressPollMs?: number; + } + ) { + const job = await this.migrationJobClient.spaceDataDbMigrationJob.findUnique({ + where: { id: jobId }, + include: { targetConnection: true }, + }); + if (!job) { + throw new CustomHttpException(`Migration job ${jobId} not found`, HttpErrorCode.NOT_FOUND); + } + if (!job.targetConnection?.encryptedUrl) { + throw new CustomHttpException( + `Migration job ${jobId} has no target connection`, + HttpErrorCode.VALIDATION_ERROR + ); + } + + const context = this.buildBaseSchemaCopyContext(job, options); + const { inventory, schemaNames, strategy, progressPollMs, startedAt } = context; + const sourceDataDb = await this.getSourceDataDbForJob(job); + const targetUrl = decryptDataDbUrl(job.targetConnection.encryptedUrl); + + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'copying', + copyStats: { + phase: 'copying_base_schemas', + progress: this.buildMigrationProgress(job, inventory, 'copying_base_schemas'), + baseSchemas: { + schemaNames, + strategy, + progressPollMs, + excludedForeignKeys: inventory.outOfScopeForeignKeys, + startedAt, + }, + }, + lastError: null, + }, + }); + + let progressClients: IBaseSchemaProgressClients | null = null; + try { + await this.installTargetPublicUndoCaptureCompatibility(targetUrl); + progressClients = this.createBaseSchemaProgressClients(strategy, sourceDataDb.url, targetUrl); + const result = await this.copyService.copyBaseSchemas({ + sourceUrl: sourceDataDb.url, + targetUrl, + schemaNames, + workDir: options.workDir, + jobs: options.jobs, + strategy, + excludedForeignKeys: inventory.outOfScopeForeignKeys, + processOptions: this.buildCancelableProcessOptions( + jobId, + options.timeoutMs, + progressPollMs + ), + hooks: progressClients + ? this.buildBaseSchemaCopyHooks(job, context, progressClients) + : undefined, + }); + const copiedRelations = await this.withMigrationHeartbeat( + () => + this.updateBaseSchemaCopyHeartbeat(job, context, { + stage: 'post_copy_stats', + updatedAt: new Date().toISOString(), + }), + () => this.inspectBaseSchemaCopyRows(targetUrl, inventory), + progressPollMs + ); + const copyStats = this.buildBaseSchemaCompletedStats(job, context, result, copiedRelations); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'copying', + copyStats, + lastError: null, + }, + }); + return copyStats; + } catch (error) { + if (await this.isProcessCancelErrorForJob(error, jobId)) { + throw error; + } + const lastError = this.buildProcessFailureMessage(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'base_schemas_failed', + progress: this.buildMigrationProgress(job, inventory, 'base_schemas_failed'), + baseSchemas: { + schemaNames, + progressPollMs, + excludedForeignKeys: inventory.outOfScopeForeignKeys, + failedAt: new Date().toISOString(), + error: lastError, + failure: this.buildProcessFailureStats(error), + }, + }, + }, + }); + throw error; + } finally { + await progressClients?.dump.destroy().catch(() => undefined); + await progressClients?.restore.destroy().catch(() => undefined); + } + } + + private buildBaseSchemaCopyContext( + job: IMigrationJobRecord, + options: { strategy?: ISpaceDataDbBaseSchemaCopyStrategy; progressPollMs?: number } + ): IBaseSchemaCopyContext { + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const requestedStrategy = options.strategy ?? 'pg_dump_stream_restore'; + return { + inventory, + schemaNames: inventory.physicalSchemas.length + ? inventory.physicalSchemas.map((schema) => schema.schemaName) + : inventory.baseIds, + strategy: + requestedStrategy === 'pg_dump_stream_restore' && inventory.outOfScopeForeignKeys.length + ? 'pg_dump_restore' + : requestedStrategy, + progressPollMs: Math.max( + 1, + Math.floor( + options.progressPollMs ?? + readPositiveIntEnv( + 'BYODB_SPACE_DATA_DB_COPY_PROGRESS_POLL_MS', + defaultCopyProgressPollMs + ) + ) + ), + startedAt: new Date().toISOString(), + }; + } + + private createBaseSchemaProgressClients( + strategy: ISpaceDataDbBaseSchemaCopyStrategy, + sourceUrl: string, + targetUrl: string + ): IBaseSchemaProgressClients | null { + return strategy === 'pg_dump_restore' || strategy === 'pg_dump_stream_restore' + ? { + dump: this.clientFactory(sourceUrl), + restore: this.clientFactory(targetUrl), + } + : null; + } + + private buildBaseSchemaCopyHooks( + job: IMigrationJobRecord, + context: IBaseSchemaCopyContext, + clients: IBaseSchemaProgressClients + ) { + return { + onDumpProgressPoll: () => + this.updateBaseSchemaCopyProgress({ + job, + ...context, + phase: 'dump', + client: clients.dump, + }), + onRestoreProgressPoll: () => + this.updateBaseSchemaCopyProgress({ + job, + ...context, + phase: 'restore', + client: clients.restore, + }), + }; + } + + private async updateBaseSchemaCopyHeartbeat( + job: IMigrationJobRecord, + context: IBaseSchemaCopyContext, + heartbeat: IBaseSchemaCopyHeartbeat + ) { + await this.migrationJobClient.spaceDataDbMigrationJob + .update({ + where: { id: job.id }, + data: { + state: 'copying', + copyStats: { + phase: 'copying_base_schemas', + progress: this.buildMigrationProgress(job, context.inventory, 'copying_base_schemas'), + baseSchemas: { + schemaNames: context.schemaNames, + strategy: context.strategy, + progressPollMs: context.progressPollMs, + excludedForeignKeys: context.inventory.outOfScopeForeignKeys, + startedAt: context.startedAt, + heartbeat, + }, + }, + lastError: null, + }, + }) + .catch(() => undefined); + } + + private buildBaseSchemaProcessStats(result: ISpaceDataDbBaseSchemaCopyResult) { + if (result.strategy === 'pgcopydb') { + return { + strategy: result.strategy, + pgcopydb: result.pgcopydb, + }; + } + if (result.strategy === 'pg_dump_stream_restore') { + return { + strategy: result.strategy, + stream: result.stream, + }; + } + return { + strategy: result.strategy, + dump: result.dump, + ...(result.restoreList ? { restoreList: result.restoreList } : {}), + ...(result.filteredRestoreList ? { filteredRestoreList: result.filteredRestoreList } : {}), + restore: result.restore, + }; + } + + private buildBaseSchemaCompletedStats( + job: IMigrationJobRecord, + context: IBaseSchemaCopyContext, + result: ISpaceDataDbBaseSchemaCopyResult, + copiedRelations: IBaseSchemaRelationCopySummary[] + ) { + return { + phase: 'base_schemas_completed', + progress: this.buildMigrationProgress(job, context.inventory, 'base_schemas_completed'), + baseSchemas: { + schemaNames: context.schemaNames, + progressPollMs: context.progressPollMs, + excludedForeignKeys: context.inventory.outOfScopeForeignKeys, + copiedRelationCount: copiedRelations.length, + totalCopiedRows: copiedRelations.reduce((sum, relation) => sum + relation.copiedRows, 0), + copiedRelations, + ...this.buildBaseSchemaProcessStats(result), + completedAt: new Date().toISOString(), + }, + }; + } + + async runMigrationJob(jobId: string, options: IRunMigrationJobOptions = {}) { + const { + workDir, + jobs, + baseSchemaCopyStrategy, + sharedTableCopyStrategy, + timeoutMs, + computedDrainTimeoutMs, + computedDrainPollMs, + computedProcessingLeaseMs, + schemaOperationDrainTimeoutMs, + schemaOperationDrainPollMs, + backgroundWriterDrainTimeoutMs, + backgroundWriterDrainPollMs, + backgroundWriterDrainProbeTimeoutMs, + backgroundWriterQueueScanBatchSize, + backgroundWriterQueueScanLimit, + tempDiskMultiplier, + tempDiskMinFreeBytes, + } = this.resolveRunMigrationJobOptions(jobId, options); + + await mkdir(workDir, { recursive: true }); + await this.assertMigrationNotCanceled(jobId); + const freezeSourceWrites = await this.shouldFreezeSourceWritesForJob(jobId); + let pause: { created: boolean } = { created: false }; + let targetArtifactsMayExist = false; + try { + await this.assertPostgresCopyToolsAvailableForJob(jobId, { + timeoutMs, + baseSchemaCopyStrategy, + }); + if (freezeSourceWrites) { + pause = await this.pauseSourceComputedForJob(jobId); + await this.waitForSourceComputedDrainForJob(jobId, { + timeoutMs: computedDrainTimeoutMs, + pollMs: computedDrainPollMs, + processingLeaseMs: computedProcessingLeaseMs, + }); + await this.waitForSchemaOperationsForJob(jobId, { + timeoutMs: schemaOperationDrainTimeoutMs, + pollMs: schemaOperationDrainPollMs, + }); + await this.waitForBackgroundWritersForJob(jobId, { + timeoutMs: backgroundWriterDrainTimeoutMs, + pollMs: backgroundWriterDrainPollMs, + probeTimeoutMs: backgroundWriterDrainProbeTimeoutMs, + queueScanBatchSize: backgroundWriterQueueScanBatchSize, + queueScanLimit: backgroundWriterQueueScanLimit, + }); + await this.assertSourceInventoryUnchangedForJob(jobId); + } + await this.assertTempWorkDirCapacityForJob(jobId, workDir, { + multiplier: tempDiskMultiplier, + minFreeBytes: tempDiskMinFreeBytes, + baseSchemaCopyStrategy, + }); + await this.assertMigrationNotCanceled(jobId); + targetArtifactsMayExist = true; + await this.copyBaseSchemasForJob(jobId, { + workDir, + jobs, + timeoutMs, + strategy: baseSchemaCopyStrategy, + }); + await this.copySharedRowsForJob(jobId, { timeoutMs, strategy: sharedTableCopyStrategy }); + return await this.validateAndSwitchJob(jobId); + } catch (error) { + if (pause.created) { + await this.resumeSourceComputedForJob(jobId).catch(() => undefined); + } + if (targetArtifactsMayExist) { + await this.cleanupTargetArtifactsForJob(jobId, 'pre_switch_failure').catch(() => undefined); + } + await this.completeFailedMigrationJob(jobId, error).catch(() => undefined); + throw error; + } + } + + private async shouldFreezeSourceWritesForJob(jobId: string) { + const job = await this.getMigrationJob(jobId); + return job.switchOnCompletion === true; + } + + async assertPostgresCopyToolsAvailableForJob( + jobId: string, + options: { + timeoutMs?: number; + baseSchemaCopyStrategy?: ISpaceDataDbBaseSchemaCopyStrategy; + } = {} + ) { + const baseSchemaCopyStrategy = options.baseSchemaCopyStrategy ?? 'pg_dump_stream_restore'; + const requiredTools = postgresCopyToolsForStrategy(baseSchemaCopyStrategy); + const startedAt = new Date().toISOString(); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + copyStats: { + phase: 'postgres_tools_checking', + postgresTools: { + requiredTools, + startedAt, + }, + }, + lastError: null, + }, + }); + + try { + const results = await this.copyService.assertPostgresToolsAvailable( + baseSchemaCopyStrategy, + this.buildCancelableProcessOptions( + jobId, + Math.min( + options.timeoutMs ?? defaultPostgresToolCheckTimeoutMs, + defaultPostgresToolCheckTimeoutMs + ) + ) + ); + const copyStats = { + phase: 'postgres_tools_checked', + postgresTools: { + requiredTools, + results, + startedAt, + completedAt: new Date().toISOString(), + }, + }; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + copyStats, + lastError: null, + }, + }); + return copyStats; + } catch (error) { + if (await this.isProcessCancelErrorForJob(error, jobId)) { + throw error; + } + const lastError = this.buildProcessFailureMessage(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'postgres_tools_unavailable', + postgresTools: { + requiredTools, + startedAt, + failedAt: new Date().toISOString(), + error: lastError, + failure: this.buildProcessFailureStats(error), + }, + }, + }, + }); + throw new CustomHttpException( + 'Required PostgreSQL client tools are unavailable for space data DB migration', + HttpErrorCode.VALIDATION_ERROR, + { + errorCode: spaceDataDbPostgresToolUnavailableErrorCode, + migrationJobId: jobId, + requiredTools, + cause: lastError, + } + ); + } + } + + private async updateBaseSchemaCopyProgress(input: { + job: IMigrationJobRecord; + inventory: ISpaceDataDbInventory; + schemaNames: string[]; + strategy: ISpaceDataDbBaseSchemaCopyStrategy; + progressPollMs: number; + startedAt: string; + phase: IBaseSchemaCopyProgressPhase; + client: IDataDbPreflightClient; + }) { + await this.updateBaseSchemaCopyHeartbeat(input.job, input, { + stage: 'progress_poll', + phase: input.phase, + updatedAt: new Date().toISOString(), + }); + + const activeCopy = await this.inspectBaseSchemaActiveCopyProgress( + input.client, + input.inventory, + input.schemaNames, + input.phase + ).catch((error) => ({ + phase: input.phase, + sampledAt: new Date().toISOString(), + activeRelationCount: 0, + activeRelations: [], + error: error instanceof Error ? error.message : String(error), + })); + + await this.migrationJobClient.spaceDataDbMigrationJob + .update({ + where: { id: input.job.id }, + data: { + state: 'copying', + copyStats: { + phase: 'copying_base_schemas', + progress: this.buildMigrationProgress( + input.job, + input.inventory, + 'copying_base_schemas' + ), + baseSchemas: { + schemaNames: input.schemaNames, + strategy: input.strategy, + progressPollMs: input.progressPollMs, + startedAt: input.startedAt, + activeCopy, + heartbeat: { + stage: 'progress_poll', + phase: input.phase, + updatedAt: activeCopy.sampledAt, + }, + progressUpdatedAt: activeCopy.sampledAt, + }, + }, + lastError: null, + }, + }) + .catch(() => undefined); + } + + private async inspectBaseSchemaActiveCopyProgress( + client: IDataDbPreflightClient, + inventory: ISpaceDataDbInventory, + schemaNames: string[], + phase: IBaseSchemaCopyProgressPhase + ): Promise { + const sampledAt = new Date().toISOString(); + if (!schemaNames.length) { + return { + phase, + sampledAt, + activeRelationCount: 0, + activeRelations: [], + }; + } + + const rows = normalizeRawRows<{ + schemaName: string; + relationName: string; + command: string | null; + copyType: string | null; + bytesProcessed: string | number | bigint | null; + bytesTotal: string | number | bigint | null; + tuplesProcessed: string | number | bigint | null; + tuplesExcluded: string | number | bigint | null; + }>( + await client.raw( + ` + SELECT + n.nspname AS "schemaName", + c.relname AS "relationName", + p.command AS "command", + p.type AS "copyType", + p.bytes_processed AS "bytesProcessed", + p.bytes_total AS "bytesTotal", + p.tuples_processed AS "tuplesProcessed", + p.tuples_excluded AS "tuplesExcluded" + FROM pg_stat_progress_copy p + JOIN pg_class c ON c.oid = p.relid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = ANY(?::text[]) + ORDER BY n.nspname ASC, c.relname ASC, p.pid ASC + `, + [schemaNames] + ) + ); + const relationStats = new Map( + inventory.physicalSchemas + .flatMap((schema) => schema.relations) + .map((relation) => [`${relation.schemaName}.${relation.relationName}`, relation] as const) + ); + const activeRelations = rows.map((row) => { + const relation = relationStats.get(`${row.schemaName}.${row.relationName}`); + return { + schemaName: row.schemaName, + relationName: row.relationName, + command: row.command ?? null, + copyType: row.copyType ?? null, + bytesProcessed: this.toNullableNumber(row.bytesProcessed), + bytesTotal: this.toNullableNumber(row.bytesTotal), + tuplesProcessed: this.toNullableNumber(row.tuplesProcessed), + tuplesExcluded: this.toNullableNumber(row.tuplesExcluded), + estimatedRows: relation?.estimatedRows ?? null, + totalBytes: relation?.totalBytes ?? null, + }; + }); + + return { + phase, + sampledAt, + activeRelationCount: activeRelations.length, + activeRelations, + }; + } + + private async installTargetPublicUndoCaptureCompatibility(targetUrl: string) { + const client = this.clientFactory(targetUrl); + try { + await client.raw(` + CREATE SCHEMA IF NOT EXISTS "public"; + CREATE OR REPLACE FUNCTION "public"."__teable_capture_undo_row"() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + DECLARE + batch_id text; + captured_record_id text; + captured_old_row jsonb; + captured_new_row jsonb; + BEGIN + batch_id := current_setting('teable.undo_batch_id', true); + + IF TG_OP = 'INSERT' THEN + captured_record_id := COALESCE(NEW."__id"::text, ''); + captured_new_row := to_jsonb(NEW); + ELSIF TG_OP = 'UPDATE' THEN + captured_record_id := COALESCE(NEW."__id"::text, OLD."__id"::text, ''); + captured_old_row := to_jsonb(OLD); + captured_new_row := to_jsonb(NEW); + ELSIF TG_OP = 'DELETE' THEN + captured_record_id := COALESCE(OLD."__id"::text, ''); + captured_old_row := to_jsonb(OLD); + END IF; + + IF batch_id IS NULL OR batch_id = '' THEN + RETURN NULL; + END IF; + + INSERT INTO "__undo_log" ( + "batch_id", + "operation", + "table_name", + "record_id", + "old_row", + "new_row" + ) + VALUES ( + batch_id, + TG_OP, + TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, + captured_record_id, + captured_old_row, + captured_new_row + ); + + RETURN NULL; + END; + $$; + `); + } finally { + await client.destroy().catch(() => undefined); + } + } + + private async inspectBaseSchemaCopyRows( + targetUrl: string, + inventory: ISpaceDataDbInventory + ): Promise { + const client = this.clientFactory(targetUrl); + try { + const summaries: IBaseSchemaRelationCopySummary[] = []; + const relations = inventory.physicalSchemas.flatMap((schema) => schema.relations); + for (const relation of relations) { + if (!relationKindsWithRows.has(relation.relationKind)) { + continue; + } + summaries.push({ + schemaName: relation.schemaName, + relationName: relation.relationName, + relationKind: relation.relationKind, + copiedRows: await this.countRows(client, relation.schemaName, relation.relationName), + estimatedRows: relation.estimatedRows, + totalBytes: relation.totalBytes, + }); + } + return summaries.sort( + (left, right) => + left.schemaName.localeCompare(right.schemaName) || + left.relationName.localeCompare(right.relationName) + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + private async withMigrationHeartbeat( + heartbeat: () => Promise, + task: () => Promise, + intervalMs: number + ): Promise { + let active = true; + const heartbeatState: { inFlight: Promise | null } = { inFlight: null }; + const runHeartbeat = () => { + if (!active || heartbeatState.inFlight) { + return heartbeatState.inFlight; + } + heartbeatState.inFlight = heartbeat() + .catch(() => undefined) + .finally(() => { + heartbeatState.inFlight = null; + }); + return heartbeatState.inFlight; + }; + + await runHeartbeat(); + const timer = setInterval( + () => { + void runHeartbeat(); + }, + Math.max(1, Math.floor(intervalMs)) + ); + try { + return await task(); + } finally { + active = false; + clearInterval(timer); + const lastHeartbeat = heartbeatState.inFlight; + if (lastHeartbeat) { + await lastHeartbeat.catch(() => undefined); + } + } + } + + private resolveRunMigrationJobOptions( + jobId: string, + options: IRunMigrationJobOptions + ): IResolvedRunMigrationJobOptions { + const requestedJobs = + options.jobs ?? readPositiveIntEnv('BYODB_SPACE_DATA_DB_COPY_JOBS', defaultMigrationCopyJobs); + const maxJobs = + options.maxJobs ?? + readPositiveIntEnv('BYODB_SPACE_DATA_DB_COPY_MAX_JOBS', defaultMigrationCopyMaxJobs); + return { + workDir: options.workDir ?? path.join(tmpdir(), 'teable-space-data-db-migrations', jobId), + jobs: Math.min(Math.max(1, Math.floor(requestedJobs)), Math.max(1, Math.floor(maxJobs))), + maxJobs, + baseSchemaCopyStrategy: + options.baseSchemaCopyStrategy ?? + readBaseSchemaCopyStrategyEnv( + 'BYODB_SPACE_DATA_DB_BASE_SCHEMA_COPY_STRATEGY', + 'pg_dump_stream_restore' + ), + sharedTableCopyStrategy: + options.sharedTableCopyStrategy ?? + readSharedTableCopyStrategyEnv( + 'BYODB_SPACE_DATA_DB_SHARED_TABLE_COPY_STRATEGY', + 'psql_copy' + ), + timeoutMs: optionOrPositiveIntEnv( + options.timeoutMs, + 'BYODB_SPACE_DATA_DB_COPY_TIMEOUT_MS', + defaultMigrationCopyTimeoutMs + ), + computedDrainTimeoutMs: optionOrPositiveIntEnv( + options.computedDrainTimeoutMs, + 'BYODB_SPACE_DATA_DB_COMPUTED_DRAIN_TIMEOUT_MS', + defaultComputedDrainTimeoutMs + ), + computedDrainPollMs: optionOrPositiveIntEnv( + options.computedDrainPollMs, + 'BYODB_SPACE_DATA_DB_COMPUTED_DRAIN_POLL_MS', + defaultComputedDrainPollMs + ), + computedProcessingLeaseMs: optionOrPositiveIntEnv( + options.computedProcessingLeaseMs, + 'BYODB_SPACE_DATA_DB_COMPUTED_PROCESSING_LEASE_MS', + defaultComputedProcessingLeaseMs + ), + schemaOperationDrainTimeoutMs: optionOrPositiveIntEnv( + options.schemaOperationDrainTimeoutMs, + 'BYODB_SPACE_DATA_DB_SCHEMA_OPERATION_DRAIN_TIMEOUT_MS', + defaultSchemaOperationDrainTimeoutMs + ), + schemaOperationDrainPollMs: optionOrPositiveIntEnv( + options.schemaOperationDrainPollMs, + 'BYODB_SPACE_DATA_DB_SCHEMA_OPERATION_DRAIN_POLL_MS', + defaultSchemaOperationDrainPollMs + ), + backgroundWriterDrainTimeoutMs: optionOrPositiveIntEnv( + options.backgroundWriterDrainTimeoutMs, + 'BYODB_SPACE_DATA_DB_BACKGROUND_WRITER_DRAIN_TIMEOUT_MS', + defaultBackgroundWriterDrainTimeoutMs + ), + backgroundWriterDrainPollMs: optionOrPositiveIntEnv( + options.backgroundWriterDrainPollMs, + 'BYODB_SPACE_DATA_DB_BACKGROUND_WRITER_DRAIN_POLL_MS', + defaultBackgroundWriterDrainPollMs + ), + backgroundWriterDrainProbeTimeoutMs: optionOrPositiveIntEnv( + options.backgroundWriterDrainProbeTimeoutMs, + 'BYODB_SPACE_DATA_DB_BACKGROUND_WRITER_DRAIN_PROBE_TIMEOUT_MS', + defaultBackgroundWriterDrainProbeTimeoutMs + ), + backgroundWriterQueueScanBatchSize: optionOrPositiveIntEnv( + options.backgroundWriterQueueScanBatchSize, + 'BYODB_SPACE_DATA_DB_BACKGROUND_WRITER_QUEUE_SCAN_BATCH_SIZE', + defaultBackgroundWriterQueueScanBatchSize + ), + backgroundWriterQueueScanLimit: optionOrPositiveIntEnv( + options.backgroundWriterQueueScanLimit, + 'BYODB_SPACE_DATA_DB_BACKGROUND_WRITER_QUEUE_SCAN_LIMIT', + defaultBackgroundWriterQueueScanLimit + ), + tempDiskMultiplier: optionOrPositiveNumberEnv( + options.tempDiskMultiplier, + 'BYODB_SPACE_DATA_DB_TEMP_DISK_MULTIPLIER', + defaultTempDiskMultiplier + ), + tempDiskMinFreeBytes: optionOrPositiveIntEnv( + options.tempDiskMinFreeBytes, + 'BYODB_SPACE_DATA_DB_TEMP_DISK_MIN_FREE_BYTES', + defaultTempDiskMinFreeBytes + ), + }; + } + + async waitForSchemaOperationsForJob( + jobId: string, + options: { timeoutMs?: number; pollMs?: number } = {} + ): Promise { + const timeoutMs = Math.max( + 0, + Math.floor(options.timeoutMs ?? defaultSchemaOperationDrainTimeoutMs) + ); + const pollMs = Math.max(1, Math.floor(options.pollMs ?? defaultSchemaOperationDrainPollMs)); + const startedAt = Date.now(); + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + + if (!inventory.baseIds.length && !inventory.tableIds.length) { + const drained = this.buildSchemaOperationDrainStats(0, []); + await this.updateSchemaOperationDrainStats( + job, + inventory, + 'schema_operations_drained', + drained + ); + return drained; + } + + await this.updateSchemaOperationDrainStats( + job, + inventory, + 'schema_operations_draining', + this.buildSchemaOperationDrainStats(0, []) + ); + + try { + for (;;) { + await this.assertMigrationNotCanceled(jobId, job.spaceId); + const stats = await this.inspectSchemaOperationDrain(inventory); + if (stats.openCount === 0) { + await this.updateSchemaOperationDrainStats( + job, + inventory, + 'schema_operations_drained', + stats + ); + return stats; + } + + if (Date.now() - startedAt >= timeoutMs) { + const lastError = `Timed out waiting for schema operations to drain for migration job ${jobId}`; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'schema_operation_drain_timeout', + progress: this.buildMigrationProgress( + job, + inventory, + 'schema_operation_drain_timeout' + ), + schemaOperations: stats, + }, + }, + }); + throw new CustomHttpException(lastError, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbSchemaOperationDrainTimeoutErrorCode, + openCount: stats.openCount, + spaceId: job.spaceId, + migrationJobId: jobId, + }); + } + + await this.updateSchemaOperationDrainStats( + job, + inventory, + 'schema_operations_draining', + stats + ); + await delay(pollMs); + } + } catch (error) { + if (error instanceof CustomHttpException) { + throw error; + } + const lastError = this.buildProcessFailureMessage(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'schema_operation_drain_failed', + progress: this.buildMigrationProgress(job, inventory, 'schema_operation_drain_failed'), + schemaOperations: { + failedAt: new Date().toISOString(), + }, + }, + }, + }); + throw error; + } + } + + async waitForBackgroundWritersForJob( + jobId: string, + options: { + timeoutMs?: number; + pollMs?: number; + probeTimeoutMs?: number; + queueScanBatchSize?: number; + queueScanLimit?: number; + } = {} + ): Promise { + const timeoutMs = Math.max( + 0, + Math.floor(options.timeoutMs ?? defaultBackgroundWriterDrainTimeoutMs) + ); + const pollMs = Math.max(1, Math.floor(options.pollMs ?? defaultBackgroundWriterDrainPollMs)); + const probeTimeoutMs = Math.max( + 0, + Math.floor(options.probeTimeoutMs ?? defaultBackgroundWriterDrainProbeTimeoutMs) + ); + const queueScanBatchSize = Math.max( + 1, + Math.floor(options.queueScanBatchSize ?? defaultBackgroundWriterQueueScanBatchSize) + ); + const queueScanLimit = Math.max( + 1, + Math.floor(options.queueScanLimit ?? defaultBackgroundWriterQueueScanLimit) + ); + const startedAt = Date.now(); + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + + await this.updateBackgroundWriterDrainStats( + job, + inventory, + 'background_writers_draining', + this.buildBackgroundWriterDrainStats( + { openCount: 0, sample: [] }, + { openCount: 0, sample: [] } + ) + ); + + try { + for (;;) { + await this.assertMigrationNotCanceled(jobId, job.spaceId); + const stats = await this.inspectBackgroundWriterDrain(job.spaceId, inventory, { + probeTimeoutMs, + queueScanBatchSize, + queueScanLimit, + }); + if (stats.openCount === 0) { + await this.updateBackgroundWriterDrainStats( + job, + inventory, + 'background_writers_drained', + stats + ); + return stats; + } + + if (Date.now() - startedAt >= timeoutMs) { + const lastError = `Timed out waiting for background writers to drain for migration job ${jobId}`; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'background_writer_drain_timeout', + progress: this.buildMigrationProgress( + job, + inventory, + 'background_writer_drain_timeout' + ), + backgroundWriters: stats, + }, + }, + }); + throw new CustomHttpException(lastError, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbBackgroundWriterDrainTimeoutErrorCode, + openCount: stats.openCount, + provisionResourceCount: stats.provisionResourceCount, + queueJobCount: stats.queueJobCount, + spaceId: job.spaceId, + migrationJobId: jobId, + }); + } + + await this.updateBackgroundWriterDrainStats( + job, + inventory, + 'background_writers_draining', + stats + ); + await delay(pollMs); + } + } catch (error) { + if (error instanceof CustomHttpException) { + throw error; + } + const lastError = error instanceof Error ? error.message : String(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'background_writer_drain_failed', + progress: this.buildMigrationProgress(job, inventory, 'background_writer_drain_failed'), + backgroundWriters: { + failedAt: new Date().toISOString(), + }, + }, + }, + }); + throw error; + } + } + + async assertSourceInventoryUnchangedForJob(jobId: string): Promise { + const job = await this.getMigrationJob(jobId); + const expected = this.normalizeInventory(job.inventory, job.spaceId); + const actual = await this.buildInventory(job.spaceId, job.targetInternalSchema, undefined, { + copySpaceIds: expected.copySpaceIds, + }); + const mismatches = this.compareInventoryForCopy(expected, actual, { + compareSharedScope: true, + }); + const checkedAt = new Date().toISOString(); + + if (!mismatches.length) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + copyStats: { + phase: 'source_inventory_verified', + progress: this.buildMigrationProgress(job, actual, 'source_inventory_verified'), + sourceInventory: { + baseCount: actual.baseIds.length, + tableCount: actual.tableIds.length, + physicalRelationCount: this.buildPhysicalRelationKeys(actual).length, + checkedAt, + }, + }, + lastError: null, + }, + }); + return; + } + + const lastError = `Source inventory changed after migration job ${jobId} was created`; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'source_inventory_changed', + progress: this.buildMigrationProgress(job, expected, 'source_inventory_changed'), + sourceInventory: { + mismatches, + checkedAt, + }, + }, + }, + }); + throw new CustomHttpException(lastError, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbInventoryChangedErrorCode, + mismatches, + spaceId: job.spaceId, + migrationJobId: jobId, + }); + } + + async assertTempWorkDirCapacityForJob( + jobId: string, + workDir: string, + options: { + multiplier?: number; + minFreeBytes?: number; + baseSchemaCopyStrategy?: ISpaceDataDbBaseSchemaCopyStrategy; + } = {} + ): Promise { + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const multiplier = Math.max(1, options.multiplier ?? defaultTempDiskMultiplier); + const minFreeBytes = Math.max(0, options.minFreeBytes ?? defaultTempDiskMinFreeBytes); + const strategy = options.baseSchemaCopyStrategy ?? 'pg_dump_stream_restore'; + const requiresFullDumpSpace = strategy !== 'pg_dump_stream_restore'; + const estimatedDumpBytes = requiresFullDumpSpace + ? Math.ceil(inventory.estimatedTotalBytes * multiplier) + : 0; + const requiredBytes = Math.max(estimatedDumpBytes, minFreeBytes); + const checkedAt = new Date().toISOString(); + const stats = (await this.statfs(workDir)) as { + bavail?: number | bigint; + bfree: number | bigint; + bsize: number | bigint; + }; + const availableBlocks = stats.bavail ?? stats.bfree; + const availableBytes = Number(availableBlocks) * Number(stats.bsize); + const tempDisk = { + workDir, + strategy, + estimatedTotalBytes: inventory.estimatedTotalBytes, + multiplier, + requiresFullDumpSpace, + requiredBytes, + availableBytes, + checkedAt, + }; + + if (availableBytes >= requiredBytes) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + copyStats: { + phase: 'temp_disk_checked', + progress: this.buildMigrationProgress(job, inventory, 'temp_disk_checked'), + tempDisk, + }, + lastError: null, + }, + }); + return; + } + + const lastError = `Insufficient temp disk space for migration job ${jobId}`; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'temp_disk_insufficient', + progress: this.buildMigrationProgress(job, inventory, 'temp_disk_insufficient'), + tempDisk, + }, + }, + }); + throw new CustomHttpException(lastError, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbTempDiskInsufficientErrorCode, + requiredBytes, + availableBytes, + workDir, + spaceId: job.spaceId, + migrationJobId: jobId, + }); + } + + async waitForSourceComputedDrainForJob( + jobId: string, + options: { timeoutMs?: number; pollMs?: number; processingLeaseMs?: number } = {} + ): Promise { + const timeoutMs = Math.max(0, Math.floor(options.timeoutMs ?? defaultComputedDrainTimeoutMs)); + const pollMs = Math.max(1, Math.floor(options.pollMs ?? defaultComputedDrainPollMs)); + const processingLeaseMs = Math.max( + 1, + Math.floor(options.processingLeaseMs ?? defaultComputedProcessingLeaseMs) + ); + const startedAt = Date.now(); + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + + if (!inventory.baseIds.length) { + const drained = this.buildComputedDrainStats({ + activeCount: 0, + reclaimableCount: 0, + oldestActiveLockedAt: null, + }); + await this.updateComputedDrainStats(job, inventory, 'computed_drained', drained); + return drained; + } + + const sourceDataDb = await this.getSourceDataDbForJob(job); + const sourceSchema = sourceDataDb.internalSchema ?? 'public'; + const client = this.clientFactory(sourceDataDb.url); + + await this.updateComputedDrainStats( + job, + inventory, + 'computed_draining', + this.buildComputedDrainStats({ + activeCount: 0, + reclaimableCount: 0, + oldestActiveLockedAt: null, + }) + ); + + try { + try { + for (;;) { + await this.assertMigrationNotCanceled(jobId, job.spaceId); + const stats = await this.inspectSourceComputedDrain( + client, + sourceSchema, + inventory.baseIds, + processingLeaseMs + ); + + if (stats.activeCount === 0) { + await this.updateComputedDrainStats(job, inventory, 'computed_drained', stats); + return stats; + } + + if (Date.now() - startedAt >= timeoutMs) { + const lastError = `Timed out waiting for source computed tasks to drain for migration job ${jobId}`; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'computed_drain_timeout', + progress: this.buildMigrationProgress(job, inventory, 'computed_drain_timeout'), + computedDrain: stats, + }, + }, + }); + throw new CustomHttpException(lastError, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbComputedDrainTimeoutErrorCode, + activeCount: stats.activeCount, + reclaimableCount: stats.reclaimableCount, + spaceId: job.spaceId, + migrationJobId: jobId, + }); + } + + await this.updateComputedDrainStats(job, inventory, 'computed_draining', stats); + await delay(pollMs); + } + } catch (error) { + if (error instanceof CustomHttpException) { + throw error; + } + const lastError = error instanceof Error ? error.message : String(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'computed_drain_failed', + progress: this.buildMigrationProgress(job, inventory, 'computed_drain_failed'), + computedDrain: { + failedAt: new Date().toISOString(), + }, + }, + }, + }); + throw error; + } + } finally { + await client.destroy().catch(() => undefined); + } + } + + async pauseSourceComputedForJob(jobId: string): Promise<{ created: boolean }> { + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const spaceIds = this.getInventoryCopySpaceIds(inventory); + const sourceDataDb = await this.getSourceDataDbForJob(job); + const sourceSchema = sourceDataDb.internalSchema ?? 'public'; + const client = this.clientFactory(sourceDataDb.url); + + try { + const valuesSql = spaceIds + .map(() => `(?, 'space', ?, now(), ?, NULL, ?, now(), ?)`) + .join(', '); + const bindings = spaceIds.flatMap((spaceId) => [ + `sdmp_${job.id}_${spaceId}`, + spaceId, + job.createdBy, + migrationPauseReason(job.id), + job.createdBy, + ]); + const rows = normalizeRawRows<{ id: string }>( + await client.raw( + ` + INSERT INTO ${qualify(sourceSchema, sharedTables.computedUpdatePauseScope)} + ("id", "scope_type", "scope_id", "paused_at", "paused_by", "resume_at", "reason", "updated_at", "updated_by") + VALUES ${valuesSql} + ON CONFLICT ("scope_type", "scope_id") DO NOTHING + RETURNING "id" + `, + bindings + ) + ); + + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + copyStats: { + phase: 'computed_paused', + progress: this.buildMigrationProgress( + job, + this.normalizeInventory(job.inventory, job.spaceId), + 'computed_paused' + ), + computedPause: { + sourceSchema, + created: rows.length > 0, + createdCount: rows.length, + spaceIds, + reason: migrationPauseReason(job.id), + pausedAt: new Date().toISOString(), + }, + }, + lastError: null, + }, + }); + + return { created: rows.length > 0 }; + } finally { + await client.destroy().catch(() => undefined); + } + } + + async resumeSourceComputedForJob(jobId: string): Promise<{ deleted: number }> { + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const sourceDataDb = await this.getSourceDataDbForJob(job); + const sourceSchema = sourceDataDb.internalSchema ?? 'public'; + const client = this.clientFactory(sourceDataDb.url); + + try { + return await this.deleteMigrationComputedPause( + client, + sourceSchema, + this.getInventoryCopySpaceIds(inventory), + job.id + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + async resumeTargetComputedForJob(jobId: string): Promise<{ deleted: number }> { + const job = await this.getMigrationJob(jobId); + if (!job.targetConnection?.encryptedUrl) { + throw new CustomHttpException( + `Migration job ${jobId} has no target connection`, + HttpErrorCode.VALIDATION_ERROR + ); + } + const client = this.clientFactory(decryptDataDbUrl(job.targetConnection.encryptedUrl)); + + try { + return await this.deleteMigrationComputedPause( + client, + job.targetInternalSchema, + this.getInventoryCopySpaceIds(this.normalizeInventory(job.inventory, job.spaceId)), + job.id + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + async getMigrationJobStatus( + spaceId: string, + jobId: string + ): Promise { + const job = (await this.migrationJobClient.spaceDataDbMigrationJob.findFirst({ + where: { id: jobId }, + include: { targetConnection: true }, + })) as IMigrationJobStatusRecord | null; + if (!job || !this.jobStatusIncludesSpace(job, spaceId)) { + throw new CustomHttpException( + `Migration job ${jobId} was not found for space ${spaceId}`, + HttpErrorCode.NOT_FOUND + ); + } + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + + return { + jobId: job.id, + spaceId: job.spaceId, + targetMode: 'migrate-space', + switchOnCompletion: job.switchOnCompletion === true, + state: job.state as IDataDbMigrationJobStatusVo['state'], + targetInternalSchema: job.targetInternalSchema, + targetConnection: job.targetConnection + ? { + provider: job.targetConnection.provider, + displayHost: job.targetConnection.displayHost ?? undefined, + displayDatabase: job.targetConnection.displayDatabase ?? undefined, + internalSchema: job.targetConnection.internalSchema, + schemaVersion: job.targetConnection.schemaVersion, + lastValidatedAt: job.targetConnection.lastValidatedAt?.toISOString(), + lastError: job.targetConnection.lastError ?? undefined, + capabilities: job.targetConnection.capabilities as IDataDbPreflightVo['capabilities'], + } + : null, + relatedSpaces: inventory.relatedSpaces, + inventory: job.inventory, + copyStats: job.copyStats, + validationStats: job.validationStats, + lastError: job.lastError, + startedAt: job.startedAt?.toISOString() ?? null, + completedAt: job.completedAt?.toISOString() ?? null, + createdTime: job.createdTime.toISOString(), + lastModifiedTime: job.lastModifiedTime?.toISOString() ?? null, + }; + } + + private jobStatusIncludesSpace(job: IMigrationJobStatusRecord, spaceId: string) { + if (job.spaceId === spaceId) { + return true; + } + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + return this.getInventorySpaceIds(inventory).includes(spaceId); + } + + async cancelMigrationForSpace( + spaceId: string, + jobId: string, + canceledBy: string + ): Promise { + const job = await this.getMigrationJob(jobId); + if ( + !this.getInventorySpaceIds(this.normalizeInventory(job.inventory, job.spaceId)).includes( + spaceId + ) + ) { + throw new CustomHttpException( + `Migration job ${jobId} was not found for space ${spaceId}`, + HttpErrorCode.NOT_FOUND + ); + } + + if (job.state === 'canceled') { + return await this.getMigrationJobStatus(spaceId, jobId); + } + + if (!cancelableSpaceDataDbMigrationStates.includes(job.state as never)) { + throw new CustomHttpException( + 'Only pre-validation space data database migrations can be canceled safely', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbMigrationCancelConflictErrorCode, + migrationJobId: jobId, + migrationState: job.state, + spaceId, + } + ); + } + + await this.resumeSourceComputedForJob(jobId); + + const lastError = `Space data database migration canceled by ${canceledBy || 'unknown user'}`; + const runTransaction = this.prismaService.$tx.bind(this.prismaService) as unknown as ( + fn: (prisma: IPrismaTransactionClient) => Promise + ) => Promise; + await runTransaction(async (prisma) => { + if (job.targetConnectionId) { + await prisma.dataDbConnection.update({ + where: { id: job.targetConnectionId }, + data: { + status: 'error', + lastError, + }, + }); + } + await prisma.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'canceled', + completedAt: new Date(), + lastError, + copyStats: { + phase: 'canceled_before_copy', + progress: this.buildMigrationProgress( + job, + this.normalizeInventory(job.inventory, job.spaceId), + 'canceled_before_copy' + ), + canceled: { + canceledBy, + canceledAt: new Date().toISOString(), + }, + }, + }, + }); + }); + + return await this.getMigrationJobStatus(spaceId, jobId); + } + + async rollbackMigrationForSpace( + spaceId: string, + jobId: string, + rolledBackBy: string + ): Promise { + const job = await this.getMigrationJob(jobId); + if ( + !this.getInventorySpaceIds(this.normalizeInventory(job.inventory, job.spaceId)).includes( + spaceId + ) + ) { + throw new CustomHttpException( + `Migration job ${jobId} was not found for space ${spaceId}`, + HttpErrorCode.NOT_FOUND + ); + } + if (job.state === 'rolled_back') { + return await this.getMigrationJobStatus(spaceId, jobId); + } + if (job.state !== 'succeeded') { + throw new CustomHttpException( + 'Only completed space data database migrations can be rolled back', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbMigrationCancelConflictErrorCode, + migrationJobId: jobId, + migrationState: job.state, + spaceId, + } + ); + } + if (job.switchOnCompletion !== true) { + throw new CustomHttpException( + 'Only migrations that switched the space data database can be rolled back', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbRollbackUnsafeErrorCode, + migrationJobId: jobId, + migrationState: job.state, + spaceId, + } + ); + } + if (!job.completedAt) { + throw new CustomHttpException( + `Migration job ${jobId} has no switch timestamp`, + HttpErrorCode.VALIDATION_ERROR + ); + } + this.assertRollbackSourceSupported(job); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { state: 'switching', lastError: null }, + }); + + let restoredBeforeThrow = false; + let rollbackCompleted = false; + try { + const rollbackProof = await this.inspectPostSwitchRollbackProof(job); + if (!rollbackProof.eligible) { + const lastError = 'Space data database migration rollback is unsafe after target writes'; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'succeeded', + lastError, + validationStats: this.mergeValidationStats(job.validationStats, { + rollback: rollbackProof, + }), + }, + }); + restoredBeforeThrow = true; + throw new CustomHttpException(lastError, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbRollbackUnsafeErrorCode, + migrationJobId: jobId, + spaceId, + rollback: rollbackProof, + }); + } + const sourceDataDb = this.getSourceDataDbFromInventory(job); + const spaceIds = this.getInventoryCopySpaceIds( + this.normalizeInventory(job.inventory, job.spaceId) + ); + + const runTransaction = this.prismaService.$tx.bind(this.prismaService) as unknown as ( + fn: (prisma: IPrismaTransactionClient) => Promise + ) => Promise; + await runTransaction(async (prisma) => { + for (const relatedSpaceId of spaceIds) { + await prisma.spaceDataDbBinding.upsert({ + where: { spaceId: relatedSpaceId }, + create: { + spaceId: relatedSpaceId, + dataDbConnectionId: job.sourceConnectionId, + mode: job.sourceConnectionId ? 'byodb' : 'default', + state: 'ready', + createdBy: rolledBackBy || job.createdBy, + }, + update: { + dataDbConnectionId: job.sourceConnectionId, + mode: job.sourceConnectionId ? 'byodb' : 'default', + state: 'ready', + }, + }); + } + await prisma.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'rolled_back', + completedAt: new Date(), + lastError: null, + validationStats: this.mergeValidationStats(job.validationStats, { + rollback: { + ...rollbackProof, + rolledBackBy, + rolledBackAt: new Date().toISOString(), + }, + }), + }, + }); + }); + + if (job.targetConnectionId) { + await this.dataDbClientManager.invalidateConnection(job.targetConnectionId); + } + if (job.sourceConnectionId) { + await this.dataDbClientManager.invalidateConnection(job.sourceConnectionId); + } + await this.resumeOriginalSourceComputedPause(job, sourceDataDb); + rollbackCompleted = true; + return await this.getMigrationJobStatus(spaceId, jobId); + } catch (error) { + if (!rollbackCompleted && !restoredBeforeThrow) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'succeeded', + lastError: error instanceof Error ? error.message : String(error), + }, + }); + } + throw error; + } + } + + private assertRollbackSourceSupported(job: IMigrationJobRecord) { + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + if (inventory.sourceDataDb.mode === 'default' && !inventory.sourceDataDb.connectionId) { + return; + } + + throw new CustomHttpException( + 'Rolling back migrations from a BYODB source is not supported yet', + HttpErrorCode.VALIDATION_ERROR + ); + } + + private getSourceDataDbFromInventory(job: IMigrationJobRecord): IResolvedDataDatabase { + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + if (inventory.sourceDataDb.mode === 'default' && !inventory.sourceDataDb.connectionId) { + return { + cacheKey: metaFallbackDataDbCacheKey, + url: getMetaDatabaseUrl(), + isMetaFallback: true, + }; + } + + this.assertRollbackSourceSupported(job); + throw new Error('Unsupported rollback source data database'); + } + + private async getSourceDataDbForJob(job: IMigrationJobRecord): Promise { + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + return await this.dataDbClientManager.getDataDatabaseForSpace(job.spaceId, { + sourceConnectionId: inventory.sourceDataDb.connectionId, + }); + } + + private async resumeOriginalSourceComputedPause( + job: IMigrationJobRecord, + sourceDataDb: IResolvedDataDatabase + ): Promise<{ deleted: number }> { + const client = this.clientFactory(sourceDataDb.url); + try { + return await this.deleteMigrationComputedPause( + client, + sourceDataDb.internalSchema ?? 'public', + this.getInventoryCopySpaceIds(this.normalizeInventory(job.inventory, job.spaceId)), + job.id + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + private async inspectPostSwitchRollbackProof( + job: IMigrationJobRecord + ): Promise { + const switchedAt = job.completedAt?.toISOString(); + const proof: IPostSwitchRollbackProof = { + eligible: false, + switchedAt: switchedAt ?? '', + checkedAt: new Date().toISOString(), + findings: [], + }; + + if (!switchedAt) { + proof.findings.push({ + object: `migration:${job.id}`, + reason: 'missing_switch_timestamp', + }); + return proof; + } + if (!job.targetConnection?.encryptedUrl) { + proof.findings.push({ + object: `migration:${job.id}`, + reason: 'missing_target_connection', + }); + return proof; + } + + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const targetClient = this.clientFactory(decryptDataDbUrl(job.targetConnection.encryptedUrl)); + + try { + await this.collectRollbackRelationFindings(targetClient, inventory, proof.findings); + await this.collectRollbackRowCountFindings( + targetClient, + job.targetInternalSchema, + inventory, + job.spaceId, + job.validationStats, + proof.findings + ); + await this.collectRollbackTimestampFindings( + targetClient, + job.targetInternalSchema, + inventory, + job.spaceId, + switchedAt, + proof.findings + ); + } catch (error) { + proof.findings.push({ + object: `migration:${job.id}`, + reason: `proof_query_failed:${error instanceof Error ? error.message : String(error)}`, + }); + } finally { + await targetClient.destroy().catch(() => undefined); + } + + return { + ...proof, + eligible: proof.findings.length === 0, + }; + } + + private async collectRollbackRelationFindings( + targetClient: IDataDbPreflightClient, + inventory: ISpaceDataDbInventory, + findings: IPostSwitchWriteFinding[] + ) { + const expectedKeys = this.buildPhysicalRelationKeys(inventory); + const actualKeys = this.buildPhysicalRelationKeysFromRelations( + await this.inspectPhysicalSchemasWithClient(targetClient, inventory.baseIds) + ); + + if (JSON.stringify(expectedKeys) === JSON.stringify(actualKeys)) { + return; + } + + findings.push({ + object: 'base:physicalRelations', + reason: 'relation_inventory_changed', + expectedCount: expectedKeys.length, + actualCount: actualKeys.length, + }); + } + + private async collectRollbackRowCountFindings( + targetClient: IDataDbPreflightClient, + targetSchema: string, + inventory: ISpaceDataDbInventory, + spaceId: string, + validationStats: unknown, + findings: IPostSwitchWriteFinding[] + ) { + const baseRows = this.getValidationRows(validationStats, 'baseSchemas'); + const sharedRows = this.getValidationRows(validationStats, 'sharedTables'); + + if (!baseRows || !sharedRows) { + findings.push({ + object: `migration:${spaceId}`, + reason: 'missing_validation_stats', + }); + return; + } + + for (const row of baseRows) { + const relation = this.parseBaseValidationObject(row.object); + if (!relation) { + findings.push({ object: row.object, reason: 'invalid_base_validation_object' }); + continue; + } + + const actualCount = await this.countRows( + targetClient, + relation.schemaName, + relation.relationName + ); + if (actualCount !== row.targetCount) { + findings.push({ + object: row.object, + reason: 'row_count_changed', + expectedCount: row.targetCount, + actualCount, + }); + } + } + + const sharedRowsByTable = new Map( + sharedRows.map((row) => [row.object.replace(/^shared:/, ''), row]) + ); + for (const plan of this.buildSharedTableCountPlans(inventory, spaceId)) { + const row = sharedRowsByTable.get(plan.table); + if (!row) { + findings.push({ + object: `shared:${plan.table}`, + reason: 'missing_shared_validation_row', + }); + continue; + } + + const actualCount = await this.countRows( + targetClient, + targetSchema, + plan.table, + plan.whereSql(targetSchema), + plan.bindings + ); + if (actualCount !== row.targetCount) { + findings.push({ + object: row.object, + reason: 'row_count_changed', + expectedCount: row.targetCount, + actualCount, + }); + } + } + } + + private async collectRollbackTimestampFindings( + targetClient: IDataDbPreflightClient, + targetSchema: string, + inventory: ISpaceDataDbInventory, + spaceId: string, + switchedAt: string, + findings: IPostSwitchWriteFinding[] + ) { + await this.collectRollbackBaseTimestampFindings(targetClient, inventory, switchedAt, findings); + await this.collectRollbackSharedTimestampFindings( + targetClient, + targetSchema, + inventory, + spaceId, + switchedAt, + findings + ); + } + + private async collectRollbackBaseTimestampFindings( + targetClient: IDataDbPreflightClient, + inventory: ISpaceDataDbInventory, + switchedAt: string, + findings: IPostSwitchWriteFinding[] + ) { + for (const schema of inventory.physicalSchemas) { + for (const relation of schema.relations) { + if (!relationKindsWithRows.has(relation.relationKind)) { + continue; + } + + const columns = await this.getExistingColumns( + targetClient, + relation.schemaName, + relation.relationName, + ['__created_time', '__last_modified_time'] + ); + if (!columns.length) { + continue; + } + + const count = await this.countRows( + targetClient, + relation.schemaName, + relation.relationName, + columns.map((column) => `${quoteIdent(column)} > ?::timestamp`).join(' OR '), + columns.map(() => switchedAt) + ); + if (count > 0) { + findings.push({ + object: `base:${relation.schemaName}.${relation.relationName}`, + reason: 'post_switch_timestamp_rows', + count, + }); + } + } + } + } + + private async collectRollbackSharedTimestampFindings( + targetClient: IDataDbPreflightClient, + targetSchema: string, + inventory: ISpaceDataDbInventory, + spaceId: string, + switchedAt: string, + findings: IPostSwitchWriteFinding[] + ) { + const plans = this.buildPostSwitchSharedWritePlans(inventory, spaceId, switchedAt); + + for (const plan of plans) { + const count = await this.countRows( + targetClient, + targetSchema, + plan.table, + plan.whereSql, + plan.bindings + ); + if (count > 0) { + findings.push({ + object: `shared:${plan.table}`, + reason: 'post_switch_timestamp_rows', + count, + }); + } + } + } + + private buildPostSwitchSharedWritePlans( + inventory: ISpaceDataDbInventory, + spaceId: string, + switchedAt: string + ) { + const plans: { table: string; whereSql: string; bindings: unknown[] }[] = []; + const spaceIds = this.getInventoryCopySpaceIds(inventory); + const pushTableScoped = (table: string) => { + if (!inventory.tableIds.length) { + return; + } + plans.push({ + table, + whereSql: `"created_time" > ?::timestamp AND "table_id" = ANY(?::text[])`, + bindings: [switchedAt, inventory.tableIds], + }); + }; + const pushBaseScoped = (table: string) => { + if (!inventory.baseIds.length) { + return; + } + plans.push({ + table, + whereSql: [ + `("created_at" > ?::timestamp OR "updated_at" > ?::timestamp)`, + `AND "base_id" = ANY(?::text[])`, + ].join(' '), + bindings: [switchedAt, switchedAt, inventory.baseIds], + }); + }; + + pushTableScoped(sharedTables.recordHistory); + pushTableScoped(sharedTables.tableTrash); + pushTableScoped(sharedTables.recordTrash); + pushBaseScoped(sharedTables.computedUpdateOutbox); + pushBaseScoped(sharedTables.computedUpdateDeadLetter); + + plans.push({ + table: sharedTables.computedUpdatePauseScope, + whereSql: [ + `("paused_at" > ?::timestamp OR "updated_at" > ?::timestamp)`, + `(`, + `("scope_type" = 'space' AND "scope_id" = ANY(?::text[]))`, + inventory.baseIds.length + ? `OR ("scope_type" = 'base' AND "scope_id" = ANY(?::text[]))` + : '', + inventory.tableIds.length + ? `OR ("scope_type" = 'table' AND "scope_id" = ANY(?::text[]))` + : '', + `)`, + ] + .filter(Boolean) + .join(' '), + bindings: [switchedAt, switchedAt, spaceIds, inventory.baseIds, inventory.tableIds].filter( + (value) => (Array.isArray(value) ? value.length > 0 : Boolean(value)) + ), + }); + + if (inventory.baseIds.length) { + plans.push({ + table: sharedTables.undoLog, + whereSql: [ + `"created_at" > ?::timestamp`, + `AND split_part("table_name", '.', 1) = ANY(?::text[])`, + ].join(' '), + bindings: [switchedAt, inventory.baseIds], + }); + } + + return plans; + } + + private getValidationRows( + validationStats: unknown, + key: 'baseSchemas' | 'sharedTables' + ): IRowCountValidation[] | null { + const stats = this.asRecord(validationStats); + const rows = stats?.[key]; + if (!Array.isArray(rows)) { + return null; + } + + return rows + .map((row) => this.asRecord(row)) + .filter((row): row is Record => Boolean(row)) + .map((row) => ({ + object: typeof row.object === 'string' ? row.object : '', + sourceCount: Number(row.sourceCount ?? 0), + targetCount: Number(row.targetCount ?? 0), + })) + .filter((row) => row.object); + } + + private parseBaseValidationObject(object: string) { + if (!object.startsWith('base:')) { + return null; + } + + const qualified = object.slice('base:'.length); + const separatorIndex = qualified.indexOf('.'); + if (separatorIndex <= 0 || separatorIndex === qualified.length - 1) { + return null; + } + + return { + schemaName: qualified.slice(0, separatorIndex), + relationName: qualified.slice(separatorIndex + 1), + }; + } + + private async getExistingColumns( + client: IDataDbPreflightClient, + schema: string, + table: string, + columns: string[] + ) { + const rows = normalizeRawRows<{ columnName: string }>( + await client.raw( + ` + SELECT column_name AS "columnName" + FROM information_schema.columns + WHERE table_schema = ? + AND table_name = ? + AND column_name = ANY(?::text[]) + `, + [schema, table, columns] + ) + ); + return rows.map((row) => row.columnName).filter((column) => columns.includes(column)); + } + + private buildPhysicalRelationKeysFromRelations(relations: ISpaceDataDbPhysicalRelation[]) { + return relations + .map((relation) => `${relation.schemaName}.${relation.relationName}:${relation.relationKind}`) + .sort(); + } + + private mergeValidationStats(validationStats: unknown, next: Record) { + return { + ...(this.asRecord(validationStats) ?? {}), + ...next, + }; + } + + async copySharedRowsForJob( + jobId: string, + options: { timeoutMs?: number; strategy?: ISpaceDataDbSharedTableCopyStrategy } + ) { + const job = await this.migrationJobClient.spaceDataDbMigrationJob.findUnique({ + where: { id: jobId }, + include: { targetConnection: true }, + }); + if (!job) { + throw new CustomHttpException(`Migration job ${jobId} not found`, HttpErrorCode.NOT_FOUND); + } + if (!job.targetConnection?.encryptedUrl) { + throw new CustomHttpException( + `Migration job ${jobId} has no target connection`, + HttpErrorCode.VALIDATION_ERROR + ); + } + + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const sourceDataDb = await this.getSourceDataDbForJob(job); + const targetUrl = decryptDataDbUrl(job.targetConnection.encryptedUrl); + const startedAt = new Date().toISOString(); + const strategy = options.strategy ?? 'psql_copy'; + const sourceSchema = sourceDataDb.internalSchema ?? 'public'; + const sharedPlanInput = { + sourceUrl: sourceDataDb.url, + targetUrl, + sourceSchema, + targetSchema: job.targetInternalSchema, + spaceId: job.spaceId, + spaceIds: this.getInventoryCopySpaceIds(inventory), + baseIds: inventory.baseIds, + tableIds: inventory.tableIds, + sharedTableIds: inventory.sharedTableIds, + }; + const fdwNamePrefix = this.buildPostgresFdwNamePrefix(jobId); + const plans = + strategy === 'postgres_fdw' + ? buildMigrationSharedTablePostgresFdwCopyPlans({ + ...sharedPlanInput, + fdwSchemaPrefix: `${fdwNamePrefix}_schema`, + serverNamePrefix: `${fdwNamePrefix}_server`, + }) + : buildMigrationSharedTablePsqlCopyPlans(sharedPlanInput); + const tableNames = plans.map((plan) => plan.table); + + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'copying', + copyStats: { + phase: 'copying_shared_rows', + progress: this.buildMigrationProgress(job, inventory, 'copying_shared_rows'), + sharedTables: { + tableNames, + totalTables: plans.length, + copiedTableCount: 0, + totalCopiedRows: 0, + strategy, + startedAt, + }, + }, + lastError: null, + }, + }); + + const copiedTables: ISharedTableCopySummary[] = []; + try { + const onTableCopied = async ( + result: ISpaceDataDbSharedTableCopyResult | ISpaceDataDbPostgresFdwSharedTableCopyResult + ) => { + copiedTables.push(this.buildSharedTableCopySummary(result)); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'copying', + copyStats: { + phase: 'copying_shared_rows', + progress: this.buildMigrationProgress(job, inventory, 'copying_shared_rows'), + sharedTables: { + tableNames, + totalTables: plans.length, + copiedTableCount: copiedTables.length, + totalCopiedRows: this.sumCopiedRows(copiedTables), + copiedTables, + strategy, + startedAt, + updatedAt: new Date().toISOString(), + }, + }, + lastError: null, + }, + }); + }; + const processOptions = this.buildCancelableProcessOptions(jobId, options.timeoutMs); + const processOptionsWithHeartbeat = { + ...processOptions, + onPoll: async () => { + await processOptions.onPoll?.(); + await this.updateSharedTableCopyHeartbeat(job, inventory, { + stage: 'copying_shared_rows', + tableNames, + totalTables: plans.length, + copiedTableCount: copiedTables.length, + totalCopiedRows: this.sumCopiedRows(copiedTables), + strategy, + updatedAt: new Date().toISOString(), + }); + }, + }; + const results = + strategy === 'postgres_fdw' + ? await this.copyService.copySharedTablesViaPostgresFdw( + plans as ReturnType, + processOptionsWithHeartbeat, + { onTableCopied } + ) + : await this.copyService.copySharedTables( + plans as ReturnType, + processOptionsWithHeartbeat, + { onTableCopied } + ); + const sharedTables = results.map((result) => this.buildSharedTableCopySummary(result)); + const copyStats = { + phase: 'shared_rows_completed', + progress: this.buildMigrationProgress(job, inventory, 'shared_rows_completed'), + sharedTables: { + tableNames, + totalTables: plans.length, + copiedTableCount: sharedTables.length, + totalCopiedRows: this.sumCopiedRows(sharedTables), + copiedTables: sharedTables, + strategy, + startedAt, + completedAt: new Date().toISOString(), + }, + }; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'copying', + copyStats, + lastError: null, + }, + }); + return copyStats; + } catch (error) { + if (await this.isProcessCancelErrorForJob(error, jobId)) { + throw error; + } + const lastError = error instanceof Error ? error.message : String(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + copyStats: { + phase: 'shared_rows_failed', + progress: this.buildMigrationProgress(job, inventory, 'shared_rows_failed'), + sharedTables: { + tableNames, + totalTables: plans.length, + copiedTableCount: copiedTables.length, + totalCopiedRows: this.sumCopiedRows(copiedTables), + copiedTables, + strategy, + startedAt, + failedAt: new Date().toISOString(), + error: lastError, + failure: this.buildProcessFailureStats(error), + }, + }, + }, + }); + throw error; + } + } + + private buildSharedTableCopySummary( + result: ISpaceDataDbSharedTableCopyResult | ISpaceDataDbPostgresFdwSharedTableCopyResult + ): ISharedTableCopySummary { + return { + strategy: result.strategy, + table: result.table, + copiedRows: result.copiedRows, + ...('source' in result ? { source: result.source } : {}), + target: result.target, + }; + } + + private async updateSharedTableCopyHeartbeat( + job: IMigrationJobRecord, + inventory: ISpaceDataDbInventory, + heartbeat: ISharedTableCopyHeartbeat + ) { + await this.migrationJobClient.spaceDataDbMigrationJob + .update({ + where: { id: job.id }, + data: { + state: 'copying', + copyStats: { + phase: 'copying_shared_rows', + progress: this.buildMigrationProgress(job, inventory, 'copying_shared_rows'), + sharedTables: heartbeat, + }, + lastError: null, + }, + }) + .catch(() => undefined); + } + + private buildPostgresFdwNamePrefix(jobId: string) { + const sanitized = jobId.replace(/\W/g, '_').slice(0, 24); + return `teable_${sanitized}_fdw`; + } + + private sumCopiedRows(results: { copiedRows: number | null }[]) { + if (results.some((result) => result.copiedRows == null)) { + return null; + } + return results.reduce((sum, result) => sum + (result.copiedRows ?? 0), 0); + } + + private buildProcessFailureStats(error: unknown): ISpaceDataDbProcessFailureStats { + if (error instanceof SpaceDataDbProcessPipelineError) { + return { + type: 'pipeline', + message: error.message, + result: error.result, + }; + } + if (error instanceof SpaceDataDbProcessError) { + return { + type: 'process', + message: error.message, + result: error.result, + }; + } + return { + type: 'unknown', + message: error instanceof Error ? error.message : String(error), + }; + } + + private buildProcessFailureMessage(error: unknown) { + const baseMessage = error instanceof Error ? error.message : String(error); + const failureStats = this.buildProcessFailureStats(error); + const detail = this.getProcessFailureDetail(failureStats); + return detail ? `${baseMessage}: ${detail}` : baseMessage; + } + + private getProcessFailureDetail(failureStats: ISpaceDataDbProcessFailureStats) { + if (failureStats.type === 'process') { + return this.formatProcessFailureResult(failureStats.result); + } + + if (failureStats.type === 'pipeline') { + const source = this.formatProcessFailureResult(failureStats.result.source); + const target = this.formatProcessFailureResult(failureStats.result.target); + return [source ? `source ${source}` : '', target ? `target ${target}` : ''] + .filter(Boolean) + .join('; '); + } + + return ''; + } + + private formatProcessFailureResult(result: ISpaceDataDbProcessFailureResult) { + const exitOrSignal = + result.exitCode != null + ? `exit ${result.exitCode}` + : result.signal + ? `signal ${result.signal}` + : ''; + const output = (result.stderr || result.stdout || '').trim(); + const outputTail = output.length > 800 ? output.slice(output.length - 800) : output; + return [result.command, exitOrSignal, outputTail].filter(Boolean).join(' - '); + } + + async validateAndSwitchJob(jobId: string) { + const validationStats = await this.validateCopyForJob(jobId); + const job = await this.getMigrationJob(jobId); + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const spaceIds = this.getInventorySpaceIds(inventory); + if (!job.targetConnectionId) { + throw new CustomHttpException( + `Migration job ${jobId} has no target connection`, + HttpErrorCode.VALIDATION_ERROR + ); + } + + const switchOnCompletion = job.switchOnCompletion === true; + let completedValidationStats: IValidationStats = { + ...validationStats, + switchOnCompletion, + switched: false, + }; + try { + await this.resumeTargetComputedForJob(jobId); + const runTransaction = this.prismaService.$tx.bind(this.prismaService) as unknown as ( + fn: (prisma: IPrismaTransactionClient) => Promise + ) => Promise; + + if (!switchOnCompletion) { + await this.resumeSourceComputedForJob(jobId); + await runTransaction(async (prisma) => { + await prisma.dataDbConnection.update({ + where: { id: job.targetConnectionId }, + data: { + status: 'ready', + lastValidatedAt: new Date(), + lastError: null, + }, + }); + await prisma.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'succeeded', + validationStats: completedValidationStats, + completedAt: new Date(), + lastError: null, + }, + }); + }); + await this.dataDbClientManager.invalidateConnection(job.targetConnectionId); + return { + state: 'succeeded', + validationStats: completedValidationStats, + }; + } + + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'switching', + lastError: null, + }, + }); + + const switchedAt = new Date(); + completedValidationStats = { + ...completedValidationStats, + switched: true, + switchedAt: switchedAt.toISOString(), + }; + await runTransaction(async (prisma) => { + await prisma.dataDbConnection.update({ + where: { id: job.targetConnectionId }, + data: { + status: 'ready', + lastValidatedAt: new Date(), + lastError: null, + }, + }); + for (const spaceId of spaceIds) { + await prisma.spaceDataDbBinding.upsert({ + where: { spaceId }, + create: { + spaceId, + dataDbConnectionId: job.targetConnectionId, + mode: 'byodb', + state: 'ready', + createdBy: job.createdBy, + }, + update: { + dataDbConnectionId: job.targetConnectionId, + mode: 'byodb', + state: 'ready', + }, + }); + } + await prisma.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'succeeded', + validationStats: completedValidationStats, + completedAt: switchedAt, + lastError: null, + }, + }); + }); + } catch (error) { + const lastError = error instanceof Error ? error.message : String(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + lastError, + }, + }); + throw error; + } + + await this.dataDbClientManager.invalidateConnection(job.targetConnectionId); + if (job.sourceConnectionId) { + await this.dataDbClientManager.invalidateConnection(job.sourceConnectionId); + } + return { + state: 'succeeded', + validationStats: completedValidationStats, + }; + } + + async validateCopyForJob(jobId: string): Promise { + const job = await this.getMigrationJob(jobId); + if (!job.targetConnection?.encryptedUrl) { + throw new CustomHttpException( + `Migration job ${jobId} has no target connection`, + HttpErrorCode.VALIDATION_ERROR + ); + } + + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const sourceDataDb = await this.getSourceDataDbForJob(job); + const sourceSchema = sourceDataDb.internalSchema ?? 'public'; + const targetConnection = job.targetConnection; + const targetUrl = decryptDataDbUrl(targetConnection.encryptedUrl); + const startedAt = new Date().toISOString(); + const progressPollMs = readPositiveIntEnv( + 'BYODB_SPACE_DATA_DB_COPY_PROGRESS_POLL_MS', + defaultCopyProgressPollMs + ); + + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'validating', + validationStats: { + phase: 'validating_copy', + progress: this.buildMigrationProgress(job, inventory, 'validating_copy'), + startedAt, + }, + lastError: null, + }, + }); + + const sourceClient = this.clientFactory(sourceDataDb.url); + const targetClient = this.clientFactory(targetUrl); + try { + const validation = await this.withMigrationHeartbeat( + () => + this.updateValidationHeartbeat(job, inventory, { + stage: 'validating_copy', + updatedAt: new Date().toISOString(), + }), + () => + this.validateCopiedData({ + sourceClient, + targetClient, + sourceSchema, + targetSchema: job.targetInternalSchema, + targetConnectionId: job.targetConnectionId, + targetEncryptedUrl: targetConnection.encryptedUrl, + inventory, + spaceId: job.spaceId, + }), + progressPollMs + ); + + if (validation.mismatches.length) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + validationStats: { + phase: 'validation_failed', + progress: this.buildMigrationProgress(job, inventory, 'validation_failed'), + mismatches: validation.mismatches, + failedAt: new Date().toISOString(), + }, + lastError: validationFailedMessage, + }, + }); + throw new CustomHttpException(validationFailedMessage, HttpErrorCode.CONFLICT, { + errorCode: spaceDataDbValidationMismatchErrorCode, + mismatches: validation.mismatches, + spaceId: job.spaceId, + }); + } + + const validationStats: IValidationStats = { + phase: 'validation_completed', + progress: this.buildMigrationProgress(job, inventory, 'validation_completed'), + targetSchemaVersion: validation.targetSchemaVersion, + routeSmoke: validation.routeSmoke, + baseSchemas: validation.baseSchemas, + sharedTables: validation.sharedTables, + undoFunction: validation.undoFunction, + completedAt: new Date().toISOString(), + }; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'validating', + validationStats, + lastError: null, + }, + }); + return validationStats; + } catch (error) { + if (error instanceof CustomHttpException) { + throw error; + } + + const lastError = error instanceof Error ? error.message : String(error); + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + state: 'failed', + validationStats: { + phase: 'validation_failed', + progress: this.buildMigrationProgress(job, inventory, 'validation_failed'), + failedAt: new Date().toISOString(), + }, + lastError, + }, + }); + throw error; + } finally { + await Promise.all([ + sourceClient.destroy().catch(() => undefined), + targetClient.destroy().catch(() => undefined), + ]); + } + } + + private async updateValidationHeartbeat( + job: IMigrationJobRecord, + inventory: ISpaceDataDbInventory, + heartbeat: IValidationHeartbeat + ) { + await this.migrationJobClient.spaceDataDbMigrationJob + .update({ + where: { id: job.id }, + data: { + state: 'validating', + validationStats: { + phase: 'validating_copy', + progress: this.buildMigrationProgress(job, inventory, 'validating_copy'), + heartbeat, + }, + lastError: null, + }, + }) + .catch(() => undefined); + } + + private async assertNoActiveMigrationForSpaces(spaceIds: string[], requestedSpaceId: string) { + const directJob = await this.migrationJobClient.spaceDataDbMigrationJob.findFirst({ + where: { + spaceId: { in: spaceIds }, + state: { in: [...activeSpaceDataDbMigrationStates] }, + }, + select: { id: true, state: true }, + }); + const activeJob = + directJob ?? + ( + await this.migrationJobClient.spaceDataDbMigrationJob.findMany({ + where: { + state: { in: [...activeSpaceDataDbMigrationStates] }, + }, + include: { targetConnection: true }, + }) + ).find((job) => + this.getInventorySpaceIds(this.normalizeInventory(job.inventory, job.spaceId)).some( + (spaceId) => spaceIds.includes(spaceId) + ) + ); + if (!activeJob) { + return; + } + + throw new CustomHttpException( + 'A space data database migration is already active', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbMigrationActiveErrorCode, + migrationJobId: activeJob.id, + migrationState: activeJob.state, + spaceId: requestedSpaceId, + relatedSpaceIds: spaceIds, + } + ); + } + + private async assertMigrationNotCanceled(jobId: string, spaceId?: string) { + const job = await this.getMigrationState(jobId); + + if (job?.state !== 'canceled') { + return; + } + + throw new CustomHttpException( + `Space data database migration job ${jobId} was canceled`, + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbMigrationCanceledErrorCode, + migrationJobId: jobId, + spaceId: spaceId ?? job.spaceId, + } + ); + } + + private buildCancelableProcessOptions( + jobId: string, + timeoutMs?: number, + pollMs?: number + ): ISpaceDataDbProcessRunOptions { + return { + timeoutMs, + pollMs, + pollTimeoutMs: pollMs + ? readPositiveIntEnv( + 'BYODB_SPACE_DATA_DB_PROCESS_POLL_TIMEOUT_MS', + Math.max(defaultCopyProgressPollTimeoutMs, pollMs * 6) + ) + : undefined, + pollFailureTimeoutMs: pollMs + ? readPositiveIntEnv( + 'BYODB_SPACE_DATA_DB_PROCESS_POLL_FAILURE_TIMEOUT_MS', + Math.max(defaultCopyProgressPollTimeoutMs, pollMs * 6) + ) + : undefined, + shouldCancel: () => this.isMigrationCanceled(jobId), + }; + } + + private async isProcessCancelErrorForJob(error: unknown, jobId: string) { + if ( + error instanceof SpaceDataDbProcessCanceledError || + error instanceof SpaceDataDbProcessPipelineCanceledError + ) { + return true; + } + return await this.isMigrationCanceled(jobId); + } + + private async isMigrationCanceled(jobId: string) { + return (await this.getMigrationState(jobId))?.state === 'canceled'; + } + + private async getMigrationState(jobId: string) { + return (await this.migrationJobClient.spaceDataDbMigrationJob.findFirst({ + where: { id: jobId }, + select: { id: true, state: true, spaceId: true }, + })) as { id: string; state: string; spaceId?: string } | null; + } + + private async completeFailedMigrationJob(jobId: string, error: unknown) { + const current = await this.getMigrationState(jobId); + if (!current || ['succeeded', 'canceled', 'rolled_back'].includes(current.state)) { + return; + } + if ( + current.state !== 'failed' && + !activeSpaceDataDbMigrationStates.includes(current.state as never) + ) { + return; + } + + await this.migrationJobClient.spaceDataDbMigrationJob.updateMany({ + where: { id: jobId, state: current.state }, + data: { + state: 'failed', + completedAt: new Date(), + lastError: error instanceof Error ? error.message : String(error), + }, + }); + } + + private assertSpacesCanJoinTargetDataDb( + relatedSpaces: ISpaceDataDbRelatedSpaces, + targetDatabaseFingerprint: string, + requestedSpaceId: string + ) { + const mismatched = relatedSpaces.spaces.filter( + (space) => + space.dataDbMode === 'byodb' && + space.dataDbDatabaseFingerprint !== targetDatabaseFingerprint + ); + if (!mismatched.length) { + return; + } + + throw new CustomHttpException( + 'Cross-space linked spaces must use the same BYODB data database URL', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbRelatedSpacesRequiredErrorCode, + spaceId: requestedSpaceId, + targetDatabaseFingerprint, + relatedSpaces, + mismatchedSpaceIds: mismatched.map((space) => space.spaceId), + } + ); + } + + private assertRelatedSpacesCanMigrateTogether( + spaceId: string, + relatedSpaces: ISpaceDataDbRelatedSpaces + ) { + const primary = relatedSpaces.spaces.find((space) => space.spaceId === spaceId); + if (primary) { + return; + } + + throw new CustomHttpException(`Space ${spaceId} not found`, HttpErrorCode.NOT_FOUND); + } + + private getRelatedSpaceIds(relatedSpaces: ISpaceDataDbRelatedSpaces) { + return relatedSpaces.spaces.map((space) => space.spaceId).sort(); + } + + private getInventorySpaceIds(inventory: ISpaceDataDbInventory) { + return inventory.spaceIds.length + ? inventory.spaceIds + : [inventory.relatedSpaces.primarySpaceId]; + } + + private getInventoryCopySpaceIds(inventory: ISpaceDataDbInventory) { + return inventory.copySpaceIds.length + ? inventory.copySpaceIds + : this.getInventorySpaceIds(inventory); + } + + private getRelatedSpaceIdsForCopy( + relatedSpaces: ISpaceDataDbRelatedSpaces, + targetUrlFingerprint: string + ) { + return relatedSpaces.spaces + .filter( + (space) => + space.dataDbMode !== 'byodb' || space.dataDbUrlFingerprint !== targetUrlFingerprint + ) + .map((space) => space.spaceId) + .sort(); + } + + private assertRequestedSpaceIsCopySource(spaceId: string, copySpaceIds: string[]) { + if (copySpaceIds.includes(spaceId)) { + return; + } + + throw new CustomHttpException( + 'Start migration from a related space that still uses the default data database', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbRelatedSpacesRequiredErrorCode, + spaceId, + copySpaceIds, + } + ); + } + + private resolveMigrationTargetInternalSchema( + dataDb: IDataDbPreflightRo, + relatedSpaces: ISpaceDataDbRelatedSpaces, + targetDatabaseFingerprint: string, + requestedSpaceId: string + ) { + const requested = resolveDataDbInternalSchema(dataDb.internalSchema, dataDb.url); + if (dataDb.internalSchema?.trim()) { + return requested; + } + + const relatedTargetSchemas = Array.from( + new Set( + relatedSpaces.spaces + .filter( + (space) => + space.dataDbMode === 'byodb' && + space.dataDbDatabaseFingerprint === targetDatabaseFingerprint && + space.dataDbInternalSchema + ) + .map((space) => space.dataDbInternalSchema as string) + ) + ).sort(); + + if (relatedTargetSchemas.length <= 1) { + return relatedTargetSchemas[0] ?? requested; + } + + throw new CustomHttpException( + 'Cross-space linked spaces must use a single BYODB internal schema before repair migration', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbRelatedSpacesRequiredErrorCode, + spaceId: requestedSpaceId, + targetDatabaseFingerprint, + relatedSpaces, + internalSchemas: relatedTargetSchemas, + } + ); + } + + private async resolveMigrationTargetInternalSchemaOrThrow( + dataDb: IDataDbPreflightRo, + relatedSpaces: ISpaceDataDbRelatedSpaces, + targetDatabaseFingerprint: string, + requestedSpaceId: string + ) { + try { + return this.resolveMigrationTargetInternalSchema( + dataDb, + relatedSpaces, + targetDatabaseFingerprint, + requestedSpaceId + ); + } catch (error) { + if (error instanceof CustomHttpException) { + throw error; + } + const preflight = await this.preflightService.preflight({ + ...dataDb, + targetMode: migrateSpaceTargetMode, + }); + if (!preflight.ok) { + throw this.buildMigrationPreflightException(preflight); + } + throw new CustomHttpException( + this.sanitizeTargetErrorMessage(error, dataDb.url), + HttpErrorCode.VALIDATION_ERROR, + { preflight } + ); + } + } + + private async runTargetPreflightOperation( + dataDb: IDataDbPreflightRo, + internalSchema: string, + spaceId: string, + operation: string, + action: () => Promise + ): Promise { + try { + return await action(); + } catch (error) { + if (error instanceof CustomHttpException) { + throw error; + } + const preflight = await this.preflightService.preflight({ + ...dataDb, + targetMode: migrateSpaceTargetMode, + internalSchema, + }); + if (!preflight.ok) { + throw this.buildMigrationPreflightException(preflight); + } + throw new CustomHttpException( + 'Target BYODB database operation failed before migration could be queued', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbTargetConflictErrorCode, + operation, + spaceId, + internalSchema, + targetError: this.sanitizeTargetErrorMessage(error, dataDb.url), + preflight, + } + ); + } + } + + private buildMigrationPreflightException(preflight: IDataDbPreflightVo) { + return new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { + preflight, + }); + } + + private sanitizeTargetErrorMessage(error: unknown, rawUrl: string) { + const message = error instanceof Error ? error.message : String(error); + return message + .replace(rawUrl, '[redacted]') + .replace(/postgresql:\/\/[^@\s]+@/g, 'postgresql://***@') + .slice(0, 1000); + } + + private assertRelatedSpacesShareSameTarget( + relatedSpaces: ISpaceDataDbRelatedSpaces, + targetUrlFingerprint: string, + requestedSpaceId: string + ) { + const mismatched = relatedSpaces.spaces.filter( + (space) => space.dataDbMode === 'byodb' && space.dataDbUrlFingerprint !== targetUrlFingerprint + ); + if (!mismatched.length) { + return; + } + + throw new CustomHttpException( + 'Cross-space linked spaces must use the same BYODB data database URL', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbRelatedSpacesRequiredErrorCode, + spaceId: requestedSpaceId, + targetUrlFingerprint, + relatedSpaces, + mismatchedSpaceIds: mismatched.map((space) => space.spaceId), + } + ); + } + + private async buildInventory( + spaceId: string, + targetInternalSchema: string, + relatedSpaces?: ISpaceDataDbRelatedSpaces, + options: { copySpaceIds?: string[] } = {} + ): Promise { + const resolvedRelatedSpaces = + relatedSpaces ?? (await resolveSpaceDataDbRelatedSpaces(this.prismaService, spaceId)); + this.assertRelatedSpacesCanMigrateTogether(spaceId, resolvedRelatedSpaces); + const spaceIds = this.getRelatedSpaceIds(resolvedRelatedSpaces); + const copySpaceIdSet = new Set(options.copySpaceIds?.length ? options.copySpaceIds : spaceIds); + const copySpaces = resolvedRelatedSpaces.spaces.filter((space) => + copySpaceIdSet.has(space.spaceId) + ); + const copySpaceIds = copySpaces.map((space) => space.spaceId).sort(); + const baseIds = copySpaces.flatMap((space) => space.baseIds).sort(); + const tableIds = copySpaces.flatMap((space) => space.tableIds).sort(); + const relatedBaseIds = resolvedRelatedSpaces.spaces.flatMap((space) => space.baseIds).sort(); + const [tables, sharedTableIds, relatedSharedTableIds] = await Promise.all([ + this.prismaService.tableMeta.findMany({ + where: { id: { in: tableIds }, deletedTime: null }, + select: { id: true, dbTableName: true }, + orderBy: { id: 'asc' }, + }), + this.findSharedTableIdsForBases(baseIds), + this.findSharedTableIdsForBases(relatedBaseIds), + ]); + + const sourceDataDb = await this.dataDbClientManager.getDataDatabaseForSpace(spaceId); + const dataPrisma = (await this.dataDbClientManager.dataPrismaForSpace( + spaceId + )) as IDataPrismaIntrospectionClient; + const [physicalSchemas, postgresExtensionDependencies, outOfScopeForeignKeys] = + await Promise.all([ + this.inspectSourcePhysicalSchemas(baseIds, dataPrisma), + this.inspectSourcePostgresExtensionDependencies(baseIds, dataPrisma), + this.inspectSourceOutOfScopeForeignKeys(baseIds, dataPrisma), + ]); + + return { + sourceDataDb: this.buildSourceDataDbInventory(sourceDataDb), + targetDataDb: { + internalSchema: targetInternalSchema, + }, + relatedSpaces: resolvedRelatedSpaces, + spaceIds, + copySpaceIds, + baseIds, + tableIds, + sharedTableIds, + relatedSharedTableIds, + dbTableNames: tables.map((table) => table.dbTableName).sort(), + physicalSchemas, + postgresExtensionDependencies, + outOfScopeForeignKeys, + estimatedTotalBytes: physicalSchemas.reduce((sum, schema) => sum + schema.totalBytes, 0), + estimatedTotalRows: physicalSchemas.reduce((sum, schema) => sum + schema.estimatedRows, 0), + }; + } + + private normalizeInventory( + inventory: unknown, + fallbackSpaceId: string = '' + ): ISpaceDataDbInventory { + const candidate = (inventory ?? {}) as Partial; + const sourceDataDb = this.normalizeInventorySourceDataDb(candidate.sourceDataDb); + const targetDataDb = this.normalizeInventoryTargetDataDb(candidate.targetDataDb); + const physicalSchemas = candidate.physicalSchemas ?? []; + const relatedSpaces = this.normalizeInventoryRelatedSpaces(candidate, fallbackSpaceId); + const spaceIds = this.normalizeInventorySpaceIds(candidate, relatedSpaces); + const copySpaceIds = candidate.copySpaceIds?.length ? candidate.copySpaceIds : spaceIds; + const tableIds = candidate.tableIds ?? []; + const sharedTableIds = candidate.sharedTableIds ?? tableIds; + const relatedSharedTableIds = this.normalizeInventoryRelatedSharedTableIds( + candidate, + relatedSpaces, + sharedTableIds + ); + return { + sourceDataDb: { + mode: sourceDataDb.mode ?? 'default', + cacheKey: sourceDataDb.cacheKey ?? metaFallbackDataDbCacheKey, + connectionId: sourceDataDb.connectionId ?? null, + internalSchema: sourceDataDb.internalSchema ?? null, + isMetaFallback: sourceDataDb.isMetaFallback ?? true, + }, + targetDataDb: { + internalSchema: targetDataDb.internalSchema, + }, + relatedSpaces, + spaceIds, + copySpaceIds, + baseIds: candidate.baseIds ?? [], + tableIds, + sharedTableIds, + relatedSharedTableIds, + dbTableNames: candidate.dbTableNames ?? [], + physicalSchemas, + postgresExtensionDependencies: candidate.postgresExtensionDependencies ?? [], + outOfScopeForeignKeys: candidate.outOfScopeForeignKeys ?? [], + estimatedTotalBytes: + candidate.estimatedTotalBytes ?? + physicalSchemas.reduce((sum, schema) => sum + (schema.totalBytes ?? 0), 0), + estimatedTotalRows: + candidate.estimatedTotalRows ?? + physicalSchemas.reduce((sum, schema) => sum + (schema.estimatedRows ?? 0), 0), + }; + } + + private normalizeInventorySourceDataDb(sourceDataDb: unknown): ISpaceDataDbInventorySourceDataDb { + const candidate = (sourceDataDb ?? {}) as Partial; + return { + mode: candidate.mode ?? 'default', + cacheKey: candidate.cacheKey ?? metaFallbackDataDbCacheKey, + connectionId: candidate.connectionId ?? null, + internalSchema: candidate.internalSchema ?? null, + isMetaFallback: candidate.isMetaFallback ?? true, + }; + } + + private normalizeInventoryTargetDataDb(targetDataDb: unknown): ISpaceDataDbInventoryTargetDataDb { + const candidate = (targetDataDb ?? {}) as Partial; + return { + internalSchema: candidate.internalSchema ?? '', + }; + } + + private normalizeInventoryRelatedSpaces( + candidate: Partial, + fallbackSpaceId: string + ): ISpaceDataDbInventoryRelatedSpaces { + const relatedSpaces = candidate.relatedSpaces as ISpaceDataDbInventoryRelatedSpaces | undefined; + if (relatedSpaces) { + return relatedSpaces; + } + + return { + primarySpaceId: fallbackSpaceId, + hasCrossSpaceLinks: false, + spaces: [], + links: [], + }; + } + + private normalizeInventorySpaceIds( + candidate: Partial, + relatedSpaces: ISpaceDataDbInventoryRelatedSpaces + ) { + if (candidate.spaceIds?.length) { + return candidate.spaceIds; + } + if (relatedSpaces.spaces.length) { + return relatedSpaces.spaces.map((space) => space.spaceId); + } + return relatedSpaces.primarySpaceId ? [relatedSpaces.primarySpaceId] : []; + } + + private normalizeInventoryRelatedSharedTableIds( + candidate: Partial, + relatedSpaces: ISpaceDataDbInventoryRelatedSpaces, + sharedTableIds: string[] + ) { + if (candidate.relatedSharedTableIds) { + return candidate.relatedSharedTableIds; + } + if (!relatedSpaces.spaces.length) { + return sharedTableIds; + } + return [...new Set(relatedSpaces.spaces.flatMap((space) => space.tableIds ?? []))].sort(); + } + + private buildMigrationProgress( + job: IMigrationJobRecord, + inventory: ISpaceDataDbInventory, + phase: string + ): IMigrationProgressStats { + const completedSteps = migrationProgressCompletedSteps[phase] ?? 0; + const percent = Math.round((completedSteps / migrationProgressTotalSteps) * 1000) / 10; + const baseCopyCompleted = + completedSteps >= migrationProgressCompletedSteps.base_schemas_completed; + const updatedAt = new Date(); + const startedAt = job.startedAt ?? null; + const elapsedMs = startedAt ? Math.max(0, updatedAt.getTime() - startedAt.getTime()) : null; + const etaMs = + elapsedMs != null && percent > 0 && percent < 100 + ? Math.round((elapsedMs * (100 - percent)) / percent) + : null; + + return { + phase, + totalSteps: migrationProgressTotalSteps, + completedSteps, + percent, + estimatedTotalBytes: inventory.estimatedTotalBytes, + completedEstimatedBytes: baseCopyCompleted ? inventory.estimatedTotalBytes : 0, + estimatedTotalRows: inventory.estimatedTotalRows, + completedEstimatedRows: baseCopyCompleted ? inventory.estimatedTotalRows : 0, + startedAt: this.toNullableIso(startedAt), + updatedAt: updatedAt.toISOString(), + etaMs, + }; + } + + private compareInventoryForCopy( + expected: ISpaceDataDbInventory, + actual: ISpaceDataDbInventory, + options: { compareSharedScope?: boolean } = {} + ): IInventoryChangedMismatch[] { + const mismatches: IInventoryChangedMismatch[] = []; + this.collectObjectMismatch( + mismatches, + 'sourceDataDb', + expected.sourceDataDb, + actual.sourceDataDb + ); + this.collectObjectMismatch( + mismatches, + 'targetDataDb', + expected.targetDataDb, + actual.targetDataDb + ); + this.collectListMismatch(mismatches, 'spaceIds', expected.spaceIds, actual.spaceIds); + this.collectListMismatch( + mismatches, + 'copySpaceIds', + expected.copySpaceIds, + actual.copySpaceIds + ); + if (expected.relatedSpaces.spaces.length || expected.relatedSpaces.links.length) { + this.collectObjectMismatch( + mismatches, + 'relatedSpaces', + expected.relatedSpaces, + actual.relatedSpaces + ); + } + this.collectListMismatch(mismatches, 'baseIds', expected.baseIds, actual.baseIds); + this.collectListMismatch(mismatches, 'tableIds', expected.tableIds, actual.tableIds); + if (options.compareSharedScope) { + this.collectListMismatch( + mismatches, + 'sharedTableIds', + expected.sharedTableIds, + actual.sharedTableIds + ); + this.collectListMismatch( + mismatches, + 'relatedSharedTableIds', + expected.relatedSharedTableIds, + actual.relatedSharedTableIds + ); + } + this.collectListMismatch( + mismatches, + 'dbTableNames', + expected.dbTableNames, + actual.dbTableNames + ); + this.collectListMismatch( + mismatches, + 'physicalRelations', + this.buildPhysicalRelationKeys(expected), + this.buildPhysicalRelationKeys(actual) + ); + this.collectObjectMismatch( + mismatches, + 'postgresExtensionDependencies', + expected.postgresExtensionDependencies, + actual.postgresExtensionDependencies + ); + this.collectObjectMismatch( + mismatches, + 'outOfScopeForeignKeys', + expected.outOfScopeForeignKeys, + actual.outOfScopeForeignKeys + ); + return mismatches; + } + + private collectObjectMismatch( + mismatches: IInventoryChangedMismatch[], + object: string, + expected: unknown, + actual: unknown + ) { + if (stableJsonStringify(expected) === stableJsonStringify(actual)) { + return; + } + mismatches.push({ + object, + reason: 'inventory_changed', + expected, + actual, + }); + } + + private collectListMismatch( + mismatches: IInventoryChangedMismatch[], + object: string, + expected: string[], + actual: string[] + ) { + const sortedExpected = [...expected].sort(); + const sortedActual = [...actual].sort(); + if (JSON.stringify(sortedExpected) === JSON.stringify(sortedActual)) { + return; + } + + const expectedSet = new Set(sortedExpected); + const actualSet = new Set(sortedActual); + mismatches.push({ + object, + reason: 'inventory_changed', + expectedCount: sortedExpected.length, + actualCount: sortedActual.length, + added: sortedActual.filter((value) => !expectedSet.has(value)).slice(0, 5), + removed: sortedExpected.filter((value) => !actualSet.has(value)).slice(0, 5), + }); + } + + private buildPhysicalRelationKeys(inventory: ISpaceDataDbInventory) { + return inventory.physicalSchemas + .flatMap((schema) => + schema.relations.map( + (relation) => `${relation.schemaName}.${relation.relationName}:${relation.relationKind}` + ) + ) + .sort(); + } + + private async findSharedTableIdsForBases(baseIds: string[]): Promise { + if (!baseIds.length) { + return []; + } + + const rows = await this.prismaService.tableMeta.findMany({ + where: { baseId: { in: baseIds } }, + select: { id: true }, + orderBy: { id: 'asc' }, + }); + + return [...new Set(rows.map((row) => row.id))].sort(); + } + + private buildSourceDataDbInventory( + sourceDataDb: IResolvedDataDatabase + ): ISpaceDataDbInventorySourceDataDb { + return { + mode: sourceDataDb.isMetaFallback ? 'default' : 'byodb', + cacheKey: sourceDataDb.cacheKey, + connectionId: sourceDataDb.connectionId ?? null, + internalSchema: sourceDataDb.internalSchema ?? null, + isMetaFallback: sourceDataDb.isMetaFallback, + }; + } + + private async inspectSourcePhysicalSchemas( + baseIds: string[], + dataPrisma: IDataPrismaIntrospectionClient + ): Promise { + if (!baseIds.length) { + return []; + } + + const rows = normalizeRawRows<{ + schemaName: string; + relationName: string; + relationKind: string; + totalBytes: string | number | bigint | null; + estimatedRows: string | number | bigint | null; + }>( + await dataPrisma.$queryRawUnsafe( + ` + SELECT + n.nspname AS "schemaName", + c.relname AS "relationName", + CASE c.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'partitioned_table' + WHEN 'S' THEN 'sequence' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized_view' + WHEN 'f' THEN 'foreign_table' + ELSE c.relkind::text + END AS "relationKind", + pg_total_relation_size(c.oid)::text AS "totalBytes", + CASE + WHEN c.relkind IN ('r', 'p', 'f') THEN GREATEST(c.reltuples, 0)::bigint::text + ELSE NULL + END AS "estimatedRows" + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = ANY($1::text[]) + AND c.relkind IN ('r', 'p', 'S', 'v', 'm', 'f') + ORDER BY n.nspname ASC, c.relname ASC + `, + baseIds + ) + ); + + const schemaMap = new Map(); + for (const row of rows) { + const schema = schemaMap.get(row.schemaName) ?? { + schemaName: row.schemaName, + relations: [], + totalBytes: 0, + estimatedRows: 0, + }; + const relation = { + schemaName: row.schemaName, + relationName: row.relationName, + relationKind: row.relationKind, + totalBytes: Number(row.totalBytes ?? 0), + estimatedRows: row.estimatedRows == null ? null : Number(row.estimatedRows), + }; + schema.relations.push(relation); + schema.totalBytes += relation.totalBytes; + schema.estimatedRows += relation.estimatedRows ?? 0; + schemaMap.set(row.schemaName, schema); + } + + return Array.from(schemaMap.values()).sort((left, right) => + left.schemaName.localeCompare(right.schemaName) + ); + } + + private async inspectSourcePostgresExtensionDependencies( + baseIds: string[], + dataPrisma: IDataPrismaIntrospectionClient + ): Promise { + if (!baseIds.length) { + return []; + } + + const rows = normalizeRawRows<{ + extensionName?: string; + objectType?: string; + schemaName?: string; + objectName?: string; + accessMethod?: string; + sourceSchemaName?: string; + sourceRelationName?: string; + sourceIndexName?: string; + }>( + await dataPrisma.$queryRawUnsafe( + ` + WITH source_indexes AS ( + SELECT + tn.nspname AS "sourceSchemaName", + tc.relname AS "sourceRelationName", + ic.relname AS "sourceIndexName", + pg_get_indexdef(i.indexrelid) AS "definition" + FROM pg_index i + JOIN pg_class tc ON tc.oid = i.indrelid + JOIN pg_namespace tn ON tn.oid = tc.relnamespace + JOIN pg_class ic ON ic.oid = i.indexrelid + WHERE tn.nspname = ANY($1::text[]) + ) + SELECT + 'pg_trgm' AS "extensionName", + 'operator_class' AS "objectType", + 'public' AS "schemaName", + 'gin_trgm_ops' AS "objectName", + 'gin' AS "accessMethod", + "sourceSchemaName", + "sourceRelationName", + "sourceIndexName" + FROM source_indexes + WHERE position('gin_trgm_ops' in "definition") > 0 + ORDER BY "sourceSchemaName" ASC, "sourceRelationName" ASC, "sourceIndexName" ASC + `, + baseIds + ) + ); + + const dependencyMap = new Map(); + for (const row of rows) { + if ( + row.extensionName !== 'pg_trgm' || + row.objectType !== 'operator_class' || + !row.schemaName || + !row.objectName || + !row.accessMethod + ) { + continue; + } + + const key = [ + row.extensionName, + row.objectType, + row.schemaName, + row.objectName, + row.accessMethod, + ].join(':'); + const dependency = dependencyMap.get(key) ?? { + extensionName: row.extensionName, + objectType: 'operator_class', + schemaName: row.schemaName, + objectName: row.objectName, + accessMethod: row.accessMethod, + sourceObjects: [], + }; + if (row.sourceSchemaName && row.sourceRelationName && row.sourceIndexName) { + dependency.sourceObjects.push( + `${row.sourceSchemaName}.${row.sourceRelationName}.${row.sourceIndexName}` + ); + } + dependencyMap.set(key, dependency); + } + + return Array.from(dependencyMap.values()) + .map((dependency) => ({ + ...dependency, + sourceObjects: [...dependency.sourceObjects].sort().slice(0, 20), + })) + .sort((left, right) => + buildPostgresExtensionDependencyKey(left).localeCompare( + buildPostgresExtensionDependencyKey(right) + ) + ); + } + + private async inspectSourceOutOfScopeForeignKeys( + baseIds: string[], + dataPrisma: IDataPrismaIntrospectionClient + ): Promise { + if (!baseIds.length) { + return []; + } + + const rows = normalizeRawRows>( + await dataPrisma.$queryRawUnsafe( + ` + SELECT + source_ns.nspname AS "schemaName", + source_rel.relname AS "tableName", + con.conname AS "constraintName", + referenced_ns.nspname AS "referencedSchemaName", + referenced_rel.relname AS "referencedTableName" + FROM pg_constraint con + JOIN pg_class source_rel ON source_rel.oid = con.conrelid + JOIN pg_namespace source_ns ON source_ns.oid = source_rel.relnamespace + JOIN pg_class referenced_rel ON referenced_rel.oid = con.confrelid + JOIN pg_namespace referenced_ns ON referenced_ns.oid = referenced_rel.relnamespace + WHERE con.contype = 'f' + AND source_ns.nspname = ANY($1::text[]) + AND referenced_ns.nspname <> ALL($1::text[]) + ORDER BY + source_ns.nspname ASC, + source_rel.relname ASC, + con.conname ASC, + referenced_ns.nspname ASC, + referenced_rel.relname ASC + `, + baseIds + ) + ); + return rows + .filter((row): row is ISpaceDataDbExcludedForeignKey => + Boolean( + row.schemaName && + row.tableName && + row.constraintName && + row.referencedSchemaName && + row.referencedTableName + ) + ) + .sort((left, right) => + [ + left.schemaName, + left.tableName, + left.constraintName, + left.referencedSchemaName, + left.referencedTableName, + ] + .join(':') + .localeCompare( + [ + right.schemaName, + right.tableName, + right.constraintName, + right.referencedSchemaName, + right.referencedTableName, + ].join(':') + ) + ); + } + + private async prepareMigrationTarget( + url: string, + internalSchema: string + ): Promise { + const preflight = await this.preflightService.preflight({ + url, + targetMode: migrateSpaceTargetMode, + internalSchema, + }); + if (!preflight.ok) { + throw new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { + preflight, + }); + } + + const schemaVersion = await this.baselineService.initialize(url, internalSchema); + const { displayHost, displayDatabase } = getDatabaseUrlDisplayParts(url); + return { + encryptedUrl: encryptDataDbUrl(url), + urlFingerprint: fingerprintDataDbConnection(url, internalSchema), + displayHost, + displayDatabase, + internalSchema, + schemaVersion, + capabilities: preflight.capabilities, + }; + } + + private async assertTargetHasNoSpaceConflicts( + url: string, + internalSchema: string, + inventory: ISpaceDataDbInventory, + spaceId: string + ) { + const client = this.clientFactory(url); + try { + const conflicts = await this.inspectTargetConflicts(client, internalSchema, inventory); + if (!conflicts.length) { + return; + } + + throw new CustomHttpException( + 'Target BYODB database already contains objects for this space migration', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbTargetConflictErrorCode, + conflicts, + spaceId, + } + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + private async assertTargetSupportsSourceDependencies( + url: string, + inventory: ISpaceDataDbInventory, + spaceId: string + ) { + const dependencies = inventory.postgresExtensionDependencies; + if (!dependencies.length) { + return; + } + + const client = this.clientFactory(url); + try { + const installErrors = await this.tryInstallTargetPostgresExtensionDependencies( + client, + dependencies + ); + const missingExtensions = await this.inspectMissingTargetExtensionDependencies( + client, + dependencies + ); + if (!missingExtensions.length) { + return; + } + + const errors = missingExtensions.map((dependency) => { + const installError = installErrors.find( + (error) => + error.extensionName === dependency.extensionName && + error.schemaName === dependency.schemaName + ); + const installErrorMessage = installError?.message + ? ` Automatic installation failed: ${installError.message}` + : ''; + return { + code: 'MISSING_POSTGRES_EXTENSION', + message: `Target database is missing ${dependency.schemaName}.${dependency.objectName}, required by source ${dependency.extensionName} indexes.${installErrorMessage}`, + remediation: + dependency.extensionName === 'pg_trgm' + ? `Allow the migration connection to create pg_trgm, or install it before migration: CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA ${dependency.schemaName};` + : `Install the ${dependency.extensionName} extension in the target database before migration.`, + }; + }); + + throw new CustomHttpException( + 'Target BYODB database is missing PostgreSQL extensions required by the source space', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbTargetExtensionMissingErrorCode, + errors, + missingExtensions, + requiredExtensions: dependencies, + spaceId, + } + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + private async tryInstallTargetPostgresExtensionDependencies( + client: IDataDbPreflightClient, + dependencies: ISpaceDataDbPostgresExtensionDependency[] + ) { + const errors: { extensionName: string; schemaName: string; message: string }[] = []; + for (const target of this.getTargetPostgresExtensionInstallTargets(dependencies)) { + for (const schemaName of target.schemaNames) { + const error = await this.tryInstallTargetPostgresExtension( + client, + target.extensionName, + schemaName + ); + if (!error) { + break; + } + errors.push(error); + } + } + + return errors; + } + + private getTargetPostgresExtensionInstallTargets( + dependencies: ISpaceDataDbPostgresExtensionDependency[] + ) { + const installTargets = new Map(); + for (const dependency of dependencies) { + if (dependency.extensionName !== 'pg_trgm') { + continue; + } + installTargets.set(dependency.extensionName, { + extensionName: dependency.extensionName, + schemaNames: [...new Set([dependency.schemaName, 'extensions'].filter(Boolean))], + }); + } + return [...installTargets.values()]; + } + + private async tryInstallTargetPostgresExtension( + client: IDataDbPreflightClient, + extensionName: string, + schemaName: string + ) { + try { + await client.raw( + `CREATE EXTENSION IF NOT EXISTS ${quoteIdent(extensionName)} WITH SCHEMA ${quoteIdent(schemaName)}` + ); + return undefined; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + extensionName, + schemaName, + message: message.length > 500 ? message.slice(message.length - 500) : message, + }; + } + } + + private async inspectMissingTargetExtensionDependencies( + client: IDataDbPreflightClient, + dependencies: ISpaceDataDbPostgresExtensionDependency[] + ) { + const operatorClassDependencies = dependencies.filter( + (dependency) => dependency.objectType === 'operator_class' + ); + if (!operatorClassDependencies.length) { + return []; + } + + const schemaNames = [ + ...new Set(operatorClassDependencies.map((dependency) => dependency.schemaName)), + ]; + const objectNames = [ + ...new Set(operatorClassDependencies.map((dependency) => dependency.objectName)), + ]; + const accessMethods = [ + ...new Set(operatorClassDependencies.map((dependency) => dependency.accessMethod)), + ]; + const rows = normalizeRawRows<{ + schemaName: string; + objectName: string; + accessMethod: string; + }>( + await client.raw( + ` + SELECT + n.nspname AS "schemaName", + opc.opcname AS "objectName", + am.amname AS "accessMethod" + FROM pg_opclass opc + JOIN pg_namespace n ON n.oid = opc.opcnamespace + JOIN pg_am am ON am.oid = opc.opcmethod + WHERE (n.nspname = ANY(?::text[]) OR n.nspname = ANY(current_schemas(true))) + AND opc.opcname = ANY(?::text[]) + AND am.amname = ANY(?::text[]) + `, + [schemaNames, objectNames, accessMethods] + ) + ); + const available = new Set( + rows.map((row) => [row.schemaName, row.objectName, row.accessMethod].join(':')) + ); + const visible = new Set(rows.map((row) => [row.objectName, row.accessMethod].join(':'))); + + return operatorClassDependencies.filter( + (dependency) => + !available.has( + [dependency.schemaName, dependency.objectName, dependency.accessMethod].join(':') + ) && !visible.has([dependency.objectName, dependency.accessMethod].join(':')) + ); + } + + private async inspectTargetConflicts( + client: IDataDbPreflightClient, + internalSchema: string, + inventory: ISpaceDataDbInventory + ): Promise { + const conflicts: ITargetConflict[] = []; + const spaceIds = this.getInventoryCopySpaceIds(inventory); + + if (inventory.baseIds.length) { + const baseSchemaRows = normalizeRawRows<{ schemaName: string }>( + await client.raw( + ` + SELECT schema_name AS "schemaName" + FROM information_schema.schemata + WHERE schema_name = ANY(?::text[]) + `, + [inventory.baseIds] + ) + ); + conflicts.push(...baseSchemaRows.map((row) => ({ object: `schema:${row.schemaName}` }))); + } + + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.recordHistory, + inventory.sharedTableIds.length ? `"table_id" = ANY(?::text[])` : '', + [inventory.sharedTableIds] + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.tableTrash, + inventory.sharedTableIds.length ? `"table_id" = ANY(?::text[])` : '', + [inventory.sharedTableIds] + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.recordTrash, + inventory.sharedTableIds.length ? `"table_id" = ANY(?::text[])` : '', + [inventory.sharedTableIds] + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.computedUpdateOutbox, + inventory.baseIds.length ? `"base_id" = ANY(?::text[])` : '', + [inventory.baseIds] + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.computedUpdateDeadLetter, + inventory.baseIds.length ? `"base_id" = ANY(?::text[])` : '', + [inventory.baseIds] + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.computedUpdateOutboxSeed, + inventory.tableIds.length ? `"table_id" = ANY(?::text[])` : '', + [inventory.tableIds] + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.computedUpdatePauseScope, + [ + `("scope_type" = 'space' AND "scope_id" = ANY(?::text[]))`, + inventory.baseIds.length ? `("scope_type" = 'base' AND "scope_id" = ANY(?::text[]))` : '', + inventory.sharedTableIds.length + ? `("scope_type" = 'table' AND "scope_id" = ANY(?::text[]))` + : '', + ] + .filter(Boolean) + .join(' OR '), + [spaceIds, inventory.baseIds, inventory.sharedTableIds].filter((value) => + Array.isArray(value) ? value.length > 0 : Boolean(value) + ) + ); + await this.pushConflictCount( + client, + conflicts, + internalSchema, + sharedTables.undoLog, + inventory.baseIds.length ? `split_part("table_name", '.', 1) = ANY(?::text[])` : '', + [inventory.baseIds] + ); + + return conflicts; + } + + private async pushConflictCount( + client: IDataDbPreflightClient, + conflicts: ITargetConflict[], + internalSchema: string, + table: string, + whereSql: string, + bindings: unknown[] + ) { + if (!whereSql) { + return; + } + + const qualifiedTable = qualify(internalSchema, table); + const existsRows = normalizeRawRows<{ exists: boolean }>( + await client.raw(`SELECT to_regclass(?::text) IS NOT NULL AS "exists"`, [qualifiedTable]) + ); + if (!existsRows[0]?.exists) { + return; + } + + const countRows = normalizeRawRows<{ count: string | number | bigint }>( + await client.raw( + `SELECT COUNT(*) AS "count" FROM ${qualifiedTable} WHERE ${whereSql}`, + bindings + ) + ); + const count = Number(countRows[0]?.count ?? 0); + if (count > 0) { + conflicts.push({ object: `table:${internalSchema}.${table}`, count }); + } + } + + async cleanupTargetArtifactsForJob( + jobId: string, + reason: string = 'manual_cleanup', + options: { truncateSharedTables?: boolean } = {} + ): Promise { + const job = await this.migrationJobClient.spaceDataDbMigrationJob.findUnique({ + where: { id: jobId }, + include: { targetConnection: true }, + }); + if (!job) { + throw new CustomHttpException(`Migration job ${jobId} not found`, HttpErrorCode.NOT_FOUND); + } + if (!job.targetConnection?.encryptedUrl) { + throw new CustomHttpException( + `Migration job ${jobId} has no target connection`, + HttpErrorCode.VALIDATION_ERROR + ); + } + if (job.state === 'succeeded' && job.switchOnCompletion === true) { + throw new CustomHttpException( + `Migration job ${jobId} has already switched; automatic target cleanup is unsafe`, + HttpErrorCode.CONFLICT + ); + } + + const inventory = this.normalizeInventory(job.inventory, job.spaceId); + const targetUrl = decryptDataDbUrl(job.targetConnection.encryptedUrl); + const client = this.clientFactory(targetUrl); + const stats: ITargetArtifactCleanupStats = { + reason, + baseSchemas: [], + sharedTables: [], + truncateSharedTables: options.truncateSharedTables === true, + startedAt: new Date().toISOString(), + }; + + try { + stats.sharedTables = await this.cleanupTargetSharedRows( + client, + job.targetInternalSchema, + inventory, + job.spaceId, + { truncate: options.truncateSharedTables === true } + ); + stats.baseSchemas = await this.cleanupTargetBaseSchemas(client, inventory.baseIds); + stats.completedAt = new Date().toISOString(); + await this.updateTargetCleanupStats(jobId, job.copyStats, stats); + return stats; + } catch (error) { + stats.failedAt = new Date().toISOString(); + stats.error = error instanceof Error ? error.message : String(error); + await this.updateTargetCleanupStats(jobId, job.copyStats, stats).catch(() => undefined); + if (error instanceof CustomHttpException) { + throw error; + } + throw new CustomHttpException( + 'Target BYODB retry artifact cleanup failed before migration could be queued', + HttpErrorCode.CONFLICT, + { + errorCode: spaceDataDbTargetCleanupFailedErrorCode, + jobId, + reason, + spaceId: job.spaceId, + internalSchema: job.targetInternalSchema, + targetError: this.sanitizeTargetErrorMessage(error, targetUrl), + cleanup: stats, + } + ); + } finally { + await client.destroy().catch(() => undefined); + } + } + + private async cleanupTargetBaseSchemas( + client: IDataDbPreflightClient, + baseIds: string[] + ): Promise { + if (!baseIds.length) { + return []; + } + + const rows = normalizeRawRows<{ schemaName: string }>( + await client.raw( + ` + SELECT schema_name AS "schemaName" + FROM information_schema.schemata + WHERE schema_name = ANY(?::text[]) + ORDER BY schema_name ASC + `, + [baseIds] + ) + ); + const existing = new Set(rows.map((row) => row.schemaName)); + const stats: ITargetArtifactCleanupStats['baseSchemas'] = []; + + for (const schemaName of [...baseIds].sort()) { + if (existing.has(schemaName)) { + await client.raw(`DROP SCHEMA IF EXISTS ${quoteIdent(schemaName)} CASCADE`); + } + stats.push({ + schemaName, + dropped: existing.has(schemaName), + }); + } + + return stats; + } + + private async cleanupTargetSharedRows( + client: IDataDbPreflightClient, + targetSchema: string, + inventory: ISpaceDataDbInventory, + spaceId: string, + options: { truncate?: boolean } = {} + ): Promise { + const plans = this.buildSharedTableCleanupPlans(inventory, spaceId); + const stats: ITargetArtifactCleanupStats['sharedTables'] = []; + const existingPlans: typeof plans = []; + + for (const plan of plans) { + const qualifiedTable = qualify(targetSchema, plan.table); + const existsRows = normalizeRawRows<{ exists: boolean }>( + await client.raw(`SELECT to_regclass(?::text) IS NOT NULL AS "exists"`, [qualifiedTable]) + ); + if (!existsRows[0]?.exists) { + stats.push({ table: plan.table, deletedRows: null }); + continue; + } + + if (options.truncate) { + existingPlans.push(plan); + continue; + } + + const deletedRows = normalizeRawRows<{ count: string | number | bigint }>( + await client.raw( + ` + WITH deleted AS ( + DELETE FROM ${qualifiedTable} + WHERE ${plan.whereSql(targetSchema)} + RETURNING 1 + ) + SELECT COUNT(*) AS "count" FROM deleted + `, + plan.bindings + ) + ); + stats.push({ + table: plan.table, + deletedRows: Number(deletedRows[0]?.count ?? 0), + }); + } + + if (options.truncate && existingPlans.length) { + await client.raw( + `TRUNCATE TABLE ${existingPlans.map((plan) => qualify(targetSchema, plan.table)).join(', ')}` + ); + stats.push( + ...existingPlans.map((plan) => ({ + table: plan.table, + deletedRows: null, + truncated: true, + })) + ); + } + + return stats; + } + + private buildSharedTableCleanupPlans(inventory: ISpaceDataDbInventory, spaceId: string) { + const plans = this.buildSharedTableCountPlans(inventory, spaceId); + const priority = new Map([ + [sharedTables.computedUpdateOutboxSeed, 0], + [sharedTables.recordHistory, 1], + [sharedTables.tableTrash, 2], + [sharedTables.recordTrash, 3], + [sharedTables.computedUpdateDeadLetter, 4], + [sharedTables.computedUpdateOutbox, 5], + [sharedTables.computedUpdatePauseScope, 6], + [sharedTables.undoLog, 7], + ]); + return [...plans].sort((left, right) => { + const leftPriority = priority.get(left.table) ?? Number.MAX_SAFE_INTEGER; + const rightPriority = priority.get(right.table) ?? Number.MAX_SAFE_INTEGER; + return leftPriority - rightPriority || left.table.localeCompare(right.table); + }); + } + + private async updateTargetCleanupStats( + jobId: string, + copyStats: unknown, + cleanup: ITargetArtifactCleanupStats + ) { + const current = this.asRecord(copyStats) ?? {}; + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: jobId }, + data: { + copyStats: { + ...current, + targetCleanup: cleanup, + }, + }, + }); + } + + private async getMigrationJob(jobId: string): Promise { + const job = await this.migrationJobClient.spaceDataDbMigrationJob.findUnique({ + where: { id: jobId }, + include: { targetConnection: true }, + }); + if (!job) { + throw new CustomHttpException(`Migration job ${jobId} not found`, HttpErrorCode.NOT_FOUND); + } + return job; + } + + private async validateCopiedData(input: { + sourceClient: IDataDbPreflightClient; + targetClient: IDataDbPreflightClient; + sourceSchema: string; + targetSchema: string; + targetConnectionId: string | null; + targetEncryptedUrl: string; + inventory: ISpaceDataDbInventory; + spaceId: string; + }): Promise<{ + targetSchemaVersion: { latest: string | null; exists: boolean }; + routeSmoke: IRouteSmokeStats; + baseSchemas: IRowCountValidation[]; + sharedTables: IRowCountValidation[]; + undoFunction: { exists: boolean }; + mismatches: IValidationMismatch[]; + }> { + const targetSchemaVersion = await this.validateTargetSchemaVersion( + input.targetClient, + input.targetSchema + ); + const routeSmoke = await this.validateRouteSmoke({ + spaceId: input.spaceId, + targetConnectionId: input.targetConnectionId, + targetEncryptedUrl: input.targetEncryptedUrl, + targetSchema: input.targetSchema, + }); + const baseValidation = await this.validateBaseSchemaRows( + input.sourceClient, + input.targetClient, + input.inventory + ); + const sharedValidation = await this.validateSharedTableRows(input); + const undoFunction = await this.validateUndoFunction(input.targetClient, input.targetSchema); + const mismatches = [ + ...baseValidation.mismatches, + ...sharedValidation.mismatches, + ...(targetSchemaVersion.exists + ? [] + : [ + { + object: `schema:${input.targetSchema}.${dataDbMigrationTable}`, + reason: 'target_schema_version_mismatch', + }, + ]), + ...(routeSmoke.ok + ? [] + : [ + { + object: `route:${input.spaceId}`, + reason: 'target_route_smoke_failed', + }, + ]), + ...(undoFunction.exists + ? [] + : [ + { + object: `function:${input.targetSchema}.__teable_capture_undo_row`, + reason: 'missing_target_function', + }, + ]), + ]; + + return { + targetSchemaVersion, + routeSmoke, + baseSchemas: baseValidation.validatedRows, + sharedTables: sharedValidation.validatedRows, + undoFunction, + mismatches, + }; + } + + private async validateBaseSchemaRows( + sourceClient: IDataDbPreflightClient, + targetClient: IDataDbPreflightClient, + inventory: ISpaceDataDbInventory + ): Promise<{ validatedRows: IRowCountValidation[]; mismatches: IValidationMismatch[] }> { + const baseIds = inventory.baseIds; + if (!baseIds.length) { + return { validatedRows: [], mismatches: [] }; + } + + const sourceRelations = await this.inspectPhysicalSchemasWithClient(sourceClient, baseIds); + const targetRelations = await this.inspectPhysicalSchemasWithClient(targetClient, baseIds); + const sourceMap = this.relationMap(sourceRelations); + const targetMap = this.relationMap(targetRelations); + const mismatches: IValidationMismatch[] = []; + const validatedRows: IRowCountValidation[] = []; + + for (const [key, sourceRelation] of sourceMap) { + const relationMismatch = this.buildBaseRelationMismatch( + key, + sourceRelation, + targetMap.get(key) + ); + if (relationMismatch) { + mismatches.push(relationMismatch); + continue; + } + const targetRelation = targetMap.get(key)!; + const columnMismatch = await this.buildColumnSignatureMismatch( + sourceClient, + targetClient, + key, + sourceRelation, + targetRelation + ); + if (columnMismatch) { + mismatches.push(columnMismatch); + } + mismatches.push( + ...(await this.buildRelationDependencySignatureMismatches( + sourceClient, + targetClient, + key, + sourceRelation, + targetRelation, + baseIds + )) + ); + const rowValidation = await this.buildBaseRelationRowValidation( + sourceClient, + targetClient, + key, + sourceRelation, + targetRelation + ); + if (rowValidation) { + validatedRows.push(rowValidation.validatedRow); + if (rowValidation.mismatch) { + mismatches.push(rowValidation.mismatch); + } + } + } + + for (const [key, targetRelation] of targetMap) { + if (!sourceMap.has(key)) { + mismatches.push({ + object: `base:${key}`, + reason: 'extra_target_relation', + targetKind: targetRelation.relationKind, + }); + } + } + + return { validatedRows, mismatches }; + } + + private buildBaseRelationMismatch( + key: string, + sourceRelation: ISpaceDataDbPhysicalRelation, + targetRelation: ISpaceDataDbPhysicalRelation | undefined + ): IValidationMismatch | null { + if (!targetRelation) { + return { + object: `base:${key}`, + reason: 'missing_target_relation', + sourceKind: sourceRelation.relationKind, + targetKind: null, + }; + } + if (targetRelation.relationKind !== sourceRelation.relationKind) { + return { + object: `base:${key}`, + reason: 'relation_kind_mismatch', + sourceKind: sourceRelation.relationKind, + targetKind: targetRelation.relationKind, + }; + } + return null; + } + + private async buildColumnSignatureMismatch( + sourceClient: IDataDbPreflightClient, + targetClient: IDataDbPreflightClient, + key: string, + sourceRelation: ISpaceDataDbPhysicalRelation, + targetRelation: ISpaceDataDbPhysicalRelation + ): Promise { + if (!relationKindsWithColumns.has(sourceRelation.relationKind)) { + return null; + } + + const [sourceColumns, targetColumns] = await Promise.all([ + this.inspectRelationColumnSignature( + sourceClient, + sourceRelation.schemaName, + sourceRelation.relationName + ), + this.inspectRelationColumnSignature( + targetClient, + targetRelation.schemaName, + targetRelation.relationName + ), + ]); + return this.isColumnSignatureEqual(sourceColumns, targetColumns) + ? null + : { + object: `base:${key}`, + reason: 'column_signature_mismatch', + sourceColumns, + targetColumns, + }; + } + + private async buildBaseRelationRowValidation( + sourceClient: IDataDbPreflightClient, + targetClient: IDataDbPreflightClient, + key: string, + sourceRelation: ISpaceDataDbPhysicalRelation, + targetRelation: ISpaceDataDbPhysicalRelation + ): Promise<{ validatedRow: IRowCountValidation; mismatch: IValidationMismatch | null } | null> { + if (!relationKindsWithRows.has(sourceRelation.relationKind)) { + return null; + } + + const [sourceCount, targetCount] = await Promise.all([ + this.countRows(sourceClient, sourceRelation.schemaName, sourceRelation.relationName), + this.countRows(targetClient, targetRelation.schemaName, targetRelation.relationName), + ]); + const validatedRow = { + object: `base:${key}`, + sourceCount, + targetCount, + }; + return { + validatedRow, + mismatch: + sourceCount === targetCount + ? null + : { + ...validatedRow, + reason: 'row_count_mismatch', + }, + }; + } + + private async buildRelationDependencySignatureMismatches( + sourceClient: IDataDbPreflightClient, + targetClient: IDataDbPreflightClient, + key: string, + sourceRelation: ISpaceDataDbPhysicalRelation, + targetRelation: ISpaceDataDbPhysicalRelation, + baseIds: string[] + ): Promise { + const mismatches: IValidationMismatch[] = []; + + if (relationKindsWithIndexSignatures.has(sourceRelation.relationKind)) { + const [sourceIndexes, targetIndexes] = await Promise.all([ + this.inspectRelationIndexSignature( + sourceClient, + sourceRelation.schemaName, + sourceRelation.relationName + ), + this.inspectRelationIndexSignature( + targetClient, + targetRelation.schemaName, + targetRelation.relationName + ), + ]); + if (JSON.stringify(sourceIndexes) !== JSON.stringify(targetIndexes)) { + mismatches.push({ + object: `base:${key}`, + reason: 'index_signature_mismatch', + sourceIndexes, + targetIndexes, + }); + } + } + + if (relationKindsWithTableDependencySignatures.has(sourceRelation.relationKind)) { + const [sourceConstraints, targetConstraints, sourceTriggers, targetTriggers] = + await Promise.all([ + this.inspectRelationConstraintSignature( + sourceClient, + sourceRelation.schemaName, + sourceRelation.relationName, + baseIds + ), + this.inspectRelationConstraintSignature( + targetClient, + targetRelation.schemaName, + targetRelation.relationName, + baseIds + ), + this.inspectRelationTriggerSignature( + sourceClient, + sourceRelation.schemaName, + sourceRelation.relationName + ), + this.inspectRelationTriggerSignature( + targetClient, + targetRelation.schemaName, + targetRelation.relationName + ), + ]); + if (JSON.stringify(sourceConstraints) !== JSON.stringify(targetConstraints)) { + mismatches.push({ + object: `base:${key}`, + reason: 'constraint_signature_mismatch', + sourceConstraints, + targetConstraints, + }); + } + if (JSON.stringify(sourceTriggers) !== JSON.stringify(targetTriggers)) { + mismatches.push({ + object: `base:${key}`, + reason: 'trigger_signature_mismatch', + sourceTriggers, + targetTriggers, + }); + } + } + + return mismatches; + } + + private async validateSharedTableRows(input: { + sourceClient: IDataDbPreflightClient; + targetClient: IDataDbPreflightClient; + sourceSchema: string; + targetSchema: string; + inventory: ISpaceDataDbInventory; + spaceId: string; + }): Promise<{ validatedRows: IRowCountValidation[]; mismatches: IValidationMismatch[] }> { + const countPlans = this.buildSharedTableCountPlans(input.inventory, input.spaceId); + const relatedCountPlans = this.buildRelatedSharedTableCountPlans( + input.inventory, + input.spaceId + ); + const relatedPlanByTable = new Map(relatedCountPlans.map((plan) => [plan.table, plan])); + const validatedRows: IRowCountValidation[] = []; + const mismatches: IValidationMismatch[] = []; + + for (const plan of countPlans) { + const relatedPlan = relatedPlanByTable.get(plan.table) ?? plan; + const [sourceCount, targetCount] = await Promise.all([ + this.countRows( + input.sourceClient, + input.sourceSchema, + plan.table, + plan.whereSql(input.sourceSchema), + plan.bindings + ), + this.countRows( + input.targetClient, + input.targetSchema, + plan.table, + plan.whereSql(input.targetSchema), + plan.bindings + ), + ]); + const rowValidation = { + object: `shared:${plan.table}`, + sourceCount, + targetCount, + }; + validatedRows.push(rowValidation); + if (sourceCount !== targetCount) { + mismatches.push({ + ...rowValidation, + reason: 'row_count_mismatch', + }); + } + + const outOfScopeTargetCount = await this.countRows( + input.targetClient, + input.targetSchema, + plan.table, + `NOT (${relatedPlan.whereSql(input.targetSchema)})`, + relatedPlan.bindings + ); + if (outOfScopeTargetCount > 0) { + mismatches.push({ + object: `shared:${plan.table}`, + reason: 'out_of_scope_target_rows', + targetCount: outOfScopeTargetCount, + }); + } + } + + return { validatedRows, mismatches }; + } + + private buildSharedTableCountPlans(inventory: ISpaceDataDbInventory, spaceId: string) { + return this.buildSharedTableCountPlansForScope( + inventory, + spaceId, + this.getInventoryCopySpaceIds(inventory), + inventory.baseIds, + inventory.tableIds, + inventory.sharedTableIds + ); + } + + private buildRelatedSharedTableCountPlans(inventory: ISpaceDataDbInventory, spaceId: string) { + if (!inventory.relatedSpaces.spaces.length) { + return this.buildSharedTableCountPlansForScope( + inventory, + spaceId, + this.getInventorySpaceIds(inventory), + inventory.baseIds, + inventory.tableIds, + inventory.relatedSharedTableIds + ); + } + + const relatedBaseIds = this.getRelatedSpaceIds(inventory.relatedSpaces).flatMap( + (relatedSpaceId) => + inventory.relatedSpaces.spaces.find((space) => space.spaceId === relatedSpaceId)?.baseIds ?? + [] + ); + const relatedTableIds = this.getRelatedSpaceIds(inventory.relatedSpaces).flatMap( + (relatedSpaceId) => + inventory.relatedSpaces.spaces.find((space) => space.spaceId === relatedSpaceId) + ?.tableIds ?? [] + ); + return this.buildSharedTableCountPlansForScope( + inventory, + spaceId, + this.getInventorySpaceIds(inventory), + relatedBaseIds.sort(), + relatedTableIds.sort(), + inventory.relatedSharedTableIds + ); + } + + private buildSharedTableCountPlansForScope( + inventory: ISpaceDataDbInventory, + spaceId: string, + spaceIds: string[], + baseIds: string[], + tableIds: string[], + sharedTableIds: string[] = tableIds + ) { + const plans: { table: string; whereSql: (schema: string) => string; bindings: unknown[] }[] = + []; + const pushTableScoped = (table: string) => { + if (sharedTableIds.length) { + plans.push({ + table, + whereSql: () => `"table_id" = ANY(?::text[])`, + bindings: [sharedTableIds], + }); + } + }; + const pushBaseScoped = (table: string) => { + if (baseIds.length) { + plans.push({ + table, + whereSql: () => `"base_id" = ANY(?::text[])`, + bindings: [baseIds], + }); + } + }; + + pushTableScoped(sharedTables.recordHistory); + pushTableScoped(sharedTables.tableTrash); + pushTableScoped(sharedTables.recordTrash); + pushBaseScoped(sharedTables.computedUpdateOutbox); + pushBaseScoped(sharedTables.computedUpdateDeadLetter); + + if (tableIds.length && baseIds.length) { + plans.push({ + table: sharedTables.computedUpdateOutboxSeed, + whereSql: (schema) => + [ + `"table_id" = ANY(?::text[])`, + `"task_id" IN (SELECT "id" FROM ${qualify( + schema, + sharedTables.computedUpdateOutbox + )} WHERE "base_id" = ANY(?::text[]))`, + ].join(' AND '), + bindings: [tableIds, baseIds], + }); + } + + plans.push({ + table: sharedTables.computedUpdatePauseScope, + whereSql: () => + [ + `("scope_type" = 'space' AND "scope_id" = ANY(?::text[]))`, + baseIds.length ? `("scope_type" = 'base' AND "scope_id" = ANY(?::text[]))` : '', + sharedTableIds.length ? `("scope_type" = 'table' AND "scope_id" = ANY(?::text[]))` : '', + ] + .filter(Boolean) + .join(' OR '), + bindings: [spaceIds, baseIds, sharedTableIds].filter((value) => + Array.isArray(value) ? value.length > 0 : Boolean(value) + ), + }); + + if (baseIds.length) { + plans.push({ + table: sharedTables.undoLog, + whereSql: () => `split_part("table_name", '.', 1) = ANY(?::text[])`, + bindings: [baseIds], + }); + } + + return plans; + } + + private async inspectPhysicalSchemasWithClient( + client: IDataDbPreflightClient, + baseIds: string[] + ): Promise { + if (!baseIds.length) { + return []; + } + + return normalizeRawRows( + await client.raw( + ` + SELECT + n.nspname AS "schemaName", + c.relname AS "relationName", + CASE c.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'partitioned_table' + WHEN 'S' THEN 'sequence' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized_view' + WHEN 'f' THEN 'foreign_table' + ELSE c.relkind::text + END AS "relationKind", + pg_total_relation_size(c.oid)::text AS "totalBytes", + CASE + WHEN c.relkind IN ('r', 'p', 'f') THEN GREATEST(c.reltuples, 0)::bigint::text + ELSE NULL + END AS "estimatedRows" + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = ANY(?::text[]) + AND c.relkind IN ('r', 'p', 'S', 'v', 'm', 'f') + ORDER BY n.nspname ASC, c.relname ASC + `, + [baseIds] + ) + ); + } + + private relationMap(relations: ISpaceDataDbPhysicalRelation[]) { + return new Map(relations.map((relation) => [this.relationKey(relation), relation])); + } + + private relationKey(relation: ISpaceDataDbPhysicalRelation) { + return `${relation.schemaName}.${relation.relationName}`; + } + + private async inspectRelationColumnSignature( + client: IDataDbPreflightClient, + schema: string, + relation: string + ): Promise { + return normalizeRawRows( + await client.raw( + ` + SELECT + a.attnum::integer AS "ordinalPosition", + a.attname AS "columnName", + format_type(a.atttypid, a.atttypmod) AS "formattedType", + a.attnotnull AS "notNull", + pg_get_expr(ad.adbin, ad.adrelid) AS "defaultExpression", + a.attidentity AS "identity", + a.attgenerated AS "generated", + coll.collname AS "collation" + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum + LEFT JOIN pg_collation coll ON coll.oid = a.attcollation + AND a.attcollation <> 0 + WHERE n.nspname = ? + AND c.relname = ? + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum ASC + `, + [schema, relation] + ) + ); + } + + private isColumnSignatureEqual( + sourceColumns: IBaseRelationColumnSignature[], + targetColumns: IBaseRelationColumnSignature[] + ) { + return ( + JSON.stringify(this.columnSignatureForComparison(sourceColumns)) === + JSON.stringify(this.columnSignatureForComparison(targetColumns)) + ); + } + + private columnSignatureForComparison(columns: IBaseRelationColumnSignature[]) { + return columns.map(({ ordinalPosition: _ordinalPosition, ...column }) => column); + } + + private async inspectRelationIndexSignature( + client: IDataDbPreflightClient, + schema: string, + relation: string + ): Promise { + return normalizeRawRows( + await client.raw( + ` + SELECT + ic.relname AS "indexName", + i.indisprimary AS "isPrimary", + i.indisunique AS "isUnique", + i.indisvalid AS "isValid", + pg_get_indexdef(i.indexrelid) AS "definition" + FROM pg_index i + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_class ic ON ic.oid = i.indexrelid + WHERE n.nspname = ? + AND c.relname = ? + ORDER BY i.indisprimary DESC, ic.relname ASC + `, + [schema, relation] + ) + ); + } + + private async inspectRelationConstraintSignature( + client: IDataDbPreflightClient, + schema: string, + relation: string, + includedForeignKeySchemas?: string[] + ): Promise { + const scopedForeignKeySql = includedForeignKeySchemas?.length + ? "AND (con.contype <> 'f' OR referenced_ns.nspname = ANY(?::text[]))" + : ''; + return normalizeRawRows( + await client.raw( + ` + SELECT + con.conname AS "constraintName", + con.contype AS "constraintType", + pg_get_constraintdef(con.oid, true) AS "definition" + FROM pg_constraint con + JOIN pg_class c ON c.oid = con.conrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_class referenced_rel ON referenced_rel.oid = con.confrelid + LEFT JOIN pg_namespace referenced_ns ON referenced_ns.oid = referenced_rel.relnamespace + WHERE n.nspname = ? + AND c.relname = ? + ${scopedForeignKeySql} + ORDER BY con.contype ASC, con.conname ASC + `, + includedForeignKeySchemas?.length + ? [schema, relation, includedForeignKeySchemas] + : [schema, relation] + ) + ); + } + + private async inspectRelationTriggerSignature( + client: IDataDbPreflightClient, + schema: string, + relation: string + ): Promise { + return normalizeRawRows( + await client.raw( + ` + SELECT + tg.tgname AS "triggerName", + tg.tgenabled AS "enabled", + pg_get_triggerdef(tg.oid, true) AS "definition" + FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = ? + AND c.relname = ? + AND NOT tg.tgisinternal + ORDER BY tg.tgname ASC + `, + [schema, relation] + ) + ); + } + + private async countRows( + client: IDataDbPreflightClient, + schema: string, + table: string, + whereSql?: string, + bindings: unknown[] = [] + ) { + const rows = normalizeRawRows<{ count: string | number | bigint }>( + await client.raw( + `SELECT COUNT(*) AS "count" FROM ${qualify(schema, table)}${ + whereSql ? ` WHERE ${whereSql}` : '' + }`, + bindings + ) + ); + return Number(rows[0]?.count ?? 0); + } + + private async inspectSourceComputedDrain( + client: IDataDbPreflightClient, + schema: string, + baseIds: string[], + processingLeaseMs: number + ): Promise { + const reclaimBefore = new Date(Date.now() - processingLeaseMs); + const rows = normalizeRawRows<{ + activeCount: string | number | bigint | null; + reclaimableCount: string | number | bigint | null; + oldestActiveLockedAt: Date | string | null; + }>( + await client.raw( + ` + SELECT + COUNT(*) FILTER ( + WHERE "status" = 'processing' + AND "locked_at" IS NOT NULL + AND "locked_at" > ?::timestamp + ) AS "activeCount", + COUNT(*) FILTER ( + WHERE "status" = 'processing' + AND ("locked_at" IS NULL OR "locked_at" <= ?::timestamp) + ) AS "reclaimableCount", + MIN("locked_at") FILTER ( + WHERE "status" = 'processing' + AND "locked_at" IS NOT NULL + AND "locked_at" > ?::timestamp + ) AS "oldestActiveLockedAt" + FROM ${qualify(schema, sharedTables.computedUpdateOutbox)} + WHERE "base_id" = ANY(?::text[]) + `, + [reclaimBefore, reclaimBefore, reclaimBefore, baseIds] + ) + ); + const row = rows[0]; + return this.buildComputedDrainStats({ + activeCount: Number(row?.activeCount ?? 0), + reclaimableCount: Number(row?.reclaimableCount ?? 0), + oldestActiveLockedAt: row?.oldestActiveLockedAt ?? null, + }); + } + + private async inspectSchemaOperationDrain( + inventory: ISpaceDataDbInventory + ): Promise { + const where = this.buildSchemaOperationWhere(inventory); + const [openCount, sample] = await Promise.all([ + this.prismaService.schemaOperation.count({ where }), + this.prismaService.schemaOperation.findMany({ + where, + select: { + id: true, + status: true, + phase: true, + baseId: true, + tableId: true, + lockedAt: true, + lockedBy: true, + lastModifiedTime: true, + }, + orderBy: [{ lastModifiedTime: 'desc' }, { createdTime: 'desc' }], + take: 10, + }), + ]); + + return this.buildSchemaOperationDrainStats(openCount, sample); + } + + private buildSchemaOperationWhere( + inventory: ISpaceDataDbInventory + ): Prisma.SchemaOperationWhereInput { + const resourceIds = [...new Set([...inventory.baseIds, ...inventory.tableIds])]; + const scopeFilters: Prisma.SchemaOperationWhereInput[] = []; + if (inventory.baseIds.length) { + scopeFilters.push({ baseId: { in: inventory.baseIds } }); + } + if (inventory.tableIds.length) { + scopeFilters.push({ tableId: { in: inventory.tableIds } }); + } + if (resourceIds.length) { + scopeFilters.push({ resourceId: { in: resourceIds } }); + } + + return { + status: { in: [...openSchemaOperationStatuses] }, + OR: scopeFilters, + }; + } + + private buildSchemaOperationDrainStats( + openCount: number, + sample: { + id: string; + status: string; + phase: string; + baseId: string | null; + tableId: string | null; + lockedAt: Date | string | null; + lockedBy: string | null; + lastModifiedTime: Date | string | null; + }[] + ): ISchemaOperationDrainStats { + return { + openCount, + sample: sample.map((operation) => ({ + id: operation.id, + status: operation.status, + phase: operation.phase, + baseId: operation.baseId, + tableId: operation.tableId, + lockedAt: this.toNullableIso(operation.lockedAt), + lockedBy: operation.lockedBy, + lastModifiedTime: this.toNullableIso(operation.lastModifiedTime), + })), + checkedAt: new Date().toISOString(), + }; + } + + private async updateSchemaOperationDrainStats( + job: IMigrationJobRecord, + inventory: ISpaceDataDbInventory, + phase: 'schema_operations_draining' | 'schema_operations_drained', + stats: ISchemaOperationDrainStats + ) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: job.id }, + data: { + copyStats: { + phase, + progress: this.buildMigrationProgress(job, inventory, phase), + schemaOperations: stats, + }, + lastError: null, + }, + }); + } + + private async inspectBackgroundWriterDrain( + spaceId: string, + inventory: ISpaceDataDbInventory, + options: { + probeTimeoutMs: number; + queueScanBatchSize: number; + queueScanLimit: number; + } + ): Promise { + const [provisionResources, queueJobs] = await Promise.all([ + withTimeout( + this.inspectProvisionResourceDrain(spaceId, inventory), + options.probeTimeoutMs, + `Timed out inspecting provisioning resources after ${options.probeTimeoutMs}ms` + ), + withTimeout( + this.inspectImportQueueDrain(inventory, options), + options.probeTimeoutMs, + `Timed out inspecting import queues after ${options.probeTimeoutMs}ms` + ), + ]); + return this.buildBackgroundWriterDrainStats(provisionResources, queueJobs); + } + + private async inspectProvisionResourceDrain( + spaceId: string, + inventory: ISpaceDataDbInventory + ): Promise<{ openCount: number; sample: IBackgroundWriterDrainStats['sample'] }> { + const baseWhere = this.buildProvisionBaseWhere(spaceId, inventory); + const tableWhere = this.buildProvisionTableWhere(inventory); + const fieldWhere = this.buildProvisionFieldWhere(inventory); + const [baseCount, tableCount, fieldCount, baseSample, tableSample, fieldSample] = + await Promise.all([ + this.prismaService.base.count({ where: baseWhere }), + tableWhere ? this.prismaService.tableMeta.count({ where: tableWhere }) : Promise.resolve(0), + fieldWhere ? this.prismaService.field.count({ where: fieldWhere }) : Promise.resolve(0), + this.prismaService.base.findMany({ + where: baseWhere, + select: { + id: true, + provisionState: true, + lastModifiedTime: true, + }, + orderBy: [{ lastModifiedTime: 'desc' }, { createdTime: 'desc' }], + take: 10, + }), + tableWhere + ? this.prismaService.tableMeta.findMany({ + where: tableWhere, + select: { + id: true, + baseId: true, + provisionState: true, + lastModifiedTime: true, + }, + orderBy: [{ lastModifiedTime: 'desc' }, { createdTime: 'desc' }], + take: 10, + }) + : Promise.resolve([]), + fieldWhere + ? this.prismaService.field.findMany({ + where: fieldWhere, + select: { + id: true, + tableId: true, + provisionState: true, + lastModifiedTime: true, + }, + orderBy: [{ lastModifiedTime: 'desc' }, { createdTime: 'desc' }], + take: 10, + }) + : Promise.resolve([]), + ]); + + return { + openCount: baseCount + tableCount + fieldCount, + sample: [ + ...baseSample.map((resource) => ({ + kind: 'provision_resource' as const, + resourceType: 'base' as const, + id: resource.id, + state: resource.provisionState, + baseId: resource.id, + tableId: null, + lastModifiedTime: this.toNullableIso(resource.lastModifiedTime), + })), + ...tableSample.map((resource) => ({ + kind: 'provision_resource' as const, + resourceType: 'table' as const, + id: resource.id, + state: resource.provisionState, + baseId: resource.baseId, + tableId: resource.id, + lastModifiedTime: this.toNullableIso(resource.lastModifiedTime), + })), + ...fieldSample.map((resource) => ({ + kind: 'provision_resource' as const, + resourceType: 'field' as const, + id: resource.id, + state: resource.provisionState, + baseId: null, + tableId: resource.tableId, + lastModifiedTime: this.toNullableIso(resource.lastModifiedTime), + })), + ], + }; + } + + private buildProvisionBaseWhere( + spaceId: string, + inventory: ISpaceDataDbInventory + ): Prisma.BaseWhereInput { + return { + provisionState: { in: [...openProvisionStates] }, + OR: [{ spaceId }, ...(inventory.baseIds.length ? [{ id: { in: inventory.baseIds } }] : [])], + }; + } + + private buildProvisionTableWhere( + inventory: ISpaceDataDbInventory + ): Prisma.TableMetaWhereInput | null { + const filters: Prisma.TableMetaWhereInput[] = []; + if (inventory.baseIds.length) { + filters.push({ baseId: { in: inventory.baseIds } }); + } + if (inventory.tableIds.length) { + filters.push({ id: { in: inventory.tableIds } }); + } + if (!filters.length) { + return null; + } + return { + provisionState: { in: [...openProvisionStates] }, + OR: filters, + }; + } + + private buildProvisionFieldWhere( + inventory: ISpaceDataDbInventory + ): Prisma.FieldWhereInput | null { + if (!inventory.tableIds.length) { + return null; + } + return { + tableId: { in: inventory.tableIds }, + provisionState: { in: [...openProvisionStates] }, + }; + } + + private async inspectImportQueueDrain( + inventory: ISpaceDataDbInventory, + options: { queueScanBatchSize: number; queueScanLimit: number } + ): Promise<{ openCount: number; sample: IBackgroundWriterDrainStats['sample'] }> { + const queues = [ + { name: BASE_IMPORT_CSV_QUEUE, queue: this.baseImportCsvQueue }, + { name: BASE_IMPORT_JUNCTION_CSV_QUEUE, queue: this.baseImportJunctionCsvQueue }, + { name: TABLE_IMPORT_CSV_CHUNK_QUEUE, queue: this.tableImportCsvChunkQueue }, + { name: TABLE_IMPORT_CSV_QUEUE, queue: this.tableImportCsvQueue }, + ]; + const samples: IBackgroundWriterDrainStats['sample'] = []; + let openCount = 0; + + for (const { name, queue } of queues) { + if (!queue) { + continue; + } + const jobs = await this.getOpenQueueJobs(queue, options); + for (const job of jobs) { + const sample = await this.buildQueueJobDrainSample(name, job, inventory); + if (sample) { + openCount++; + samples.push(sample); + } + } + } + + return { openCount, sample: samples }; + } + + private async getOpenQueueJobs( + queue: Queue, + options: { queueScanBatchSize: number; queueScanLimit: number } + ) { + const openJobs = []; + const batchSize = Math.max(1, options.queueScanBatchSize); + const scanLimit = Math.max(1, options.queueScanLimit); + let start = 0; + + while (start < scanLimit) { + const end = Math.min(start + batchSize - 1, scanLimit - 1); + const jobs = await queue.getJobs([...openQueueJobStates] as never, start, end, false); + if (!jobs.length) { + break; + } + + for (const job of jobs) { + const state = await this.getQueueJobState(job); + if (this.isOpenQueueJobState(state)) { + openJobs.push(job); + } + } + + if (jobs.length < batchSize) { + break; + } + start += batchSize; + } + + return openJobs; + } + + private async buildQueueJobDrainSample( + queueName: string, + job: { + id?: string | number; + name?: string; + data?: unknown; + timestamp?: number; + getState?: () => Promise; + }, + inventory: ISpaceDataDbInventory + ): Promise { + const data = this.asRecord(job.data); + if (!data) { + return null; + } + const scope = this.getQueueJobScope(data); + if (!this.queueJobTouchesInventory(scope, inventory)) { + return null; + } + const state = await this.getQueueJobState(job); + return { + kind: 'queue_job', + queueName, + id: String(job.id ?? job.name ?? 'unknown'), + state, + baseId: scope.baseId, + tableId: scope.tableId, + timestamp: + typeof job.timestamp === 'number' ? this.toNullableIso(new Date(job.timestamp)) : null, + }; + } + + private getQueueJobScope(data: Record) { + const table = this.asRecord(data.table); + const tableIdMap = this.asRecord(data.tableIdMap); + const tableIdMapValues = tableIdMap + ? Object.values(tableIdMap).filter((value): value is string => typeof value === 'string') + : []; + return { + baseId: typeof data.baseId === 'string' ? data.baseId : null, + tableId: + (table && typeof table.id === 'string' ? table.id : null) ?? + (typeof data.tableId === 'string' ? data.tableId : null) ?? + (tableIdMapValues.length === 1 ? tableIdMapValues[0] : null), + tableIds: tableIdMapValues, + }; + } + + private queueJobTouchesInventory( + scope: { baseId: string | null; tableId: string | null; tableIds: string[] }, + inventory: ISpaceDataDbInventory + ) { + const baseIds = new Set(inventory.baseIds); + const tableIds = new Set(inventory.tableIds); + return ( + (scope.baseId != null && baseIds.has(scope.baseId)) || + (scope.tableId != null && tableIds.has(scope.tableId)) || + scope.tableIds.some((tableId) => tableIds.has(tableId)) + ); + } + + private async getQueueJobState(job: { state?: string; getState?: () => Promise }) { + if (typeof job.getState === 'function') { + return await job.getState(); + } + return job.state ?? 'unknown'; + } + + private isOpenQueueJobState(state: string) { + return (openQueueJobStates as readonly string[]).includes(state); + } + + private asRecord(value: unknown): Record | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return null; + } + return value as Record; + } + + private toNullableNumber(value: unknown): number | null { + if (value == null) { + return null; + } + const number = Number(value); + return Number.isFinite(number) ? number : null; + } + + private buildBackgroundWriterDrainStats( + provisionResources: { openCount: number; sample: IBackgroundWriterDrainStats['sample'] }, + queueJobs: { openCount: number; sample: IBackgroundWriterDrainStats['sample'] } + ): IBackgroundWriterDrainStats { + return { + openCount: provisionResources.openCount + queueJobs.openCount, + provisionResourceCount: provisionResources.openCount, + queueJobCount: queueJobs.openCount, + sample: [...provisionResources.sample, ...queueJobs.sample].slice(0, 10), + checkedAt: new Date().toISOString(), + }; + } + + private async updateBackgroundWriterDrainStats( + job: IMigrationJobRecord, + inventory: ISpaceDataDbInventory, + phase: 'background_writers_draining' | 'background_writers_drained', + stats: IBackgroundWriterDrainStats + ) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: job.id }, + data: { + copyStats: { + phase, + progress: this.buildMigrationProgress(job, inventory, phase), + backgroundWriters: stats, + }, + lastError: null, + }, + }); + } + + private buildComputedDrainStats(input: { + activeCount: number; + reclaimableCount: number; + oldestActiveLockedAt: Date | string | null; + }): IComputedDrainStats { + return { + activeCount: input.activeCount, + reclaimableCount: input.reclaimableCount, + oldestActiveLockedAt: this.toNullableIso(input.oldestActiveLockedAt), + checkedAt: new Date().toISOString(), + }; + } + + private async updateComputedDrainStats( + job: IMigrationJobRecord, + inventory: ISpaceDataDbInventory, + phase: 'computed_draining' | 'computed_drained', + stats: IComputedDrainStats + ) { + await this.migrationJobClient.spaceDataDbMigrationJob.update({ + where: { id: job.id }, + data: { + copyStats: { + phase, + progress: this.buildMigrationProgress(job, inventory, phase), + computedDrain: stats, + }, + lastError: null, + }, + }); + } + + private toNullableIso(value: Date | string | null): string | null { + if (!value) { + return null; + } + if (value instanceof Date) { + return value.toISOString(); + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); + } + + private async validateUndoFunction(client: IDataDbPreflightClient, targetSchema: string) { + const rows = normalizeRawRows<{ exists: boolean }>( + await client.raw(`SELECT to_regprocedure(?::text) IS NOT NULL AS "exists"`, [ + `${targetSchema}.__teable_capture_undo_row()`, + ]) + ); + return { exists: Boolean(rows[0]?.exists) }; + } + + private async validateTargetSchemaVersion(client: IDataDbPreflightClient, targetSchema: string) { + const latest = this.baselineService.getLatestSchemaVersion(); + if (!latest) { + return { latest, exists: true }; + } + + const rows = normalizeRawRows<{ exists: boolean }>( + await client.raw( + ` + SELECT EXISTS ( + SELECT 1 + FROM ${qualify(targetSchema, dataDbMigrationTable)} + WHERE "id" = ? + ) AS "exists" + `, + [latest] + ) + ); + return { latest, exists: Boolean(rows[0]?.exists) }; + } + + private async validateRouteSmoke(input: { + spaceId: string; + targetConnectionId: string | null; + targetEncryptedUrl: string; + targetSchema: string; + }): Promise { + if (!input.targetConnectionId) { + return { + ok: false, + connectionId: null, + internalSchema: null, + cacheKey: null, + isMetaFallback: true, + error: 'Migration job has no target connection id', + }; + } + + try { + const resolved = await this.dataDbClientManager.getDataDatabaseForSpace(input.spaceId, { + previewBinding: { + spaceId: input.spaceId, + connectionId: input.targetConnectionId, + encryptedUrl: input.targetEncryptedUrl, + internalSchema: input.targetSchema, + }, + }); + const connectionId = resolved.connectionId ?? null; + const internalSchema = resolved.internalSchema ?? null; + const ok = + !resolved.isMetaFallback && + connectionId === input.targetConnectionId && + internalSchema === input.targetSchema && + resolved.cacheKey === input.targetConnectionId; + + return { + ok, + connectionId, + internalSchema, + cacheKey: resolved.cacheKey, + isMetaFallback: resolved.isMetaFallback, + }; + } catch (error) { + return { + ok: false, + connectionId: null, + internalSchema: null, + cacheKey: null, + isMetaFallback: true, + error: error instanceof Error ? error.message : String(error), + }; + } + } + + private async deleteMigrationComputedPause( + client: IDataDbPreflightClient, + schema: string, + spaceIds: string[], + jobId: string + ) { + const rows = normalizeRawRows<{ id: string }>( + await client.raw( + ` + DELETE FROM ${qualify(schema, sharedTables.computedUpdatePauseScope)} + WHERE "scope_type" = ? + AND "scope_id" = ANY(?::text[]) + AND "reason" = ? + RETURNING "id" + `, + ['space', spaceIds, migrationPauseReason(jobId)] + ) + ); + return { deleted: rows.length }; + } + + private get migrationJobClient(): IMigrationJobClient { + return this.prismaService as unknown as IMigrationJobClient; + } +} diff --git a/apps/nestjs-backend/src/features/space/space-data-db-process-runner.service.spec.ts b/apps/nestjs-backend/src/features/space/space-data-db-process-runner.service.spec.ts new file mode 100644 index 0000000000..fecf1bb292 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-process-runner.service.spec.ts @@ -0,0 +1,790 @@ +import { EventEmitter } from 'events'; +import { PassThrough } from 'stream'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + SpaceDataDbProcessCanceledError, + SpaceDataDbProcessError, + SpaceDataDbProcessPipelineCanceledError, + SpaceDataDbProcessPipelineError, + SpaceDataDbProcessRunnerService, + type ISpaceDataDbProcessSpawn, +} from './space-data-db-process-runner.service'; + +const secretUrl = 'postgresql://user:secret@example/db'; +const redactedUrl = 'postgresql://user:***@example/db'; +const invalidPercentTokenSecret = 'pa%ss$word'; +const copySourceSql = 'COPY (SELECT 1) TO STDOUT'; +const copyTargetSql = 'COPY "meta"."record_history" FROM STDIN'; +const sharedTableLabel = 'shared-table:record_history'; +const pgDumpCommand = 'pg_dump'; +const pgRestoreCommand = 'pg_restore'; +const pgCustomFormatArg = '--format=custom'; + +class FakeProcess extends EventEmitter { + readonly stdin = new PassThrough(); + readonly stderr = new PassThrough(); + readonly stdout = new PassThrough(); + readonly pid = 1234; + readonly kill = vi.fn().mockReturnValue(true); + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; +} + +const expectProcessTiming = (result: { + startedAt: string; + completedAt: string; + durationMs: number; +}) => { + expect(Date.parse(result.startedAt)).not.toBeNaN(); + expect(Date.parse(result.completedAt)).not.toBeNaN(); + expect(result.durationMs).toBeGreaterThanOrEqual(0); +}; + +describe('SpaceDataDbProcessRunnerService', () => { + let processRef: FakeProcess; + let spawnProcess: ReturnType>; + + beforeEach(() => { + processRef = new FakeProcess(); + spawnProcess = vi.fn(() => processRef); + }); + + it('runs commands with spawn arrays and shell disabled', async () => { + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run({ + command: pgDumpCommand, + args: ['--schema', 'bsexxx', secretUrl], + }); + + processRef.stderr.write('dump complete'); + processRef.emit('close', 0, null); + + const result = await promise; + expect(result).toMatchObject({ + command: pgDumpCommand, + args: ['--schema', 'bsexxx', redactedUrl], + exitCode: 0, + stderr: 'dump complete', + }); + expectProcessTiming(result); + expect(spawnProcess).toHaveBeenCalledWith( + pgDumpCommand, + ['--schema', 'bsexxx', secretUrl], + expect.objectContaining({ + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }) + ); + }); + + it('redacts postgres_fdw user mapping passwords from process args and output', async () => { + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const sql = `CREATE USER MAPPING FOR CURRENT_USER SERVER "srv" OPTIONS (user 'teable', password 'source_secret')`; + const promise = service.run({ + command: 'psql', + args: ['--command', sql], + }); + + processRef.stdout.write(`ran ${sql}`); + processRef.emit('close', 0, null); + + const result = await promise; + expect(result.args[1]).toContain(`password '***'`); + expect(result.stdout).toContain(`password '***'`); + }); + + it('keeps stdout and stderr limits independent', async () => { + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgRestoreCommand, + args: ['--list'], + }, + { stdoutLimit: 8, stderrLimit: 4 } + ); + + processRef.stdout.write('stdout-long'); + processRef.stderr.write('stderr-long'); + processRef.emit('close', 0, null); + + const result = await promise; + expect(result.stdout).toBe('out-long'); + expect(result.stderr).toBe('long'); + }); + + it('rejects non-zero exits with redacted args and stderr', async () => { + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run({ + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }); + + processRef.stderr.write(`failed ${secretUrl}`); + processRef.emit('close', 1, null); + + try { + await promise; + throw new Error('Expected process to fail'); + } catch (error) { + expect(error).toBeInstanceOf(SpaceDataDbProcessError); + const result = (error as SpaceDataDbProcessError).result; + expect(result).toMatchObject({ + command: pgRestoreCommand, + args: ['--dbname', redactedUrl], + exitCode: 1, + stderr: `failed ${redactedUrl}`, + }); + expectProcessTiming(result); + } + }); + + it('redacts libpq invalid percent token output from process failures', async () => { + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run({ + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }); + + processRef.stderr.write( + `pg_restore: error: invalid percent-encoded token: "${invalidPercentTokenSecret}"` + ); + processRef.emit('close', 1, null); + + try { + await promise; + throw new Error('Expected process to fail'); + } catch (error) { + const result = (error as SpaceDataDbProcessError).result; + expect(result.stderr).toContain('invalid percent-encoded token: "***"'); + expect(result.stderr).not.toContain(invalidPercentTokenSecret); + } + }); + + it('kills timed-out processes and rejects with a process error', async () => { + vi.useFakeTimers(); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgDumpCommand, + args: ['--schema', 'bsexxx'], + }, + { timeoutMs: 100 } + ); + const assertion = expect(promise).rejects.toBeInstanceOf(SpaceDataDbProcessError); + + await vi.advanceTimersByTimeAsync(100); + + await assertion; + expect(processRef.kill).toHaveBeenCalledWith('SIGTERM'); + vi.useRealTimers(); + }); + + it('kills running commands when cancellation is requested', async () => { + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgDumpCommand, + args: ['--schema', 'bsexxx'], + }, + { shouldCancel: () => true } + ); + + await expect(promise).rejects.toBeInstanceOf(SpaceDataDbProcessCanceledError); + expect(processRef.kill).toHaveBeenCalledWith('SIGTERM'); + }); + + it('polls progress hooks while a command is running', async () => { + vi.useFakeTimers(); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const onPoll = vi.fn().mockResolvedValue(undefined); + const promise = service.run( + { + command: pgDumpCommand, + args: ['--schema', 'bsexxx'], + }, + { pollMs: 50, onPoll } + ); + + await vi.advanceTimersByTimeAsync(50); + processRef.emit('close', 0, null); + const result = await promise; + + expect(result.exitCode).toBe(0); + expect(onPoll).toHaveBeenCalled(); + vi.useRealTimers(); + }); + + it('rejects a command when a progress poll hangs', async () => { + vi.useFakeTimers(); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgDumpCommand, + args: ['--schema', 'bsexxx'], + }, + { + pollMs: 50, + pollTimeoutMs: 100, + onPoll: () => new Promise(() => undefined), + } + ); + const assertion = expect(promise).rejects.toMatchObject({ + message: expect.stringContaining('Process progress poll timed out after 100ms'), + }); + + await vi.advanceTimersByTimeAsync(100); + + await assertion; + expect(processRef.kill).toHaveBeenCalledWith('SIGTERM'); + vi.useRealTimers(); + }); + + it('rejects a command when progress polls keep failing', async () => { + vi.useFakeTimers(); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgDumpCommand, + args: ['--schema', 'bsexxx'], + }, + { + pollMs: 50, + pollFailureTimeoutMs: 100, + onPoll: () => { + throw new Error('the database system is in recovery mode'); + }, + } + ); + const assertion = expect(promise).rejects.toMatchObject({ + message: expect.stringContaining('Process progress poll failed for 100ms'), + }); + + await vi.advanceTimersByTimeAsync(150); + + await assertion; + expect(processRef.kill).toHaveBeenCalledWith('SIGTERM'); + vi.useRealTimers(); + }); + + it('resolves a successful command when child exit arrives without close', async () => { + vi.useFakeTimers(); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgDumpCommand, + args: ['--format=directory', secretUrl], + }, + { exitFallbackMs: 50 } + ); + + processRef.emit('exit', 0, null); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + command: pgDumpCommand, + args: ['--format=directory', redactedUrl], + exitCode: 0, + }); + expectProcessTiming(result); + vi.useRealTimers(); + }); + + it('resolves a successful command when stdio closes before child close arrives', async () => { + vi.useFakeTimers(); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + const promise = service.run( + { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + { exitFallbackMs: 50 } + ); + + processRef.stdout.end(); + processRef.stderr.end(); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + command: pgRestoreCommand, + args: ['--dbname', redactedUrl], + exitCode: 0, + }); + expectProcessTiming(result); + vi.useRealTimers(); + }); + + it('pipes source stdout into target stdin with shell disabled', async () => { + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + const targetChunks: Buffer[] = []; + targetProcess.stdin.on('data', (chunk: Buffer) => targetChunks.push(chunk)); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline({ + source: { + command: 'psql', + args: ['--command', copySourceSql, secretUrl], + }, + target: { + command: 'psql', + args: ['--command', copyTargetSql, secretUrl], + }, + }); + + sourceProcess.stdout.write('row-1\n'); + sourceProcess.stdout.end('row-2\n'); + sourceProcess.emit('close', 0, null); + targetProcess.stdout.write('COPY 2\n'); + targetProcess.emit('close', 0, null); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: 'psql', args: ['--command', copySourceSql, redactedUrl] }, + target: { + command: 'psql', + args: ['--command', copyTargetSql, redactedUrl], + stdout: 'COPY 2\n', + }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + expect(Buffer.concat(targetChunks).toString('utf8')).toBe('row-1\nrow-2\n'); + expect(spawnProcess).toHaveBeenNthCalledWith( + 1, + 'psql', + ['--command', copySourceSql, secretUrl], + expect.objectContaining({ shell: false, stdio: ['ignore', 'pipe', 'pipe'] }) + ); + expect(spawnProcess).toHaveBeenNthCalledWith( + 2, + 'psql', + ['--command', copyTargetSql, secretUrl], + expect.objectContaining({ shell: false, stdio: ['pipe', 'pipe', 'pipe'] }) + ); + }); + + it('resolves a successful pipeline when child exit events arrive without close events', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + + sourceProcess.emit('exit', 0, null); + targetProcess.emit('exit', 0, null); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: pgDumpCommand, exitCode: 0 }, + target: { command: pgRestoreCommand, exitCode: 0 }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + vi.useRealTimers(); + }); + + it('resolves a successful pipeline when stdio closes before child close events arrive', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + + sourceProcess.stdout.end(); + sourceProcess.stderr.end(); + targetProcess.stdin.end(); + targetProcess.stdout.end(); + targetProcess.stderr.end(); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: pgDumpCommand, exitCode: 0 }, + target: { command: pgRestoreCommand, exitCode: 0 }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + vi.useRealTimers(); + }); + + it('ends target stdin when source exits before stdout closes', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + targetProcess.stdin.on('finish', () => { + targetProcess.emit('exit', 0, null); + targetProcess.emit('close', 0, null); + }); + + sourceProcess.emit('exit', 0, null); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: pgDumpCommand, exitCode: 0 }, + target: { command: pgRestoreCommand, exitCode: 0 }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + vi.useRealTimers(); + }); + + it('resolves a successful pipeline when source stdio closes and target exits before source close arrives', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + + sourceProcess.stdout.end(); + sourceProcess.stderr.end(); + targetProcess.emit('exit', 0, null); + targetProcess.emit('close', 0, null); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: pgDumpCommand, exitCode: 0 }, + target: { command: pgRestoreCommand, exitCode: 0 }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + vi.useRealTimers(); + }); + + it('resolves a successful pipeline when source exits and target stdio closes before target exit arrives', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + + sourceProcess.emit('exit', 0, null); + targetProcess.stdin.end(); + targetProcess.stdout.end(); + targetProcess.stderr.end(); + await vi.advanceTimersByTimeAsync(50); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: pgDumpCommand, exitCode: 0 }, + target: { command: pgRestoreCommand, exitCode: 0 }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + vi.useRealTimers(); + }); + + it('resolves a successful pipeline when child exit status changes without exit or close events', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + + sourceProcess.stdout.end(); + sourceProcess.stderr.end(); + targetProcess.stdin.end(); + targetProcess.stdout.end(); + targetProcess.stderr.end(); + sourceProcess.exitCode = 0; + targetProcess.exitCode = 0; + await vi.advanceTimersByTimeAsync(100); + + const result = await promise; + expect(result).toMatchObject({ + source: { command: pgDumpCommand, exitCode: 0 }, + target: { command: pgRestoreCommand, exitCode: 0 }, + }); + expectProcessTiming(result.source); + expectProcessTiming(result.target); + vi.useRealTimers(); + }); + + it('rejects a pipeline when a child process disappears without exit or close events', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + sourceProcess.kill.mockImplementation((signal?: NodeJS.Signals | 0) => + signal === 0 ? false : true + ); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { exitFallbackMs: 50 } + ); + const assertion = expect(promise).rejects.toMatchObject({ + message: expect.stringContaining('exited without status'), + result: expect.objectContaining({ + source: expect.objectContaining({ + exitCode: null, + signal: null, + }), + }), + }); + + await vi.advanceTimersByTimeAsync(50); + + await assertion; + expect(targetProcess.kill).toHaveBeenCalledWith('SIGTERM'); + vi.useRealTimers(); + }); + + it('rejects a pipeline when a progress poll hangs', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { + pollMs: 50, + pollTimeoutMs: 100, + onPoll: () => new Promise(() => undefined), + } + ); + const assertion = expect(promise).rejects.toMatchObject({ + message: expect.stringContaining('Process progress poll timed out after 100ms'), + result: expect.objectContaining({ + source: expect.objectContaining({ signal: 'SIGTERM' }), + target: expect.objectContaining({ signal: 'SIGTERM' }), + }), + }); + + await vi.advanceTimersByTimeAsync(100); + + await assertion; + expect(sourceProcess.kill).toHaveBeenCalledWith('SIGTERM'); + expect(targetProcess.kill).toHaveBeenCalledWith('SIGTERM'); + vi.useRealTimers(); + }); + + it('rejects a pipeline when progress polls keep failing', async () => { + vi.useFakeTimers(); + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { + command: pgDumpCommand, + args: [pgCustomFormatArg, secretUrl], + }, + target: { + command: pgRestoreCommand, + args: ['--dbname', secretUrl], + }, + }, + { + pollMs: 50, + pollFailureTimeoutMs: 100, + onPoll: () => { + throw new Error('the database system is in recovery mode'); + }, + } + ); + const assertion = expect(promise).rejects.toMatchObject({ + message: expect.stringContaining('Process progress poll failed for 100ms'), + result: expect.objectContaining({ + source: expect.objectContaining({ signal: 'SIGTERM' }), + target: expect.objectContaining({ signal: 'SIGTERM' }), + }), + }); + + await vi.advanceTimersByTimeAsync(150); + + await assertion; + expect(sourceProcess.kill).toHaveBeenCalledWith('SIGTERM'); + expect(targetProcess.kill).toHaveBeenCalledWith('SIGTERM'); + vi.useRealTimers(); + }); + + it('kills the target process when the source COPY process exits non-zero', async () => { + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline({ + source: { command: 'psql', args: ['--command', 'COPY bad TO STDOUT', secretUrl] }, + target: { command: 'psql', args: ['--command', 'COPY good FROM STDIN', secretUrl] }, + label: sharedTableLabel, + }); + + sourceProcess.stderr.write(`source failed ${secretUrl}`); + sourceProcess.emit('close', 1, null); + + await expect(promise).rejects.toMatchObject({ + result: expect.objectContaining({ + label: sharedTableLabel, + source: expect.objectContaining({ + exitCode: 1, + stderr: `source failed ${redactedUrl}`, + }), + }), + }); + await expect(promise).rejects.toBeInstanceOf(SpaceDataDbProcessPipelineError); + expect(targetProcess.kill).toHaveBeenCalledWith('SIGTERM'); + }); + + it('redacts libpq invalid percent token output from pipeline failures', async () => { + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline({ + source: { command: pgDumpCommand, args: [pgCustomFormatArg, secretUrl] }, + target: { command: pgRestoreCommand, args: ['--dbname', secretUrl] }, + label: 'base-schemas', + }); + + targetProcess.stderr.write( + `pg_restore: error: invalid percent-encoded token: "${invalidPercentTokenSecret}"` + ); + targetProcess.emit('close', 1, null); + + await expect(promise).rejects.toMatchObject({ + result: expect.objectContaining({ + target: expect.objectContaining({ + stderr: expect.stringContaining('invalid percent-encoded token: "***"'), + }), + }), + }); + await expect(promise).rejects.toBeInstanceOf(SpaceDataDbProcessPipelineError); + }); + + it('kills both COPY processes when cancellation is requested', async () => { + const sourceProcess = new FakeProcess(); + const targetProcess = new FakeProcess(); + spawnProcess = vi.fn().mockReturnValueOnce(sourceProcess).mockReturnValueOnce(targetProcess); + const service = new SpaceDataDbProcessRunnerService(spawnProcess); + + const promise = service.runPipeline( + { + source: { command: 'psql', args: ['--command', 'COPY source TO STDOUT', secretUrl] }, + target: { command: 'psql', args: ['--command', 'COPY target FROM STDIN', secretUrl] }, + label: sharedTableLabel, + }, + { shouldCancel: () => true } + ); + + await expect(promise).rejects.toBeInstanceOf(SpaceDataDbProcessPipelineCanceledError); + expect(sourceProcess.kill).toHaveBeenCalledWith('SIGTERM'); + expect(targetProcess.kill).toHaveBeenCalledWith('SIGTERM'); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/space-data-db-process-runner.service.ts b/apps/nestjs-backend/src/features/space/space-data-db-process-runner.service.ts new file mode 100644 index 0000000000..6498ffe715 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-process-runner.service.ts @@ -0,0 +1,1143 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { SpawnOptions } from 'child_process'; +import { spawn as nodeSpawn } from 'child_process'; +import { Inject, Injectable, Optional } from '@nestjs/common'; +import type { + ISpaceDataDbProcessPipelinePlan, + ISpaceDataDbProcessPlan, +} from './space-data-db-copy-plan'; + +export const SPACE_DATA_DB_PROCESS_SPAWN = Symbol('SPACE_DATA_DB_PROCESS_SPAWN'); + +type IProcessLike = { + pid?: number; + stdin?: NodeJS.WritableStream | null; + stderr?: NodeJS.ReadableStream | null; + stdout?: NodeJS.ReadableStream | null; + exitCode?: number | null; + signalCode?: NodeJS.Signals | null; + kill(signal?: NodeJS.Signals | 0): boolean; + on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): void; + on(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): void; + on(event: 'error', listener: (error: Error) => void): void; +}; + +export type ISpaceDataDbProcessSpawn = ( + command: string, + args: string[], + options: SpawnOptions +) => IProcessLike; + +export type ISpaceDataDbProcessRunOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + stderrLimit?: number; + stdoutLimit?: number; + cancelPollMs?: number; + exitFallbackMs?: number; + pollTimeoutMs?: number; + pollFailureTimeoutMs?: number; + shouldCancel?: () => boolean | Promise; + pollMs?: number; + onPoll?: () => void | Promise; +}; + +export type ISpaceDataDbProcessRunResult = { + command: string; + args: string[]; + exitCode: number; + signal: NodeJS.Signals | null; + stderr: string; + stdout: string; + startedAt: string; + completedAt: string; + durationMs: number; +}; + +type ISpaceDataDbProcessPartialResult = Omit & { + exitCode: number | null; +}; + +export type ISpaceDataDbProcessPipelineResult = { + source: ISpaceDataDbProcessRunResult; + target: ISpaceDataDbProcessRunResult; +}; + +const defaultCancelPollMs = 1_000; +const defaultExitFallbackMs = 5_000; +const defaultStderrLimit = 16 * 1024; + +const runProgressPoll = async ( + options: ISpaceDataDbProcessRunOptions, + isSettled: () => boolean +) => { + if (!options.onPoll || isSettled()) { + return; + } + await options.onPoll(); +}; + +const createProgressPollRunner = ( + options: ISpaceDataDbProcessRunOptions, + isSettled: () => boolean, + onTimeout: (error: Error) => void +) => { + let inFlight = false; + let timeout: NodeJS.Timeout | undefined; + let firstFailureAtMs: number | null = null; + + const clear = () => { + if (timeout) { + clearTimeout(timeout); + timeout = undefined; + } + }; + + const run = () => { + if (!options.onPoll || isSettled() || inFlight) { + return; + } + + inFlight = true; + let timedOut = false; + const pollTimeoutMs = + options.pollTimeoutMs && options.pollTimeoutMs > 0 + ? Math.max(1, Math.floor(options.pollTimeoutMs)) + : undefined; + const pollFailureTimeoutMs = + options.pollFailureTimeoutMs && options.pollFailureTimeoutMs > 0 + ? Math.max(1, Math.floor(options.pollFailureTimeoutMs)) + : undefined; + + if (pollTimeoutMs) { + timeout = setTimeout(() => { + if (timedOut || isSettled()) { + return; + } + timedOut = true; + inFlight = false; + timeout = undefined; + onTimeout(new Error(`Process progress poll timed out after ${pollTimeoutMs}ms`)); + }, pollTimeoutMs); + } + + void runProgressPoll(options, isSettled) + .then(() => { + firstFailureAtMs = null; + }) + .catch((error) => { + if (timedOut || isSettled()) { + return; + } + if (!pollFailureTimeoutMs) { + return; + } + const now = Date.now(); + firstFailureAtMs ??= now; + if (now - firstFailureAtMs < pollFailureTimeoutMs) { + return; + } + const message = error instanceof Error ? error.message : String(error); + timedOut = true; + inFlight = false; + clear(); + onTimeout( + new Error(`Process progress poll failed for ${pollFailureTimeoutMs}ms: ${message}`) + ); + }) + .finally(() => { + if (timedOut) { + return; + } + inFlight = false; + clear(); + }); + }; + + return { run, clear }; +}; + +const redactSecrets = (value: string) => + value + .replace(/(postgres(?:ql)?:\/\/[^:\s/@]+:)[^@\s]+(@)/gi, '$1***$2') + .replace(/(invalid percent-encoded token:\s*["'])[^"']*(["'])/gi, '$1***$2') + .replace(/(password\s+')[^']*(')/gi, '$1***$2') + .replace(/(password=)[^'\s,)]+/gi, '$1***'); + +const appendBounded = (current: string, chunk: unknown, limit: number) => { + const next = + current + redactSecrets(Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk)); + if (next.length <= limit) { + return next; + } + return next.slice(next.length - limit); +}; + +const redactArgs = (args: string[]) => args.map((arg) => redactSecrets(arg)); + +export class SpaceDataDbProcessError extends Error { + constructor( + message: string, + readonly result: Omit & { + exitCode: number | null; + } + ) { + super(message); + } +} + +export class SpaceDataDbProcessPipelineError extends Error { + constructor( + message: string, + readonly result: { + label?: string; + source: ISpaceDataDbProcessPartialResult; + target: ISpaceDataDbProcessPartialResult; + } + ) { + super(message); + } +} + +export class SpaceDataDbProcessCanceledError extends SpaceDataDbProcessError { + constructor(result: SpaceDataDbProcessError['result']) { + super('Process canceled', result); + } +} + +export class SpaceDataDbProcessPipelineCanceledError extends SpaceDataDbProcessPipelineError { + constructor(result: SpaceDataDbProcessPipelineError['result']) { + super('Process pipeline canceled', result); + } +} + +type IProcessState = { + command: string; + args: string[]; + stderr: string; + stdout: string; + exitCode: number | null; + signal: NodeJS.Signals | null; + startedAt: string; + startedAtMs: number; + completedAt: string | null; + durationMs: number | null; +}; + +type IReadableState = NodeJS.ReadableStream & { + closed?: boolean; + destroyed?: boolean; + readableEnded?: boolean; +}; + +type IWritableState = NodeJS.WritableStream & { + closed?: boolean; + destroyed?: boolean; + writableEnded?: boolean; + writableFinished?: boolean; +}; + +const nowTiming = () => { + const now = new Date(); + return { + iso: now.toISOString(), + ms: now.getTime(), + }; +}; + +const durationSince = (startedAtMs: number, completedAtMs: number) => + Math.max(0, completedAtMs - startedAtMs); + +const initProcessState = (plan: ISpaceDataDbProcessPlan): IProcessState => { + const startedAt = nowTiming(); + return { + command: plan.command, + args: redactArgs(plan.args), + stderr: '', + stdout: '', + exitCode: null, + signal: null, + startedAt: startedAt.iso, + startedAtMs: startedAt.ms, + completedAt: null, + durationMs: null, + }; +}; + +const completeProcessState = (state: IProcessState) => { + if (state.completedAt) { + return; + } + const completedAt = nowTiming(); + state.completedAt = completedAt.iso; + state.durationMs = durationSince(state.startedAtMs, completedAt.ms); +}; + +const toPartialResult = (state: IProcessState): ISpaceDataDbProcessPartialResult => ({ + command: state.command, + args: state.args, + exitCode: state.exitCode, + signal: state.signal, + stderr: state.stderr, + stdout: state.stdout, + startedAt: state.startedAt, + completedAt: state.completedAt ?? new Date(state.startedAtMs).toISOString(), + durationMs: state.durationMs ?? 0, +}); + +const toRunResult = (state: IProcessState): ISpaceDataDbProcessRunResult => ({ + command: state.command, + args: state.args, + exitCode: state.exitCode ?? 0, + signal: state.signal, + stderr: state.stderr, + stdout: state.stdout, + startedAt: state.startedAt, + completedAt: state.completedAt ?? new Date(state.startedAtMs).toISOString(), + durationMs: state.durationMs ?? 0, +}); + +const isProcessGoneWithoutExitStatus = (child: IProcessLike, state: IProcessState) => { + if (state.exitCode !== null || state.signal !== null || child.pid == null) { + return false; + } + try { + return child.kill(0) === false; + } catch { + return true; + } +}; + +@Injectable() +export class SpaceDataDbProcessRunnerService { + constructor( + @Optional() + @Inject(SPACE_DATA_DB_PROCESS_SPAWN) + private readonly spawnProcess: ISpaceDataDbProcessSpawn = nodeSpawn as ISpaceDataDbProcessSpawn + ) {} + + async run( + plan: ISpaceDataDbProcessPlan, + options: ISpaceDataDbProcessRunOptions = {} + ): Promise { + const stderrLimit = options.stderrLimit ?? defaultStderrLimit; + const stdoutLimit = options.stdoutLimit ?? stderrLimit; + const redactedArgs = redactArgs(plan.args); + const startedAt = nowTiming(); + const child = this.spawnProcess(plan.command, plan.args, { + cwd: options.cwd, + env: options.env ?? process.env, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + let stdout = ''; + let cancelTimer: NodeJS.Timeout | undefined; + let pollTimer: NodeJS.Timeout | undefined; + let timeout: NodeJS.Timeout | undefined; + let exitFallbackTimer: NodeJS.Timeout | undefined; + let stdioFallbackTimer: NodeJS.Timeout | undefined; + let progressPollCleanup: (() => void) | undefined; + let exitCode: number | null = null; + let signal: NodeJS.Signals | null = null; + let exited = false; + let closed = false; + let stdoutClosed = !child.stdout; + let stderrClosed = !child.stderr; + + child.stderr?.on('data', (chunk) => { + stderr = appendBounded(stderr, chunk, stderrLimit); + }); + child.stdout?.on('data', (chunk) => { + stdout = appendBounded(stdout, chunk, stdoutLimit); + }); + + const buildResult = ( + exitCode: TExitCode, + signal: NodeJS.Signals | null + ) => { + const completedAt = nowTiming(); + return { + command: plan.command, + args: redactedArgs, + exitCode, + signal, + stderr, + stdout, + startedAt: startedAt.iso, + completedAt: completedAt.iso, + durationMs: durationSince(startedAt.ms, completedAt.ms), + }; + }; + + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (fn: () => void) => { + if (settled) { + return; + } + settled = true; + if (timeout) { + clearTimeout(timeout); + } + if (cancelTimer) { + clearInterval(cancelTimer); + } + if (pollTimer) { + clearInterval(pollTimer); + } + if (exitFallbackTimer) { + clearTimeout(exitFallbackTimer); + } + if (stdioFallbackTimer) { + clearTimeout(stdioFallbackTimer); + } + progressPollCleanup?.(); + fn(); + }; + + const cancel = () => { + child.kill('SIGTERM'); + settle(() => { + reject(new SpaceDataDbProcessCanceledError(buildResult(null, 'SIGTERM'))); + }); + }; + const checkCancel = async () => { + if (!options.shouldCancel || settled) { + return; + } + if (await options.shouldCancel()) { + cancel(); + } + }; + if (options.shouldCancel) { + cancelTimer = setInterval( + () => { + void checkCancel().catch(() => undefined); + }, + Math.max(1, options.cancelPollMs ?? defaultCancelPollMs) + ); + void checkCancel().catch(() => undefined); + } + if (options.onPoll) { + const progressPoll = createProgressPollRunner( + options, + () => settled, + (error) => { + child.kill('SIGTERM'); + settle(() => { + reject( + new SpaceDataDbProcessError( + `${error.message}: ${plan.command} ${redactedArgs.join(' ')}`, + buildResult(null, 'SIGTERM') + ) + ); + }); + } + ); + progressPollCleanup = progressPoll.clear; + pollTimer = setInterval( + progressPoll.run, + Math.max(1, options.pollMs ?? defaultCancelPollMs) + ); + progressPoll.run(); + } + + if (options.timeoutMs && options.timeoutMs > 0) { + timeout = setTimeout(() => { + child.kill('SIGTERM'); + settle(() => { + reject( + new SpaceDataDbProcessError( + `Process timed out: ${plan.command} ${redactedArgs.join(' ')}`, + buildResult(null, 'SIGTERM') + ) + ); + }); + }, options.timeoutMs); + } + + child.on('error', (error) => { + settle(() => { + reject( + new SpaceDataDbProcessError( + `Process failed to start: ${plan.command} ${redactedArgs.join(' ')}`, + { + ...buildResult(null, null), + exitCode: null, + signal: null, + stderr: stderr || redactSecrets(error.message), + } + ) + ); + }); + }); + + const settleFromExitStatus = () => { + const result = buildResult(exitCode ?? 0, signal); + if (exitCode === 0 || (exitCode === null && !stderr)) { + settle(() => resolve(result)); + return; + } + settle(() => { + reject( + new SpaceDataDbProcessError( + `Process exited with code ${exitCode}: ${plan.command} ${redactedArgs.join(' ')}`, + result + ) + ); + }); + }; + const scheduleExitFallback = () => { + if (settled || exitFallbackTimer || !exited || closed) { + return; + } + exitFallbackTimer = setTimeout( + () => { + exitFallbackTimer = undefined; + if (settled || closed) { + return; + } + settleFromExitStatus(); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + }; + const scheduleStdioFallback = () => { + if (settled || stdioFallbackTimer || !stdoutClosed || !stderrClosed || closed) { + return; + } + stdioFallbackTimer = setTimeout( + () => { + stdioFallbackTimer = undefined; + if (settled || closed) { + return; + } + settleFromExitStatus(); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + }; + const markStdoutClosed = () => { + stdoutClosed = true; + scheduleStdioFallback(); + }; + const markStderrClosed = () => { + stderrClosed = true; + scheduleStdioFallback(); + }; + child.stdout?.on('end', markStdoutClosed); + child.stdout?.on('close', markStdoutClosed); + child.stderr?.on('end', markStderrClosed); + child.stderr?.on('close', markStderrClosed); + child.on('exit', (childExitCode, childSignal) => { + exited = true; + exitCode = childExitCode; + signal = childSignal; + scheduleExitFallback(); + }); + child.on('close', (exitCode, signal) => { + closed = true; + exited = true; + const result = buildResult(exitCode ?? 0, signal); + if (exitCode === 0) { + settle(() => resolve(result)); + return; + } + settle(() => { + reject( + new SpaceDataDbProcessError( + `Process exited with code ${exitCode}: ${plan.command} ${redactedArgs.join(' ')}`, + result + ) + ); + }); + }); + }); + } + + async runPipeline( + plan: ISpaceDataDbProcessPipelinePlan, + options: ISpaceDataDbProcessRunOptions = {} + ): Promise { + const stderrLimit = options.stderrLimit ?? defaultStderrLimit; + const stdoutLimit = options.stdoutLimit ?? stderrLimit; + const sourceState = initProcessState(plan.source); + const targetState = initProcessState(plan.target); + const sourceChild = this.spawnProcess(plan.source.command, plan.source.args, { + cwd: options.cwd, + env: options.env ?? process.env, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const targetChild = this.spawnProcess(plan.target.command, plan.target.args, { + cwd: options.cwd, + env: options.env ?? process.env, + shell: false, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + sourceChild.stderr?.on('data', (chunk) => { + sourceState.stderr = appendBounded(sourceState.stderr, chunk, stderrLimit); + }); + targetChild.stderr?.on('data', (chunk) => { + targetState.stderr = appendBounded(targetState.stderr, chunk, stderrLimit); + }); + targetChild.stdout?.on('data', (chunk) => { + targetState.stdout = appendBounded(targetState.stdout, chunk, stdoutLimit); + }); + + return await new Promise((resolve, reject) => { + let settled = false; + let cancelTimer: NodeJS.Timeout | undefined; + let pollTimer: NodeJS.Timeout | undefined; + let timeout: NodeJS.Timeout | undefined; + let exitFallbackTimer: NodeJS.Timeout | undefined; + let stdioFallbackTimer: NodeJS.Timeout | undefined; + let successfulTargetFallbackTimer: NodeJS.Timeout | undefined; + let sourceInputFallbackTimer: NodeJS.Timeout | undefined; + let progressPollCleanup: (() => void) | undefined; + const timers: { processStatePollTimer?: NodeJS.Timeout } = {}; + let sourceClosed = false; + let targetClosed = false; + let sourceExited = false; + let targetExited = false; + let sourceStdoutClosed = false; + let sourceStderrClosed = !sourceChild.stderr; + let targetStdinClosed = false; + let targetStdoutClosed = !targetChild.stdout; + let targetStderrClosed = !targetChild.stderr; + const clearTimers = () => { + if (timeout) { + clearTimeout(timeout); + } + if (cancelTimer) { + clearInterval(cancelTimer); + } + if (pollTimer) { + clearInterval(pollTimer); + } + if (exitFallbackTimer) { + clearTimeout(exitFallbackTimer); + } + if (stdioFallbackTimer) { + clearTimeout(stdioFallbackTimer); + } + if (successfulTargetFallbackTimer) { + clearTimeout(successfulTargetFallbackTimer); + } + if (sourceInputFallbackTimer) { + clearTimeout(sourceInputFallbackTimer); + } + if (timers.processStatePollTimer) { + clearInterval(timers.processStatePollTimer); + } + progressPollCleanup?.(); + }; + const rejectPipeline = (message: string, killSource: boolean, killTarget: boolean) => { + if (settled) { + return; + } + settled = true; + clearTimers(); + if (killSource) { + sourceState.signal = sourceState.signal ?? 'SIGTERM'; + sourceChild.kill('SIGTERM'); + } + if (killTarget) { + targetState.signal = targetState.signal ?? 'SIGTERM'; + targetChild.kill('SIGTERM'); + } + completeProcessState(sourceState); + completeProcessState(targetState); + reject( + new SpaceDataDbProcessPipelineError(message, { + label: plan.label, + source: toPartialResult(sourceState), + target: toPartialResult(targetState), + }) + ); + }; + const resolveIfComplete = () => { + if (settled || sourceState.exitCode !== 0 || targetState.exitCode !== 0) { + return; + } + settled = true; + clearTimers(); + completeProcessState(sourceState); + completeProcessState(targetState); + resolve({ + source: toRunResult(sourceState), + target: toRunResult(targetState), + }); + }; + + if (!sourceChild.stdout || !targetChild.stdin) { + rejectPipeline( + `Process pipeline could not be opened: ${plan.source.command} -> ${plan.target.command}`, + true, + true + ); + return; + } + + const cancelPipeline = () => { + if (settled) { + return; + } + settled = true; + clearTimers(); + sourceState.signal = sourceState.signal ?? 'SIGTERM'; + targetState.signal = targetState.signal ?? 'SIGTERM'; + sourceChild.kill('SIGTERM'); + targetChild.kill('SIGTERM'); + completeProcessState(sourceState); + completeProcessState(targetState); + reject( + new SpaceDataDbProcessPipelineCanceledError({ + label: plan.label, + source: toPartialResult(sourceState), + target: toPartialResult(targetState), + }) + ); + }; + const checkCancel = async () => { + if (!options.shouldCancel || settled) { + return; + } + if (await options.shouldCancel()) { + cancelPipeline(); + } + }; + if (options.shouldCancel) { + cancelTimer = setInterval( + () => { + void checkCancel().catch(() => undefined); + }, + Math.max(1, options.cancelPollMs ?? defaultCancelPollMs) + ); + void checkCancel().catch(() => undefined); + } + if (options.onPoll) { + const progressPoll = createProgressPollRunner( + options, + () => settled, + (error) => { + rejectPipeline( + `${error.message}: ${plan.source.command} -> ${plan.target.command}`, + true, + true + ); + } + ); + progressPollCleanup = progressPoll.clear; + pollTimer = setInterval( + progressPoll.run, + Math.max(1, options.pollMs ?? defaultCancelPollMs) + ); + progressPoll.run(); + } + + if (options.timeoutMs && options.timeoutMs > 0) { + timeout = setTimeout(() => { + sourceState.signal = sourceState.signal ?? 'SIGTERM'; + targetState.signal = targetState.signal ?? 'SIGTERM'; + rejectPipeline( + `Process pipeline timed out: ${plan.source.command} -> ${plan.target.command}`, + true, + true + ); + }, options.timeoutMs); + } + + const scheduleExitFallback = () => { + if ( + settled || + exitFallbackTimer || + !sourceExited || + !targetExited || + (sourceClosed && targetClosed) + ) { + return; + } + + exitFallbackTimer = setTimeout( + () => { + exitFallbackTimer = undefined; + if (settled || (sourceClosed && targetClosed)) { + return; + } + if (sourceState.exitCode === 0 && targetState.exitCode === 0) { + resolveIfComplete(); + return; + } + rejectPipeline( + `Process pipeline exited before stdio closed: ${plan.source.command} -> ${plan.target.command}`, + false, + false + ); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + }; + const isSourceOutputSettled = () => + (sourceStdoutClosed && sourceStderrClosed) || + (sourceState.exitCode === 0 && !sourceState.stderr); + const scheduleStdioFallback = () => { + if ( + settled || + stdioFallbackTimer || + !isSourceOutputSettled() || + !targetStdinClosed || + !targetStdoutClosed || + !targetStderrClosed || + (sourceClosed && targetClosed) + ) { + return; + } + + stdioFallbackTimer = setTimeout( + () => { + stdioFallbackTimer = undefined; + if (settled || (sourceClosed && targetClosed)) { + return; + } + if ( + !isSourceOutputSettled() || + !targetStdinClosed || + !targetStdoutClosed || + !targetStderrClosed + ) { + return; + } + if ( + (sourceState.exitCode !== null && sourceState.exitCode !== 0) || + (targetState.exitCode !== null && targetState.exitCode !== 0) + ) { + rejectPipeline( + `Process pipeline stdio closed after a non-zero exit: ${plan.source.command} -> ${plan.target.command}`, + false, + false + ); + return; + } + if ( + (sourceState.exitCode === null || targetState.exitCode === null) && + (sourceState.stderr || targetState.stderr) + ) { + rejectPipeline( + `Process pipeline stdio closed without exit status and stderr output: ${plan.source.command} -> ${plan.target.command}`, + false, + false + ); + return; + } + settled = true; + clearTimers(); + completeProcessState(sourceState); + completeProcessState(targetState); + resolve({ + source: toRunResult(sourceState), + target: toRunResult(targetState), + }); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + }; + const scheduleSuccessfulTargetFallback = () => { + if ( + settled || + successfulTargetFallbackTimer || + targetState.exitCode !== 0 || + sourceState.exitCode !== null || + !sourceStdoutClosed || + !sourceStderrClosed || + sourceState.stderr + ) { + return; + } + + successfulTargetFallbackTimer = setTimeout( + () => { + successfulTargetFallbackTimer = undefined; + if ( + settled || + targetState.exitCode !== 0 || + sourceState.exitCode !== null || + !sourceStdoutClosed || + !sourceStderrClosed || + sourceState.stderr + ) { + return; + } + settled = true; + clearTimers(); + completeProcessState(sourceState); + completeProcessState(targetState); + resolve({ + source: toRunResult(sourceState), + target: toRunResult(targetState), + }); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + }; + const endTargetStdin = () => { + if (targetStdinClosed) { + return; + } + const stdin = targetChild.stdin as typeof targetChild.stdin & { + destroyed?: boolean; + writableEnded?: boolean; + }; + if (stdin.destroyed || stdin.writableEnded) { + markTargetStdinClosed(); + return; + } + stdin.end(); + }; + const scheduleSourceInputFallback = () => { + if ( + settled || + sourceInputFallbackTimer || + sourceStdoutClosed || + targetStdinClosed || + sourceState.exitCode !== 0 + ) { + return; + } + sourceInputFallbackTimer = setTimeout( + () => { + sourceInputFallbackTimer = undefined; + if (settled || sourceStdoutClosed || targetStdinClosed || sourceState.exitCode !== 0) { + return; + } + endTargetStdin(); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + }; + const syncProcessExitStatus = () => { + const syncOne = (child: IProcessLike, state: IProcessState, setExited: () => void) => { + if (state.exitCode !== null || state.signal !== null) { + return false; + } + const exitCode = child.exitCode; + const signalCode = child.signalCode; + if (exitCode == null && signalCode == null) { + return false; + } + state.exitCode = exitCode ?? null; + state.signal = signalCode ?? null; + completeProcessState(state); + setExited(); + return true; + }; + const sourceChanged = syncOne(sourceChild, sourceState, () => { + sourceExited = true; + }); + const targetChanged = syncOne(targetChild, targetState, () => { + targetExited = true; + }); + if (!sourceChanged && !targetChanged) { + return; + } + scheduleSourceInputFallback(); + scheduleExitFallback(); + scheduleStdioFallback(); + scheduleSuccessfulTargetFallback(); + }; + const rejectMissingExitedProcesses = () => { + if (settled) { + return; + } + const sourceMissing = isProcessGoneWithoutExitStatus(sourceChild, sourceState); + const targetMissing = isProcessGoneWithoutExitStatus(targetChild, targetState); + if (!sourceMissing && !targetMissing) { + return; + } + rejectPipeline( + `Process pipeline exited without status: ${plan.source.command} -> ${plan.target.command}`, + !sourceMissing, + !targetMissing + ); + }; + const syncStdioState = () => { + const sourceStdout = sourceChild.stdout as IReadableState; + const sourceStderr = sourceChild.stderr as IReadableState | null | undefined; + const targetStdin = targetChild.stdin as IWritableState; + const targetStdout = targetChild.stdout as IReadableState | null | undefined; + const targetStderr = targetChild.stderr as IReadableState | null | undefined; + + if (sourceStdout.closed || sourceStdout.destroyed || sourceStdout.readableEnded) { + markSourceStdoutClosed(); + } + if ( + !sourceStderr || + sourceStderr.closed || + sourceStderr.destroyed || + sourceStderr.readableEnded + ) { + markSourceStderrClosed(); + } + if ( + targetStdin.closed || + targetStdin.destroyed || + targetStdin.writableEnded || + targetStdin.writableFinished + ) { + markTargetStdinClosed(); + } + if ( + !targetStdout || + targetStdout.closed || + targetStdout.destroyed || + targetStdout.readableEnded + ) { + markTargetStdoutClosed(); + } + if ( + !targetStderr || + targetStderr.closed || + targetStderr.destroyed || + targetStderr.readableEnded + ) { + markTargetStderrClosed(); + } + }; + const markSourceStdoutClosed = () => { + sourceStdoutClosed = true; + endTargetStdin(); + scheduleStdioFallback(); + scheduleSuccessfulTargetFallback(); + }; + const markSourceStderrClosed = () => { + sourceStderrClosed = true; + scheduleStdioFallback(); + scheduleSuccessfulTargetFallback(); + }; + const markTargetStdinClosed = () => { + targetStdinClosed = true; + scheduleStdioFallback(); + }; + const markTargetStdoutClosed = () => { + targetStdoutClosed = true; + scheduleStdioFallback(); + }; + const markTargetStderrClosed = () => { + targetStderrClosed = true; + scheduleStdioFallback(); + }; + + sourceChild.stdout.on('end', markSourceStdoutClosed); + sourceChild.stdout.on('close', markSourceStdoutClosed); + sourceChild.stderr?.on('end', markSourceStderrClosed); + sourceChild.stderr?.on('close', markSourceStderrClosed); + targetChild.stdin.on('finish', markTargetStdinClosed); + targetChild.stdin.on('close', markTargetStdinClosed); + targetChild.stdout?.on('end', markTargetStdoutClosed); + targetChild.stdout?.on('close', markTargetStdoutClosed); + targetChild.stderr?.on('end', markTargetStderrClosed); + targetChild.stderr?.on('close', markTargetStderrClosed); + timers.processStatePollTimer = setInterval( + () => { + if (settled) { + return; + } + syncStdioState(); + syncProcessExitStatus(); + rejectMissingExitedProcesses(); + }, + Math.max(1, options.exitFallbackMs ?? defaultExitFallbackMs) + ); + sourceChild.stdout.on('error', (error) => { + sourceState.stderr = appendBounded(sourceState.stderr, error.message, stderrLimit); + rejectPipeline( + `Source COPY stream failed: ${plan.source.command} ${sourceState.args.join(' ')}`, + false, + true + ); + }); + targetChild.stdin.on('error', (error) => { + targetState.stderr = appendBounded(targetState.stderr, error.message, stderrLimit); + rejectPipeline( + `Target COPY stream failed: ${plan.target.command} ${targetState.args.join(' ')}`, + true, + false + ); + }); + + sourceChild.on('error', (error) => { + sourceState.stderr = sourceState.stderr || redactSecrets(error.message); + rejectPipeline( + `Source process failed to start: ${plan.source.command} ${sourceState.args.join(' ')}`, + false, + true + ); + }); + targetChild.on('error', (error) => { + targetState.stderr = targetState.stderr || redactSecrets(error.message); + rejectPipeline( + `Target process failed to start: ${plan.target.command} ${targetState.args.join(' ')}`, + true, + false + ); + }); + sourceChild.on('exit', (exitCode, signal) => { + sourceExited = true; + sourceState.exitCode = exitCode; + sourceState.signal = signal; + completeProcessState(sourceState); + scheduleSourceInputFallback(); + scheduleExitFallback(); + scheduleStdioFallback(); + }); + targetChild.on('exit', (exitCode, signal) => { + targetExited = true; + targetState.exitCode = exitCode; + targetState.signal = signal; + completeProcessState(targetState); + scheduleExitFallback(); + scheduleSuccessfulTargetFallback(); + }); + + sourceChild.on('close', (exitCode, signal) => { + sourceClosed = true; + sourceExited = true; + sourceState.exitCode = exitCode; + sourceState.signal = signal; + completeProcessState(sourceState); + if (exitCode !== 0) { + rejectPipeline( + `Source process exited with code ${exitCode}: ${plan.source.command} ${sourceState.args.join(' ')}`, + false, + true + ); + return; + } + endTargetStdin(); + resolveIfComplete(); + }); + targetChild.on('close', (exitCode, signal) => { + targetClosed = true; + targetExited = true; + targetState.exitCode = exitCode; + targetState.signal = signal; + completeProcessState(targetState); + if (exitCode !== 0) { + rejectPipeline( + `Target process exited with code ${exitCode}: ${plan.target.command} ${targetState.args.join(' ')}`, + true, + false + ); + return; + } + resolveIfComplete(); + scheduleSuccessfulTargetFallback(); + }); + + sourceChild.stdout.pipe(targetChild.stdin); + }); + } +} diff --git a/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts b/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts new file mode 100644 index 0000000000..e4063ae996 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts @@ -0,0 +1,230 @@ +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IDataDbConnectionSummaryVo } from '@teable/openapi'; +import { extractForeignTableId } from '../base/cross-space-detection.util'; +import { decryptDataDbUrl } from './data-db-url-secret'; +import { fingerprintDatabaseUrl } from './data-db-preflight.service'; + +export type ISpaceDataDbRelatedSpaces = NonNullable; +export type ISpaceDataDbRelatedSpaceInfo = ISpaceDataDbRelatedSpaces['spaces'][number]; +export type ISpaceDataDbRelatedSpaceLink = ISpaceDataDbRelatedSpaces['links'][number]; + +type IRelatedFieldRow = { + id: string; + type: string; + isLookup: boolean | null; + isConditionalLookup: boolean | null; + options: string | null; + lookupOptions: string | null; + tableId: string; + table: { + id: string; + base: { + spaceId: string; + }; + }; +}; + +const dataDbMode = (mode: string | null | undefined): 'default' | 'byodb' => + mode === 'byodb' ? 'byodb' : 'default'; + +const dataDbBindingState = ( + state: string | null | undefined +): ISpaceDataDbRelatedSpaceInfo['dataDbState'] => { + if ( + state === 'ready' || + state === 'validating' || + state === 'initializing' || + state === 'migrating' || + state === 'error' || + state === 'disabled' + ) { + return state; + } + return undefined; +}; + +export const resolveSpaceDataDbRelatedSpaces = async ( + prismaService: PrismaService, + primarySpaceId: string +): Promise => { + const fields = (await prismaService.field.findMany({ + where: { + deletedTime: null, + table: { + deletedTime: null, + base: { + deletedTime: null, + space: { + deletedTime: null, + }, + }, + }, + OR: [ + { + type: FieldType.Link, + OR: [{ isLookup: false }, { isLookup: null }], + }, + { type: FieldType.ConditionalRollup }, + { isLookup: true, isConditionalLookup: true }, + ], + }, + select: { + id: true, + type: true, + isLookup: true, + isConditionalLookup: true, + options: true, + lookupOptions: true, + tableId: true, + table: { + select: { + id: true, + base: { + select: { + spaceId: true, + }, + }, + }, + }, + }, + })) as IRelatedFieldRow[]; + + const foreignTableIds = Array.from( + new Set(fields.map((field) => extractForeignTableId(field)).filter((id): id is string => !!id)) + ); + const foreignTables = foreignTableIds.length + ? await prismaService.tableMeta.findMany({ + where: { + id: { in: foreignTableIds }, + deletedTime: null, + base: { + deletedTime: null, + space: { + deletedTime: null, + }, + }, + }, + select: { + id: true, + base: { + select: { + spaceId: true, + }, + }, + }, + }) + : []; + const foreignSpaceByTableId = new Map( + foreignTables.map((table) => [table.id, table.base.spaceId]) + ); + + const links: ISpaceDataDbRelatedSpaceLink[] = []; + const adjacency = new Map>(); + const connect = (left: string, right: string) => { + if (left === right) return; + const leftSet = adjacency.get(left) ?? new Set(); + leftSet.add(right); + adjacency.set(left, leftSet); + const rightSet = adjacency.get(right) ?? new Set(); + rightSet.add(left); + adjacency.set(right, rightSet); + }; + + for (const field of fields) { + const foreignTableId = extractForeignTableId(field); + if (!foreignTableId) continue; + const fromSpaceId = field.table.base.spaceId; + const toSpaceId = foreignSpaceByTableId.get(foreignTableId); + if (!toSpaceId || fromSpaceId === toSpaceId) continue; + connect(fromSpaceId, toSpaceId); + links.push({ + fromSpaceId, + fromTableId: field.tableId, + fromFieldId: field.id, + toSpaceId, + toTableId: foreignTableId, + }); + } + + const relatedSpaceIds = new Set([primarySpaceId]); + const queue = [primarySpaceId]; + for (let index = 0; index < queue.length; index++) { + const current = queue[index]; + for (const next of adjacency.get(current) ?? []) { + if (relatedSpaceIds.has(next)) continue; + relatedSpaceIds.add(next); + queue.push(next); + } + } + + const [spaces, bindings] = await Promise.all([ + prismaService.space.findMany({ + where: { id: { in: [...relatedSpaceIds] }, deletedTime: null }, + select: { + id: true, + name: true, + baseGroup: { + where: { deletedTime: null }, + select: { + id: true, + tables: { + where: { deletedTime: null }, + select: { id: true }, + }, + }, + }, + }, + }), + prismaService.spaceDataDbBinding.findMany({ + where: { spaceId: { in: [...relatedSpaceIds] } }, + include: { dataDbConnection: true }, + }), + ]); + + const bindingBySpaceId = new Map(bindings.map((binding) => [binding.spaceId, binding])); + const relatedSpaces = spaces + .map((space) => { + const binding = bindingBySpaceId.get(space.id); + const connection = binding?.dataDbConnection; + return { + spaceId: space.id, + name: space.name, + isPrimary: space.id === primarySpaceId, + baseIds: space.baseGroup.map((base) => base.id).sort(), + tableIds: space.baseGroup.flatMap((base) => base.tables.map((table) => table.id)).sort(), + dataDbMode: dataDbMode(binding?.mode), + dataDbState: dataDbBindingState(binding?.state), + dataDbConnectionId: connection?.id ?? null, + dataDbUrlFingerprint: connection?.urlFingerprint ?? null, + dataDbDatabaseFingerprint: connection?.encryptedUrl + ? fingerprintDatabaseUrl(decryptDataDbUrl(connection.encryptedUrl)) + : null, + dataDbDisplayHost: connection?.displayHost ?? null, + dataDbDisplayDatabase: connection?.displayDatabase ?? null, + dataDbInternalSchema: connection?.internalSchema ?? null, + }; + }) + .sort((left, right) => + left.isPrimary === right.isPrimary + ? left.name.localeCompare(right.name) || left.spaceId.localeCompare(right.spaceId) + : left.isPrimary + ? -1 + : 1 + ); + + const componentIds = new Set(relatedSpaces.map((space) => space.spaceId)); + return { + primarySpaceId, + hasCrossSpaceLinks: relatedSpaces.length > 1, + spaces: relatedSpaces, + links: links + .filter((link) => componentIds.has(link.fromSpaceId) && componentIds.has(link.toSpaceId)) + .sort( + (left, right) => + left.fromSpaceId.localeCompare(right.fromSpaceId) || + left.toSpaceId.localeCompare(right.toSpaceId) || + left.fromFieldId.localeCompare(right.fromFieldId) + ), + }; +}; diff --git a/apps/nestjs-backend/src/features/space/space.controller.ts b/apps/nestjs-backend/src/features/space/space.controller.ts index 3c76151727..d012e7cacf 100644 --- a/apps/nestjs-backend/src/features/space/space.controller.ts +++ b/apps/nestjs-backend/src/features/space/space.controller.ts @@ -5,6 +5,9 @@ import type { ICreateSpaceVo, IUpdateSpaceVo, IGetSpaceVo, + IDataDbConnectionSummaryVo, + IDataDbMigrationJobStatusVo, + IDataDbPreflightVo, EmailInvitationVo, ListSpaceInvitationLinkVo, CreateSpaceInvitationLinkVo, @@ -17,6 +20,8 @@ import type { import { createSpaceRoSchema, ICreateSpaceRo, + dataDbPreflightRoSchema, + IDataDbPreflightRo, updateSpaceRoSchema, IUpdateSpaceRo, emailSpaceInvitationRoSchema, @@ -43,6 +48,7 @@ import { spaceSearchRoSchema, ISpaceSearchRo, } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../custom.exception'; import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; import { Events } from '../../event-emitter/events'; @@ -50,15 +56,46 @@ import { ZodValidationPipe } from '../../zod.validation.pipe'; import { Permissions } from '../auth/decorators/permissions.decorator'; import { CollaboratorService } from '../collaborator/collaborator.service'; import { InvitationService } from '../invitation/invitation.service'; +import { DataDbBindingService } from './data-db-binding.service'; +import { DataDbPreflightService } from './data-db-preflight.service'; +import { + migrateSpaceTargetMode, + spaceDataDbAdminOnlyErrorCode, + spaceDataDbAdminOnlyMessage, +} from './space-data-db-migration.constants'; +import { SpaceDataDbMigrationService } from './space-data-db-migration.service'; import { SpaceService } from './space.service'; + +const rejectSpaceDataDbMigrationFromSpaceApi = () => { + throw new CustomHttpException(spaceDataDbAdminOnlyMessage, HttpErrorCode.RESTRICTED_RESOURCE, { + errorCode: spaceDataDbAdminOnlyErrorCode, + }); +}; + @Controller('api/space/') export class SpaceController { constructor( - private readonly spaceService: SpaceService, - private readonly invitationService: InvitationService, - private readonly collaboratorService: CollaboratorService + protected readonly spaceService: SpaceService, + protected readonly invitationService: InvitationService, + protected readonly collaboratorService: CollaboratorService, + protected readonly dataDbPreflightService: DataDbPreflightService, + protected readonly dataDbBindingService: DataDbBindingService, + protected readonly cls: ClsService, + protected readonly spaceDataDbMigrationService: SpaceDataDbMigrationService ) {} + @Post('data-db/preflight') + @Permissions('space|create') + async preflightDataDb( + @Body(new ZodValidationPipe(dataDbPreflightRoSchema)) + dataDbPreflightRo: IDataDbPreflightRo + ): Promise { + if (dataDbPreflightRo.targetMode === migrateSpaceTargetMode) { + rejectSpaceDataDbMigrationFromSpaceApi(); + } + return await this.dataDbPreflightService.preflight(dataDbPreflightRo); + } + @Post() @Permissions('space|create') @EmitControllerEvent(Events.SPACE_CREATE) @@ -86,6 +123,84 @@ export class SpaceController { return await this.spaceService.getSpaceById(spaceId); } + @Permissions('space|read') + @Get(':spaceId/data-db') + async getSpaceDataDb(@Param('spaceId') spaceId: string): Promise { + return await this.dataDbPreflightService.getSummary(spaceId); + } + + @Permissions('space|update') + @Patch(':spaceId/data-db') + async updateSpaceDataDb( + @Param('spaceId') spaceId: string, + @Body(new ZodValidationPipe(dataDbPreflightRoSchema)) + dataDbPreflightRo: IDataDbPreflightRo + ): Promise { + if (dataDbPreflightRo.targetMode === migrateSpaceTargetMode) { + rejectSpaceDataDbMigrationFromSpaceApi(); + } + await this.dataDbBindingService.updateBindingForSpace( + spaceId, + this.cls.get('user.id') ?? '', + dataDbPreflightRo + ); + return await this.dataDbPreflightService.getSummary(spaceId); + } + + @Permissions('space|update') + @Post(':spaceId/data-db/retest') + async retestSpaceDataDb(@Param('spaceId') spaceId: string): Promise { + await this.dataDbBindingService.retestBinding(spaceId); + return await this.dataDbPreflightService.getSummary(spaceId); + } + + @Permissions('space|update') + @Post(':spaceId/data-db/retry') + async retrySpaceDataDbMigration( + @Param('spaceId') spaceId: string + ): Promise { + await this.dataDbBindingService.retryMigrationForSpace(spaceId); + return await this.dataDbPreflightService.getSummary(spaceId); + } + + @Permissions('space|read') + @Get(':spaceId/data-db/migration/:jobId') + async getSpaceDataDbMigration( + @Param('spaceId') spaceId: string, + @Param('jobId') jobId: string + ): Promise { + rejectSpaceDataDbMigrationFromSpaceApi(); + return await this.spaceDataDbMigrationService.getMigrationJobStatus(spaceId, jobId); + } + + @Permissions('space|update') + @Post(':spaceId/data-db/migration/:jobId/cancel') + async cancelSpaceDataDbMigration( + @Param('spaceId') spaceId: string, + @Param('jobId') jobId: string + ): Promise { + rejectSpaceDataDbMigrationFromSpaceApi(); + return await this.spaceDataDbMigrationService.cancelMigrationForSpace( + spaceId, + jobId, + this.cls.get('user.id') ?? '' + ); + } + + @Permissions('space|update') + @Post(':spaceId/data-db/migration/:jobId/rollback') + async rollbackSpaceDataDbMigration( + @Param('spaceId') spaceId: string, + @Param('jobId') jobId: string + ): Promise { + rejectSpaceDataDbMigrationFromSpaceApi(); + return await this.spaceDataDbMigrationService.rollbackMigrationForSpace( + spaceId, + jobId, + this.cls.get('user.id') ?? '' + ); + } + @Permissions('space|read') @Get() async getSpaceList(): Promise { diff --git a/apps/nestjs-backend/src/features/space/space.module.ts b/apps/nestjs-backend/src/features/space/space.module.ts index 27282ac5be..eccd05da88 100644 --- a/apps/nestjs-backend/src/features/space/space.module.ts +++ b/apps/nestjs-backend/src/features/space/space.module.ts @@ -1,13 +1,23 @@ import { Module } from '@nestjs/common'; +import { EventJobModule } from '../../event-emitter/event-job/event-job.module'; import { PermissionModule } from '../auth/permission.module'; +import { BASE_IMPORT_CSV_QUEUE } from '../base/base-import-processor/base-import-csv.processor'; +import { BASE_IMPORT_JUNCTION_CSV_QUEUE } from '../base/base-import-processor/base-import-junction.processor'; import { BaseModule } from '../base/base.module'; import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { TABLE_IMPORT_CSV_CHUNK_QUEUE } from '../import/open-api/import-csv-chunk.processor'; +import { TABLE_IMPORT_CSV_QUEUE } from '../import/open-api/import-csv.processor'; import { InvitationModule } from '../invitation/invitation.module'; import { SettingOpenApiModule } from '../setting/open-api/setting-open-api.module'; import { SettingModule } from '../setting/setting.module'; import { DataDbBaselineService } from './data-db-baseline.service'; import { DataDbBindingService } from './data-db-binding.service'; import { DataDbPreflightService } from './data-db-preflight.service'; +import { SpaceDataDbCopyService } from './space-data-db-copy.service'; +import { SpaceDataDbMigrationGuardService } from './space-data-db-migration-guard.service'; +import { SpaceDataDbMigrationWorkerService } from './space-data-db-migration-worker.service'; +import { SpaceDataDbMigrationService } from './space-data-db-migration.service'; +import { SpaceDataDbProcessRunnerService } from './space-data-db-process-runner.service'; import { SpaceController } from './space.controller'; import { SpaceService } from './space.service'; import { TemplateSpaceInitService } from './template-space-init/template-space.init.service'; @@ -20,6 +30,11 @@ import { TemplateSpaceInitService } from './template-space-init/template-space.i DataDbPreflightService, DataDbBaselineService, DataDbBindingService, + SpaceDataDbCopyService, + SpaceDataDbMigrationService, + SpaceDataDbMigrationWorkerService, + SpaceDataDbMigrationGuardService, + SpaceDataDbProcessRunnerService, ], exports: [ SpaceService, @@ -27,6 +42,11 @@ import { TemplateSpaceInitService } from './template-space-init/template-space.i DataDbPreflightService, DataDbBaselineService, DataDbBindingService, + SpaceDataDbCopyService, + SpaceDataDbMigrationService, + SpaceDataDbMigrationWorkerService, + SpaceDataDbMigrationGuardService, + SpaceDataDbProcessRunnerService, ], imports: [ SettingModule, @@ -35,6 +55,10 @@ import { TemplateSpaceInitService } from './template-space-init/template-space.i InvitationModule, BaseModule, PermissionModule, + EventJobModule.registerQueue(BASE_IMPORT_CSV_QUEUE), + EventJobModule.registerQueue(BASE_IMPORT_JUNCTION_CSV_QUEUE), + EventJobModule.registerQueue(TABLE_IMPORT_CSV_CHUNK_QUEUE), + EventJobModule.registerQueue(TABLE_IMPORT_CSV_QUEUE), ], }) export class SpaceModule {} diff --git a/apps/nestjs-backend/src/features/space/space.service.spec.ts b/apps/nestjs-backend/src/features/space/space.service.spec.ts index 124150b9d0..fcb6bc4d47 100644 --- a/apps/nestjs-backend/src/features/space/space.service.spec.ts +++ b/apps/nestjs-backend/src/features/space/space.service.spec.ts @@ -1,5 +1,6 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; +import { Role } from '@teable/core'; import { GlobalModule } from '../../global/global.module'; import { SpaceModule } from './space.module'; import { SpaceService } from './space.service'; @@ -18,4 +19,73 @@ describe('SpaceService', () => { it('should be defined', () => { expect(service).toBeDefined(); }); + + describe('getBaseListBySpaceId', () => { + it('returns v2 status for canary-space bases even when the base is not new-base v2', async () => { + const spaceId = 'spc1'; + const createdTime = new Date('2026-06-13T00:00:00.000Z'); + const base = { + id: 'bse1', + name: 'Base', + order: 1, + spaceId, + icon: null, + createdBy: 'usr1', + lastModifiedTime: createdTime, + createdTime, + v2Enabled: false, + }; + const prismaService = { + base: { + findMany: vi.fn().mockResolvedValue([base]), + }, + user: { + findMany: vi.fn().mockResolvedValue([{ id: 'usr1', name: 'Nee', avatar: null }]), + }, + baseShare: { + findMany: vi.fn().mockResolvedValue([]), + }, + }; + const collaboratorService = { + getCurrentUserCollaboratorsBaseAndSpaceArray: vi.fn().mockResolvedValue({ + spaceIds: [spaceId], + roleMap: { [spaceId]: Role.Owner }, + }), + }; + const baseService = { + enrichBaseListV2Status: vi.fn(async (baseList: (typeof base)[]) => + baseList.map((base) => ({ + ...base, + isCanary: true, + v2Status: { useV2: true, reason: 'space_feature' as const }, + })) + ), + }; + const testService = new SpaceService( + prismaService as never, + {} as never, + baseService as never, + collaboratorService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ); + + const result = await testService.getBaseListBySpaceId(spaceId); + + expect(baseService.enrichBaseListV2Status).toHaveBeenCalledWith([base]); + expect(result[0]).toMatchObject({ + id: base.id, + role: Role.Owner, + isCanary: true, + v2Status: { useV2: true, reason: 'space_feature' }, + }); + expect(result[0]).not.toHaveProperty('v2Enabled'); + }); + }); }); diff --git a/apps/nestjs-backend/src/features/space/space.service.ts b/apps/nestjs-backend/src/features/space/space.service.ts index f981f7f4cf..d13cda26a9 100644 --- a/apps/nestjs-backend/src/features/space/space.service.ts +++ b/apps/nestjs-backend/src/features/space/space.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable } from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import type { IRole } from '@teable/core'; import { HttpErrorCode, @@ -41,6 +41,7 @@ import { SettingOpenApiService } from '../setting/open-api/setting-open-api.serv import { SettingService } from '../setting/setting.service'; import { normalizeSpaceAIIntegrationConfig } from './ai-integration-config'; import { DataDbBindingService } from './data-db-binding.service'; +import { SpaceDataDbMigrationGuardService } from './space-data-db-migration-guard.service'; @Injectable() export class SpaceService { @@ -56,9 +57,15 @@ export class SpaceService { protected readonly dataDbBindingService: DataDbBindingService, @ThresholdConfig() protected readonly thresholdConfig: IThresholdConfig, @InjectModel('CUSTOM_KNEX') protected readonly knex: Knex, - @InjectDbProvider() protected readonly dbProvider: IDbProvider + @InjectDbProvider() protected readonly dbProvider: IDbProvider, + @Optional() + protected readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertSpaceWritable(spaceId: string) { + await this.spaceDataDbMigrationGuard?.assertSpaceWritable(spaceId); + } + protected supportsByodbSpaceCreation() { return false; } @@ -240,6 +247,7 @@ export class SpaceService { } async updateSpace(spaceId: string, updateSpaceRo: IUpdateSpaceRo) { + await this.assertSpaceWritable(spaceId); const userId = this.cls.get('user.id'); return await this.prismaService.space.update({ @@ -259,6 +267,7 @@ export class SpaceService { } async deleteSpace(spaceId: string) { + await this.assertSpaceWritable(spaceId); const userId = this.cls.get('user.id'); await this.prismaService.$tx(async () => { @@ -308,6 +317,7 @@ export class SpaceService { createdBy: true, lastModifiedTime: true, createdTime: true, + v2Enabled: true, }, where: { spaceId, @@ -333,11 +343,14 @@ export class SpaceService { const userMap = keyBy(userList, 'id'); const sharedBaseIds = new Set(sharedBaseList.map((s) => s.baseId)); - return baseList.map((base) => { + const baseListWithV2Status = await this.baseService.enrichBaseListV2Status(baseList); + + return baseListWithV2Status.map((base) => { + const { v2Enabled, ...baseInfo } = base; const role = roleMap[base.id] || roleMap[base.spaceId]; const createdUser = userMap[base.createdBy]; return { - ...base, + ...baseInfo, role, isShared: sharedBaseIds.has(base.id), lastModifiedTime: base.lastModifiedTime?.toISOString(), @@ -546,6 +559,7 @@ export class SpaceService { } async permanentDeleteSpace(spaceId: string, ignorePermissionCheck: boolean = false) { + await this.assertSpaceWritable(spaceId); if (!ignorePermissionCheck) { const accessTokenId = this.cls.get('accessTokenId'); await this.permissionService.validPermissions(spaceId, ['space|delete'], accessTokenId, true); diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts index 0712e15247..0754a65c14 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Inject, Injectable, Optional } from '@nestjs/common'; import { CellFormat, FieldKeyType, FieldType } from '@teable/core'; import type { IFieldRo, IFieldVo, ILinkFieldOptionsRo, IRecord } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; @@ -27,6 +27,7 @@ import type { IClsStore } from '../../../types/cls'; import { AuditScope } from '../../audit/audit-scope'; import { Audit } from '../../audit/audit.decorator'; import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { V2ContainerService } from '../../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; import { ViewService } from '../../view/view.service'; @@ -48,9 +49,20 @@ export class TableOpenApiV2Service { @InjectDbProvider() private readonly dbProvider: IDbProvider, private readonly tableDuplicateLegacyService: TableDuplicateService, private readonly audit: AuditScope, - private readonly cls: ClsService + private readonly cls: ClsService, + @Optional() + @Inject(SpaceDataDbMigrationGuardService) + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + private async collectCrossSpaceAffectedFields( tableId: string ): Promise> { @@ -91,6 +103,7 @@ export class TableOpenApiV2Service { ro as unknown as Record, }) async createTable(baseId: string, createTableRo: ICreateTableWithDefault): Promise { + await this.assertBaseWritable(baseId); const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); const context = await this.v2ContextFactory.createContext(container); @@ -123,6 +136,7 @@ export class TableOpenApiV2Service { tableId: string, mode: 'soft' | 'permanent' = 'soft' ): Promise { + await this.assertBaseWritable(baseId); const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); const context = await this.v2ContextFactory.createContext(container); @@ -149,6 +163,7 @@ export class TableOpenApiV2Service { } async restoreTable(baseId: string, tableId: string): Promise { + await this.assertBaseWritable(baseId); const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); const context = await this.v2ContextFactory.createContext(container); @@ -185,6 +200,8 @@ export class TableOpenApiV2Service { tableId: string, duplicateTableRo: IDuplicateTableRo ): Promise { + await this.assertBaseWritable(baseId); + await this.assertTableWritable(tableId); // The v2 duplicate command does not run cross-space validation when // creating fields, so a table containing any cross-space link would // silently produce another cross-space copy. Delegate to the v1 path, diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts index 86297db571..9bc7b5a69f 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts @@ -1,4 +1,4 @@ -import { NotFoundException, Injectable, Logger } from '@nestjs/common'; +import { Inject, NotFoundException, Injectable, Logger, Optional } from '@nestjs/common'; import type { FieldAction, IFieldRo, @@ -60,6 +60,7 @@ import { createFieldInstanceByVo } from '../../field/model/factory'; import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; import { RecordService } from '../../record/record.service'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; import { TableDuplicateService } from '../table-duplicate.service'; import { TableService } from '../table.service'; @@ -87,9 +88,20 @@ export class TableOpenApiService { private readonly cls: ClsService, private readonly eventEmitterService: EventEmitterService, private readonly tableMutationCacheInvalidator: TableMutationCacheInvalidator, - private readonly audit: AuditScope + private readonly audit: AuditScope, + @Optional() + @Inject(SpaceDataDbMigrationGuardService) + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + private async createView(tableId: string, viewRos: IViewRo[]) { const viewCreationPromises = viewRos.map(async (viewRo) => { return this.viewOpenApiService.createView(tableId, viewRo); @@ -280,6 +292,7 @@ export class TableOpenApiService { } async createTable(baseId: string, tableRo: ICreateTableWithDefault): Promise { + await this.assertBaseWritable(baseId); let createdTable: ITableVo | undefined; const schema = await this.prismaService .$tx(async () => { @@ -364,6 +377,8 @@ export class TableOpenApiService { } async duplicateTable(baseId: string, tableId: string, tableRo: IDuplicateTableRo) { + await this.assertBaseWritable(baseId); + await this.assertTableWritable(tableId); return await this.tableDuplicateService.duplicateTable(baseId, tableId, tableRo); } @@ -427,6 +442,7 @@ export class TableOpenApiService { } async permanentDeleteTables(baseId: string, tableIds: string[]) { + await this.assertBaseWritable(baseId); // If the table has already been deleted, exceptions may occur // If the table hasn't been deleted and permanent deletion is executed directly, // we need to handle the deletion of associated data @@ -575,6 +591,7 @@ export class TableOpenApiService { } async deleteTable(baseId: string, tableId: string) { + await this.assertBaseWritable(baseId); try { await this.detachLink(tableId); } catch (e) { @@ -604,6 +621,7 @@ export class TableOpenApiService { } async restoreTable(baseId: string, tableId: string) { + await this.assertBaseWritable(baseId); return await this.prismaService.$tx( async (prisma) => { const { deletedTime } = await prisma.trash.findFirstOrThrow({ @@ -684,6 +702,7 @@ export class TableOpenApiService { } async updateDbTableName(baseId: string, tableId: string, dbTableNameRo: string) { + await this.assertBaseWritable(baseId); const dbTableName = this.dbProvider.joinDbTableName(baseId, dbTableNameRo); const existDbTableName = await this.prismaService.tableMeta .findFirst({ @@ -800,6 +819,7 @@ export class TableOpenApiService { } async shuffle(baseId: string) { + await this.assertBaseWritable(baseId); const tables = await this.prismaService.tableMeta.findMany({ where: { baseId, deletedTime: null, provisionState: ProvisionState.ready }, select: { id: true }, @@ -817,6 +837,7 @@ export class TableOpenApiService { } async updateOrder(baseId: string, tableId: string, orderRo: IUpdateOrderRo) { + await this.assertBaseWritable(baseId); const { anchorId, position } = orderRo; const tablesOrder = await this.prismaService.txClient().tableMeta.findMany({ diff --git a/apps/nestjs-backend/src/features/table/table-duplicate.service.spec.ts b/apps/nestjs-backend/src/features/table/table-duplicate.service.spec.ts index b51607352f..8fc2b9cf58 100644 --- a/apps/nestjs-backend/src/features/table/table-duplicate.service.spec.ts +++ b/apps/nestjs-backend/src/features/table/table-duplicate.service.spec.ts @@ -2,8 +2,100 @@ import { CellValueType, DbFieldType, FieldType, Relationship } from '@teable/cor import Knex from 'knex'; import type { Knex as KnexType } from 'knex'; import { vi } from 'vitest'; +import { DuplicateTableQueryPostgres } from '../../db-provider/duplicate-table/duplicate-query.postgres'; import { TableDuplicateService } from './table-duplicate.service'; +describe('TableDuplicateService.duplicateTableData', () => { + it('skips target-only columns when copying rows', async () => { + const dataKnex = Knex({ client: 'pg' }); + const executedSql: string[] = []; + const sourceColumnSql = 'source-column-info'; + const targetColumnSql = 'target-column-info'; + const dataPrisma = { + $queryRawUnsafe: vi.fn(async (sql: string): Promise => { + if (sql === sourceColumnSql) { + return [ + { name: '__id' }, + { name: '__version' }, + { name: '__auto_number' }, + { name: 'Name' }, + { name: 'Visible_Fields' }, + ] as T; + } + if (sql === targetColumnSql) { + return [ + { name: '__id' }, + { name: '__version' }, + { name: '__auto_number' }, + { name: 'Name' }, + { name: 'Visible_Fields' }, + { name: 'AJ_Story_Fields' }, + ] as T; + } + if (sql.includes('count(*)')) { + return [{ count: 1 }] as T; + } + return [] as T; + }), + $executeRawUnsafe: vi.fn(async (sql: string) => { + executedSql.push(sql); + return 1; + }), + }; + const fieldFindMany = vi.fn().mockResolvedValue([]); + const service = Object.create(TableDuplicateService.prototype) as TableDuplicateService; + + ( + service as unknown as { + prismaService: { + txClient: () => { + tableMeta: { findFirst: () => Promise<{ id: string }> }; + field: { findMany: typeof fieldFindMany }; + }; + }; + dbProvider: { + columnInfo: (tableName: string) => string; + duplicateTableQuery: (queryBuilder: KnexType.QueryBuilder) => DuplicateTableQueryPostgres; + }; + dataKnex: KnexType; + } + ).prismaService = { + txClient: () => ({ + tableMeta: { findFirst: async () => ({ id: 'tblTarget' }) }, + field: { findMany: fieldFindMany }, + }), + }; + ( + service as unknown as { + dbProvider: { + columnInfo: (tableName: string) => string; + duplicateTableQuery: (queryBuilder: KnexType.QueryBuilder) => DuplicateTableQueryPostgres; + }; + } + ).dbProvider = { + columnInfo: (tableName: string) => + tableName === 'bseSource.SourceTable' ? sourceColumnSql : targetColumnSql, + duplicateTableQuery: (queryBuilder) => new DuplicateTableQueryPostgres(queryBuilder), + }; + (service as unknown as { dataKnex: KnexType }).dataKnex = dataKnex; + + await service.duplicateTableData( + 'bseSource.SourceTable', + 'bseTarget.TargetTable', + {}, + {}, + [], + dataPrisma + ); + + expect(executedSql).toHaveLength(1); + expect(executedSql[0]).toContain('"Name", "Visible_Fields"'); + expect(executedSql[0]).not.toContain('AJ_Story_Fields'); + + await dataKnex.destroy(); + }); +}); + describe('TableDuplicateService.duplicateLinkJunction', () => { const createLinkFieldRaw = ({ id, diff --git a/apps/nestjs-backend/src/features/table/table-duplicate.service.ts b/apps/nestjs-backend/src/features/table/table-duplicate.service.ts index 6140c764d9..24e0f3af3e 100644 --- a/apps/nestjs-backend/src/features/table/table-duplicate.service.ts +++ b/apps/nestjs-backend/src/features/table/table-duplicate.service.ts @@ -334,6 +334,8 @@ export class TableDuplicateService { !name.startsWith('__fk_fld') && !computedSet.has(name) ); + const sourceColumnSet = new Set(allSourceColumns); + const copyableFieldColumns = newFieldColumns.filter((name) => sourceColumnSet.has(name)); const oldFkColumns = oldOriginColumns.filter((name) => name.startsWith('__fk_fld')); @@ -381,12 +383,12 @@ export class TableDuplicateService { // use new table field columns info // old table contains ghost columns or customer columns - const oldColumns = newFieldColumns + const oldColumns = copyableFieldColumns .concat(oldRowColumns) .concat(oldFkColumns) .filter((dbFieldName) => !excludeColumnsSet.has(dbFieldName)); - const newColumns = newFieldColumns + const newColumns = copyableFieldColumns .concat(newRowColumns) .concat(newFkColumns) .filter((dbFieldName) => !excludeColumnsSet.has(dbFieldName)); diff --git a/apps/nestjs-backend/src/features/table/table-freeze.service.spec.ts b/apps/nestjs-backend/src/features/table/table-freeze.service.spec.ts new file mode 100644 index 0000000000..7056a261f2 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/table-freeze.service.spec.ts @@ -0,0 +1,57 @@ +import { HttpErrorCode } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CustomHttpException } from '../../custom.exception'; +import { TableService } from './table.service'; + +describe('TableService write freeze', () => { + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + const prismaService = { + txClient: vi.fn(), + }; + const batchService = { + saveRawOps: vi.fn(), + }; + const migrationGuard = { + assertBaseWritable: vi.fn(), + }; + + const service = () => + new TableService( + { get: vi.fn().mockReturnValue('usrxxx') } as never, + prismaService as never, + {} as never, + batchService as never, + { driver: 'pg' } as never, + {} as never, + migrationGuard as never + ); + + beforeEach(() => { + vi.clearAllMocks(); + migrationGuard.assertBaseWritable.mockRejectedValue(freezeError); + }); + + it('rejects table creation before locking or metadata writes when the base space is migrating', async () => { + await expect( + service().createTable('bsexxx', { name: 'Blocked table', fields: [] } as never) + ).rejects.toBe(freezeError); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledWith('bsexxx'); + expect(prismaService.txClient).not.toHaveBeenCalled(); + expect(batchService.saveRawOps).not.toHaveBeenCalled(); + }); + + it('rejects table deletion before metadata writes when the base space is migrating', async () => { + await expect(service().deleteTable('bsexxx', 'tblxxx', new Date())).rejects.toBe(freezeError); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledWith('bsexxx'); + expect(prismaService.txClient).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/table/table.service.ts b/apps/nestjs-backend/src/features/table/table.service.ts index 1e4808a6cc..f201e60de1 100644 --- a/apps/nestjs-backend/src/features/table/table.service.ts +++ b/apps/nestjs-backend/src/features/table/table.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import type { IOtOperation, ISnapshotBase } from '@teable/core'; import { DriverClient, @@ -26,6 +26,7 @@ import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; import { convertNameToValidCharacter } from '../../utils/name-conversion'; import { BatchService } from '../calculation/batch.service'; +import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service'; type IDataPrismaExecutor = { $executeRawUnsafe(query: string, ...values: unknown[]): PromiseLike; @@ -45,9 +46,15 @@ export class TableService implements IReadonlyAdapterService { private readonly dataDbClientManager: DataDbClientManager, private readonly batchService: BatchService, @InjectDbProvider() private readonly dbProvider: IDbProvider, - @InjectModel(DATA_KNEX) private readonly knex: Knex + @InjectModel(DATA_KNEX) private readonly knex: Knex, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + generateValidName(name: string) { return convertNameToValidCharacter(name, 40); } @@ -82,6 +89,7 @@ export class TableService implements IReadonlyAdapterService { private async createDBTable(baseId: string, tableRo: ICreateTableRo, createTable = true) { const userId = this.cls.get('user.id'); + await this.assertBaseWritable(baseId); await this.lockBaseRow(baseId); const tableRaws = await this.prismaService.txClient().tableMeta.findMany({ @@ -305,6 +313,7 @@ export class TableService implements IReadonlyAdapterService { } async deleteTable(baseId: string, tableId: string, deletedTime: Date) { + await this.assertBaseWritable(baseId); const result = await this.prismaService.txClient().tableMeta.findFirst({ where: { id: tableId, baseId, deletedTime: null }, }); @@ -340,6 +349,7 @@ export class TableService implements IReadonlyAdapterService { } async restoreTable(baseId: string, tableId: string) { + await this.assertBaseWritable(baseId); const result = await this.prismaService.txClient().tableMeta.findFirst({ where: { id: tableId, baseId, deletedTime: { not: null } }, }); @@ -386,6 +396,7 @@ export class TableService implements IReadonlyAdapterService { | 'views' > ) { + await this.assertBaseWritable(baseId); const select = Object.keys(input).reduce<{ [key: string]: boolean }>((acc, key) => { acc[key] = true; return acc; diff --git a/apps/nestjs-backend/src/features/trash/trash-freeze.service.spec.ts b/apps/nestjs-backend/src/features/trash/trash-freeze.service.spec.ts new file mode 100644 index 0000000000..e9261183f3 --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/trash-freeze.service.spec.ts @@ -0,0 +1,148 @@ +import { HttpErrorCode } from '@teable/core'; +import { TableTrashType, TrashType } from '@teable/openapi'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CustomHttpException } from '../../custom.exception'; +import { TrashService } from './trash.service'; + +describe('TrashService write freeze', () => { + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + const txClient = { + trash: { + findUniqueOrThrow: vi.fn(), + deleteMany: vi.fn(), + }, + }; + const prismaService = { + $tx: vi.fn(async (fn: (client: typeof txClient) => Promise) => fn(txClient)), + $queryRawUnsafe: vi.fn().mockResolvedValue([]), + txClient: vi.fn(() => txClient), + }; + const dataDbClientManager = { + dataPrismaForTable: vi.fn(), + }; + const spaceService = { + permanentDeleteSpace: vi.fn(), + }; + const baseService = { + permanentDeleteBase: vi.fn(), + }; + const tableOpenApiService = { + permanentDeleteTables: vi.fn(), + restoreTable: vi.fn(), + }; + const migrationGuard = { + assertSpaceWritable: vi.fn(), + assertBaseWritable: vi.fn(), + assertTableWritable: vi.fn(), + }; + + const service = () => + new TrashService( + { del: vi.fn() } as never, + prismaService as never, + { get: vi.fn().mockReturnValue('usrxxx') } as never, + {} as never, + { validPermissions: vi.fn() } as never, + spaceService as never, + baseService as never, + tableOpenApiService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never, + {} as never, + {} as never, + migrationGuard as never + ); + + beforeEach(() => { + vi.clearAllMocks(); + migrationGuard.assertSpaceWritable.mockRejectedValue(freezeError); + migrationGuard.assertBaseWritable.mockRejectedValue(freezeError); + migrationGuard.assertTableWritable.mockRejectedValue(freezeError); + txClient.trash.findUniqueOrThrow.mockResolvedValue({ + id: 'trhxxx', + resourceId: 'bsexxx', + resourceType: TrashType.Base, + parentId: null, + }); + }); + + it('rejects meta trash restore before restoring or deleting trash metadata', async () => { + await expect(service().restoreTrash('trhxxx')).rejects.toBe(freezeError); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledWith('bsexxx'); + expect(baseService.permanentDeleteBase).not.toHaveBeenCalled(); + expect(tableOpenApiService.restoreTable).not.toHaveBeenCalled(); + expect(txClient.trash.deleteMany).not.toHaveBeenCalled(); + }); + + it('rejects data-plane table trash restore before reading target table trash rows', async () => { + await expect(service().restoreTableResource('optrhxxx', 'tblxxx')).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(dataDbClientManager.dataPrismaForTable).not.toHaveBeenCalled(); + }); + + it('rejects trash reset before resetting table trash rows', async () => { + await expect( + service().resetTrashItems({ resourceType: TrashType.Table, resourceId: 'tblxxx' }) + ).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(dataDbClientManager.dataPrismaForTable).not.toHaveBeenCalled(); + }); + + it('rejects permanent delete before deleting base or table resources', async () => { + await expect( + service().deleteResource({ + resourceType: TrashType.Table, + resourceId: 'tblxxx', + parentId: 'bsexxx', + }) + ).rejects.toBe(freezeError); + await expect( + service().deleteResource({ resourceType: TrashType.Base, resourceId: 'bsexxx' }) + ).rejects.toBe(freezeError); + + expect(migrationGuard.assertBaseWritable).toHaveBeenCalledWith('bsexxx'); + expect(tableOpenApiService.permanentDeleteTables).not.toHaveBeenCalled(); + expect(baseService.permanentDeleteBase).not.toHaveBeenCalled(); + }); + + it('rejects restoring table-scoped trash snapshots before field, view, or record writes', async () => { + const tableTrash = { + tableId: 'tblxxx', + resourceType: TableTrashType.Field, + snapshot: JSON.stringify([{ id: 'fldxxx' }]), + createdTime: new Date(), + }; + dataDbClientManager.dataPrismaForTable.mockResolvedValue({ + tableTrash: { + findUniqueOrThrow: vi.fn().mockResolvedValue(tableTrash), + delete: vi.fn(), + }, + recordTrash: { + findMany: vi.fn(), + deleteMany: vi.fn(), + }, + }); + + await expect(service().restoreTableResource('optrhxxx', 'tblxxx')).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(dataDbClientManager.dataPrismaForTable).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/trash/trash.service.ts b/apps/nestjs-backend/src/features/trash/trash.service.ts index 56f6056010..c4c0799e12 100644 --- a/apps/nestjs-backend/src/features/trash/trash.service.ts +++ b/apps/nestjs-backend/src/features/trash/trash.service.ts @@ -1,8 +1,7 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable } from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import type { FieldType, IFieldVo } from '@teable/core'; import { FieldKeyType, HttpErrorCode, IdPrefix, Role } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService, type Prisma } from '@teable/db-main-prisma'; import type { IResetTrashItemsRo, @@ -40,6 +39,7 @@ import { CanaryService, type IV2Decision } from '../canary/canary.service'; import { FieldOpenApiService } from '../field/open-api/field-open-api.service'; import { RecordOpenApiService } from '../record/open-api/record-open-api.service'; import { RecordService } from '../record/record.service'; +import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service'; import { SpaceService } from '../space/space.service'; import { TableOpenApiV2Service } from '../table/open-api/table-open-api-v2.service'; import { TableOpenApiService } from '../table/open-api/table-open-api.service'; @@ -52,15 +52,54 @@ import { resolveV2TrashRecordDisplayName } from './v2-trash-record-name'; type IRecordTrashSnapshot = IDeleteRecordsPayload['records'][number]; +type ITableTrashDelegate = { + findMany(args: TArgs): Promise< + Array<{ + id: string; + tableId: string; + resourceType: string; + snapshot: string; + createdBy: string; + createdTime: Date; + }> + >; + findUniqueOrThrow(args: TArgs): Promise<{ + tableId: string; + resourceType: string; + snapshot: string; + createdTime: Date; + }>; + delete(args: TArgs): Promise; + deleteMany(args: TArgs): Promise; +}; + +type IRecordTrashDelegate = { + findMany(args: TArgs): Promise< + Array<{ + id: string; + recordId: string; + snapshot: string; + createdTime: Date; + }> + >; + deleteMany(args: TArgs): Promise; +}; + type ITrashDataPrisma = { - tableTrash: DataPrismaService['tableTrash']; - recordTrash: DataPrismaService['recordTrash']; + tableTrash: ITableTrashDelegate; + recordTrash: IRecordTrashDelegate; }; type IScopedTrashDataPrisma = ITrashDataPrisma & { txClient?: () => ITrashDataPrisma; - $tx?: DataPrismaService['$tx']; - $transaction?: DataPrismaService['$transaction']; + $tx?: ( + fn: (prisma: ITrashDataPrisma) => Promise, + options?: { timeout?: number } + ) => Promise; + $transaction?: ( + fn: (prisma: ITrashDataPrisma) => Promise, + options?: { timeout?: number } + ) => Promise; }; @Injectable() @@ -84,9 +123,40 @@ export class TrashService { protected readonly canaryService: CanaryService, protected readonly dataDbClientManager: DataDbClientManager, @ThresholdConfig() protected readonly thresholdConfig: IThresholdConfig, - @InjectModel(META_KNEX) protected readonly knex: Knex + @InjectModel(META_KNEX) protected readonly knex: Knex, + @Optional() + protected readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertSpaceWritable(spaceId: string) { + await this.spaceDataDbMigrationGuard?.assertSpaceWritable(spaceId); + } + + private async assertBaseWritable(baseId: string) { + await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId); + } + + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + + private async assertTrashResourceWritable( + resourceType: TrashType, + resourceId: string, + parentId?: string | null + ) { + switch (resourceType) { + case TrashType.Space: + return await this.assertSpaceWritable(resourceId); + case TrashType.Base: + return await this.assertBaseWritable(resourceId); + case TrashType.Table: + return parentId + ? await this.assertBaseWritable(parentId) + : await this.assertTableWritable(resourceId); + } + } + private getTrashDataPrismaExecutor(prisma: IScopedTrashDataPrisma): ITrashDataPrisma { return prisma.txClient?.() ?? prisma; } @@ -770,6 +840,7 @@ export class TrashService { }); } + await this.assertBaseWritable(decision.baseId); await this.assertParentNotTrashed(decision.baseId); await this.restoreTableV2(decision.baseId, decision.tableId); } @@ -783,6 +854,7 @@ export class TrashService { async restoreResource(trash: { resourceType: TrashType; resourceId: string }) { const { resourceType, resourceId } = trash; + await this.assertTrashResourceWritable(resourceType, resourceId); switch (resourceType) { case TrashType.Space: return this.restoreSpace(resourceId); @@ -816,6 +888,7 @@ export class TrashService { } ); } + await this.assertTableWritable(routedTableId); const lookupDataPrisma = this.getTrashDataPrismaExecutor( await this.trashDataPrismaForTable(routedTableId) ); @@ -846,6 +919,9 @@ export class TrashService { } ); }); + if (tableId !== routedTableId) { + await this.assertTableWritable(tableId); + } const dataPrisma = routedTableId ? lookupDataPrisma : this.getTrashDataPrismaExecutor(await this.trashDataPrismaForTable(tableId)); @@ -1053,6 +1129,11 @@ export class TrashService { }); await this.assertParentNotTrashed(trash.parentId); + await this.assertTrashResourceWritable( + trash.resourceType as TrashType, + trash.resourceId, + trash.parentId + ); await this.restoreResource({ resourceType: trash.resourceType as TrashType, @@ -1107,6 +1188,8 @@ export class TrashService { ); } + await this.assertTrashResourceWritable(resourceType, resourceId); + if (resourceType === TrashType.Base) { await this.resetBaseTrashResource(resetTrashItemsRo); } @@ -1219,6 +1302,7 @@ export class TrashService { ignorePermissionCheck = false ): Promise { const { resourceType, resourceId, parentId } = trash; + await this.assertTrashResourceWritable(resourceType, resourceId, parentId); switch (resourceType) { case TrashType.Space: diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo-freeze.service.spec.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo-freeze.service.spec.ts new file mode 100644 index 0000000000..1b8aa5b32e --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo-freeze.service.spec.ts @@ -0,0 +1,117 @@ +import { HttpErrorCode } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CustomHttpException } from '../../../custom.exception'; +import { UndoRedoService } from './undo-redo.service'; + +describe('UndoRedoService write freeze', () => { + const freezeError = new CustomHttpException( + 'Space data database migration is in progress', + HttpErrorCode.CONFLICT, + { + errorCode: 'SPACE_DATA_DB_MIGRATING', + migrationJobId: 'sdmjxxx', + } + ); + const v2ContainerService = { + getContainerForTable: vi.fn(), + }; + const v2ContextFactory = { + createContext: vi.fn(), + }; + const cls = { + get: vi.fn(), + }; + const cacheService = { + get: vi.fn(), + }; + const undoRedoStackService = { + popUndo: vi.fn(), + popRedo: vi.fn(), + }; + const undoRedoOperationService = { + undo: vi.fn(), + redo: vi.fn(), + }; + const migrationGuard = { + assertTableWritable: vi.fn(), + }; + + const service = () => + new UndoRedoService( + v2ContainerService as never, + v2ContextFactory as never, + cls as never, + cacheService as never, + undoRedoStackService as never, + undoRedoOperationService as never, + migrationGuard as never + ); + + beforeEach(() => { + vi.clearAllMocks(); + migrationGuard.assertTableWritable.mockRejectedValue(freezeError); + }); + + it('rejects undo before reading engine preference or popping the undo stack', async () => { + await expect(service().undo('tblxxx', 'winxxx')).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(cls.get).not.toHaveBeenCalled(); + expect(cacheService.get).not.toHaveBeenCalled(); + expect(undoRedoStackService.popUndo).not.toHaveBeenCalled(); + expect(v2ContainerService.getContainerForTable).not.toHaveBeenCalled(); + }); + + it('rejects redo before reading engine preference or popping the redo stack', async () => { + await expect(service().redo('tblxxx', 'winxxx')).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(cls.get).not.toHaveBeenCalled(); + expect(cacheService.get).not.toHaveBeenCalled(); + expect(undoRedoStackService.popRedo).not.toHaveBeenCalled(); + expect(v2ContainerService.getContainerForTable).not.toHaveBeenCalled(); + }); + + it('rejects streaming undo and redo before starting stack replay', async () => { + const undoIterator = service().undoStream('tblxxx', 'winxxx')[Symbol.asyncIterator](); + const redoIterator = service().redoStream('tblxxx', 'winxxx')[Symbol.asyncIterator](); + + await expect(undoIterator.next()).rejects.toBe(freezeError); + await expect(redoIterator.next()).rejects.toBe(freezeError); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledTimes(2); + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(cls.get).not.toHaveBeenCalled(); + expect(cacheService.get).not.toHaveBeenCalled(); + expect(undoRedoStackService.popUndo).not.toHaveBeenCalled(); + expect(undoRedoStackService.popRedo).not.toHaveBeenCalled(); + expect(v2ContainerService.getContainerForTable).not.toHaveBeenCalled(); + }); + + it('allows non-migrating undo to continue through the normal stack path', async () => { + const push = vi.fn(); + const operation = { id: 'opxxx' }; + const reverseOperation = { id: 'opreverse' }; + + migrationGuard.assertTableWritable.mockResolvedValue(undefined); + cls.get.mockReturnValue('usrxxx'); + cacheService.get.mockResolvedValue('v1'); + undoRedoStackService.popUndo.mockResolvedValue({ + operation, + push, + }); + undoRedoOperationService.undo.mockResolvedValue(reverseOperation); + + await expect(service().undo('tblxxx', 'winxxx')).resolves.toEqual({ + body: { + status: 'fulfilled', + }, + engine: 'v1', + }); + + expect(migrationGuard.assertTableWritable).toHaveBeenCalledWith('tblxxx'); + expect(undoRedoStackService.popUndo).toHaveBeenCalledWith('tblxxx', 'winxxx'); + expect(undoRedoOperationService.undo).toHaveBeenCalledWith(operation); + expect(push).toHaveBeenCalledWith(reverseOperation); + }); +}); diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts index 89a48f1acf..adb6889e79 100644 --- a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import type { IRedoVo, IUndoRedoStreamEvent, IUndoVo } from '@teable/openapi'; import { RedoCommand, @@ -18,6 +18,7 @@ import { ClsService } from 'nestjs-cls'; import { CacheService } from '../../../cache/cache.service'; import type { ICacheStore } from '../../../cache/types'; import type { IClsStore } from '../../../types/cls'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { V2ContainerService } from '../../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; import { UndoRedoOperationService } from '../stack/undo-redo-operation.service'; @@ -93,10 +94,14 @@ export class UndoRedoService { private readonly cls: ClsService, private readonly cacheService: CacheService, private readonly undoRedoStackService: UndoRedoStackService, - private readonly undoRedoOperationService: UndoRedoOperationService + private readonly undoRedoOperationService: UndoRedoOperationService, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} async undo(tableId: string, windowId: string): Promise> { + await this.assertTableWritable(tableId); + const preferredEngine = await this.getPreferredEngine(tableId, windowId); if (preferredEngine === 'v1') { const v1Result = await this.executeV1Undo(tableId, windowId); @@ -121,6 +126,8 @@ export class UndoRedoService { } async redo(tableId: string, windowId: string): Promise> { + await this.assertTableWritable(tableId); + const preferredEngine = await this.getPreferredEngine(tableId, windowId); if (preferredEngine === 'v1') { const v1Result = await this.executeV1Redo(tableId, windowId); @@ -145,13 +152,21 @@ export class UndoRedoService { } async *undoStream(tableId: string, windowId: string): AsyncIterable { + await this.assertTableWritable(tableId); + yield* this.executeUndoRedoStream(tableId, windowId, 'undo'); } async *redoStream(tableId: string, windowId: string): AsyncIterable { + await this.assertTableWritable(tableId); + yield* this.executeUndoRedoStream(tableId, windowId, 'redo'); } + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + private async *executeUndoRedoStream( tableId: string, windowId: string, diff --git a/apps/nestjs-backend/src/features/user/user.service.spec.ts b/apps/nestjs-backend/src/features/user/user.service.spec.ts index a8eaf2d610..15d0fb60a5 100644 --- a/apps/nestjs-backend/src/features/user/user.service.spec.ts +++ b/apps/nestjs-backend/src/features/user/user.service.spec.ts @@ -1,5 +1,6 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; +import { vi } from 'vitest'; import { GlobalModule } from '../../global/global.module'; import { UserModule } from './user.module'; import { UserService } from './user.service'; @@ -7,6 +8,39 @@ import { UserService } from './user.service'; describe('UserService', () => { let service: UserService; + const createNotifyMetaMergeService = (notifyMeta?: string | null) => { + const queryRaw = vi.fn().mockResolvedValue([{ notifyMeta }]); + const update = vi.fn().mockResolvedValue(undefined); + const tx = vi.fn(async (fn: () => Promise) => await fn()); + const prismaService = { + $tx: tx, + txClient: () => ({ + $queryRaw: queryRaw, + user: { + update, + }, + }), + }; + const mergeService = new UserService( + prismaService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ); + + return { + mergeService, + queryRaw, + tx, + update, + }; + }; + beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ imports: [GlobalModule, UserModule], @@ -18,4 +52,49 @@ describe('UserService', () => { it('should be defined', () => { expect(service).toBeDefined(); }); + + it('merges notify meta updates with existing values', async () => { + const { mergeService, queryRaw, tx, update } = createNotifyMetaMergeService( + JSON.stringify({ email: true, appBuilderChatIntroDismissed: true }) + ); + + await mergeService.updateNotifyMeta('usrTest', { email: false }); + + expect(tx).toHaveBeenCalledTimes(1); + expect(queryRaw).toHaveBeenCalledTimes(1); + expect(queryRaw.mock.calls[0][0].join('')).toContain('FOR UPDATE'); + expect(queryRaw.mock.calls[0][1]).toBe('usrTest'); + expect(update).toHaveBeenCalledWith({ + data: { + notifyMeta: JSON.stringify({ email: false, appBuilderChatIntroDismissed: true }), + }, + where: { id: 'usrTest', deletedTime: null }, + }); + }); + + it('keeps existing notify switches when dismissing app builder intro', async () => { + const { mergeService, update } = createNotifyMetaMergeService(JSON.stringify({ email: true })); + + await mergeService.updateNotifyMeta('usrTest', { appBuilderChatIntroDismissed: true }); + + expect(update).toHaveBeenCalledWith({ + data: { + notifyMeta: JSON.stringify({ email: true, appBuilderChatIntroDismissed: true }), + }, + where: { id: 'usrTest', deletedTime: null }, + }); + }); + + it('ignores malformed existing notify meta when merging updates', async () => { + const { mergeService, update } = createNotifyMetaMergeService('legacy-invalid-json'); + + await mergeService.updateNotifyMeta('usrTest', { appBuilderChatIntroDismissed: true }); + + expect(update).toHaveBeenCalledWith({ + data: { + notifyMeta: JSON.stringify({ appBuilderChatIntroDismissed: true }), + }, + where: { id: 'usrTest', deletedTime: null }, + }); + }); }); diff --git a/apps/nestjs-backend/src/features/user/user.service.ts b/apps/nestjs-backend/src/features/user/user.service.ts index 57d79cd7e1..852d7bf032 100644 --- a/apps/nestjs-backend/src/features/user/user.service.ts +++ b/apps/nestjs-backend/src/features/user/user.service.ts @@ -26,8 +26,8 @@ import type { IClsStore } from '../../types/cls'; import StorageAdapter from '../attachments/plugins/adapter'; import { InjectStorageAdapter } from '../attachments/plugins/storage'; import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; -import { Audit } from '../audit/audit.decorator'; import { AuditScope } from '../audit/audit-scope'; +import { Audit } from '../audit/audit.decorator'; import { UserModel } from '../model/user'; import { SettingService } from '../setting/setting.service'; @@ -334,14 +334,39 @@ export class UserService { } async updateNotifyMeta(id: string, notifyMetaRo: IUserNotifyMeta) { - await this.prismaService.txClient().user.update({ - data: { - notifyMeta: JSON.stringify(notifyMetaRo), - }, - where: { id, deletedTime: null }, + await this.prismaService.$tx(async () => { + const [user] = await this.prismaService.txClient().$queryRaw< + Array<{ notifyMeta: string | null }> + >` + SELECT "notify_meta" AS "notifyMeta" + FROM "users" + WHERE "id" = ${id} + AND "deleted_time" IS NULL + FOR UPDATE + `; + const prevNotifyMeta = this.parseNotifyMeta(user?.notifyMeta); + + await this.prismaService.txClient().user.update({ + data: { + notifyMeta: JSON.stringify({ ...prevNotifyMeta, ...notifyMetaRo }), + }, + where: { id, deletedTime: null }, + }); }); } + private parseNotifyMeta(notifyMeta?: string | null): IUserNotifyMeta { + if (!notifyMeta) { + return {}; + } + + try { + return JSON.parse(notifyMeta) as IUserNotifyMeta; + } catch { + return {}; + } + } + async updateLang(id: string, lang: string) { await this.prismaService.txClient().user.update({ data: { diff --git a/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts index 829204ebc8..beb18cd5cb 100644 --- a/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts @@ -5,11 +5,13 @@ import { FieldId, FieldUpdated, RecordId, + RecordReordered, RecordsBatchCreated, RecordsBatchUpdated, RecordsDeleted, TableActionTriggerRequested, TableId, + ViewId, type IExecutionContext, type IEventHandler, } from '@teable/v2-core'; @@ -576,6 +578,61 @@ describe('V2ActionTriggerService', () => { ]); }); + it('emits setRecord with explicit empty fieldIds for record reorders', async () => { + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2RecordReorderedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId } = createIds(); + const recordId = RecordId.create(`rec${'d'.repeat(16)}`)._unsafeUnwrap(); + const event = RecordReordered.create({ + baseId, + tableId, + viewId: ViewId.create(`viw${'e'.repeat(16)}`)._unsafeUnwrap(), + recordIds: [recordId], + ordersByRecordId: { [recordId.toString()]: 3.375 }, + previousOrdersByRecordId: { [recordId.toString()]: 3 }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + // empty fieldIds means "no cell value changed": field-aware listeners + // (row count, aggregations) must be able to skip refreshing on reorders + expect(submitted).toEqual([{ actionKey: 'setRecord', payload: { fieldIds: [] } }]); + }); + it('emits setRecord presence payload with fieldIds for large batch updates', async () => { let channelSubmitted: string | undefined; let submitted: IPresencePayload | undefined; diff --git a/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts index d5f55b05bc..6db223e8b9 100644 --- a/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts @@ -206,7 +206,10 @@ class V2RecordUpdatedActionTriggerProjection implements IEventHandler> { - emitActionTrigger(this.shareDbService, event.tableId.toString(), [{ actionKey: 'setRecord' }]); + const fieldIds = event.changes.map((c) => c.fieldId); + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { actionKey: 'setRecord', payload: { fieldIds } }, + ]); return ok(undefined); } } @@ -247,7 +250,9 @@ class V2RecordsBatchUpdatedActionTriggerProjection implements IEventHandler> { - emitActionTrigger(this.shareDbService, event.tableId.toString(), [{ actionKey: 'setRecord' }]); + // reorder changes row order only — the explicit empty fieldIds tells + // field-aware listeners (row count, aggregations) that no cell value + // changed, so they can skip refreshing + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { actionKey: 'setRecord', payload: { fieldIds: [] } }, + ]); return ok(undefined); } } diff --git a/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.spec.ts index 93aab08c5b..347943292a 100644 --- a/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.spec.ts @@ -19,6 +19,23 @@ import { V2RecordUpdatedCollaboratorNotificationProjection, } from './v2-collaborator-notification.service'; +const createScheduledContext = (actorId = 'usrActor000000001') => { + const scheduled: Array<() => Promise | void> = []; + const context = { + actorId: { toString: () => actorId }, + scheduleBackgroundTask: vi.fn((task: () => Promise | void) => { + scheduled.push(task); + }), + }; + return { context, scheduled }; +}; + +const flushScheduled = async (scheduled: Array<() => Promise | void>) => { + while (scheduled.length) { + await scheduled.shift()?.(); + } +}; + const createV2ContainerService = () => { const userFieldsQuery = { innerJoin: vi.fn().mockReturnThis(), @@ -79,11 +96,10 @@ describe('V2CollaboratorNotificationDispatcher', () => { it('sends collaborator notification for v2 created records with notified user fields', async () => { const { dispatcher, notificationService, recordService } = createDispatcher(); const projection = new V2RecordCreatedCollaboratorNotificationProjection(dispatcher); + const { context, scheduled } = createScheduledContext(); const result = await projection.handle( - { - actorId: { toString: () => 'usrActor000000001' }, - } as never, + context as never, { tableId: { toString: () => 'tblNotify00000001' }, recordId: { toString: () => 'recNotify00000001' }, @@ -97,6 +113,11 @@ describe('V2CollaboratorNotificationDispatcher', () => { ); expect(result._unsafeUnwrap()).toBeUndefined(); + expect(recordService.getRecordsHeadWithIds).not.toHaveBeenCalled(); + expect(notificationService.sendCollaboratorNotify).not.toHaveBeenCalled(); + + await flushScheduled(scheduled); + expect(recordService.getRecordsHeadWithIds).toHaveBeenCalledWith('tblNotify00000001', [ 'recNotify00000001', ]); @@ -117,11 +138,10 @@ describe('V2CollaboratorNotificationDispatcher', () => { it('sends collaborator notification for v2 updated records with notified user field changes', async () => { const { dispatcher, notificationService } = createDispatcher(); const projection = new V2RecordUpdatedCollaboratorNotificationProjection(dispatcher); + const { context, scheduled } = createScheduledContext(); const result = await projection.handle( - { - actorId: { toString: () => 'usrActor000000001' }, - } as never, + context as never, { source: 'user', tableId: { toString: () => 'tblNotify00000001' }, @@ -142,6 +162,10 @@ describe('V2CollaboratorNotificationDispatcher', () => { ); expect(result._unsafeUnwrap()).toBeUndefined(); + expect(notificationService.sendCollaboratorNotify).not.toHaveBeenCalled(); + + await flushScheduled(scheduled); + expect(notificationService.sendCollaboratorNotify).toHaveBeenCalledTimes(1); expect(notificationService.sendCollaboratorNotify).toHaveBeenCalledWith( expect.objectContaining({ @@ -154,11 +178,10 @@ describe('V2CollaboratorNotificationDispatcher', () => { it('ignores non-user v2 update events', async () => { const { dispatcher, notificationService } = createDispatcher(); const projection = new V2RecordUpdatedCollaboratorNotificationProjection(dispatcher); + const { context, scheduled } = createScheduledContext(); const result = await projection.handle( - { - actorId: { toString: () => 'usrActor000000001' }, - } as never, + context as never, { source: 'computed', tableId: { toString: () => 'tblNotify00000001' }, @@ -174,6 +197,7 @@ describe('V2CollaboratorNotificationDispatcher', () => { ); expect(result._unsafeUnwrap()).toBeUndefined(); + expect(scheduled).toHaveLength(0); expect(notificationService.sendCollaboratorNotify).not.toHaveBeenCalled(); }); }); diff --git a/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.ts b/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.ts index e59c4729e4..521c3b6352 100644 --- a/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-collaborator-notification.service.ts @@ -17,6 +17,7 @@ import { RecordsBatchCreated, RecordsBatchUpdated, RecordUpdated, + scheduleExecutionContextBackgroundTask, } from '@teable/v2-core'; import type { DependencyContainer } from '@teable/v2-di'; import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; @@ -45,6 +46,26 @@ type IUserFieldOptions = { }; const maxRecordTitles = 10; +const collaboratorNotificationLogger = new Logger('V2CollaboratorNotificationProjection'); + +const scheduleCollaboratorNotificationRun = ( + context: IExecutionContext, + task: () => Promise, + eventType: string +): void => { + scheduleExecutionContextBackgroundTask(context, async () => { + try { + await task(); + } catch (error) { + collaboratorNotificationLogger.error( + `Error handling ${eventType} collaborator notification projection: ${ + error instanceof Error ? error.message : String(error) + }`, + error instanceof Error ? error.stack : undefined + ); + } + }); +}; const getNotificationDb = async ( v2ContainerService: V2ContainerService @@ -240,16 +261,21 @@ export class V2RecordCreatedCollaboratorNotificationProjection context: IExecutionContext, event: RecordCreated ): Promise> { - await this.dispatcher.notifyUserFields({ - actorId: context.actorId.toString(), - tableId: event.tableId.toString(), - records: [ - { - id: event.recordId.toString(), - fields: fieldValuesToObject(event.fieldValues), - }, - ], - }); + scheduleCollaboratorNotificationRun( + context, + () => + this.dispatcher.notifyUserFields({ + actorId: context.actorId.toString(), + tableId: event.tableId.toString(), + records: [ + { + id: event.recordId.toString(), + fields: fieldValuesToObject(event.fieldValues), + }, + ], + }), + 'record create' + ); return ok(undefined); } } @@ -264,14 +290,19 @@ export class V2RecordsBatchCreatedCollaboratorNotificationProjection context: IExecutionContext, event: RecordsBatchCreated ): Promise> { - await this.dispatcher.notifyUserFields({ - actorId: context.actorId.toString(), - tableId: event.tableId.toString(), - records: event.records.map((record: RecordValuesDTO) => ({ - id: record.recordId, - fields: fieldValuesToObject(record.fields), - })), - }); + scheduleCollaboratorNotificationRun( + context, + () => + this.dispatcher.notifyUserFields({ + actorId: context.actorId.toString(), + tableId: event.tableId.toString(), + records: event.records.map((record: RecordValuesDTO) => ({ + id: record.recordId, + fields: fieldValuesToObject(record.fields), + })), + }), + 'batch record create' + ); return ok(undefined); } } @@ -290,16 +321,21 @@ export class V2RecordUpdatedCollaboratorNotificationProjection return ok(undefined); } - await this.dispatcher.notifyUserFields({ - actorId: context.actorId.toString(), - tableId: event.tableId.toString(), - records: [ - { - id: event.recordId.toString(), - fields: changesToNewValues(event.changes), - }, - ], - }); + scheduleCollaboratorNotificationRun( + context, + () => + this.dispatcher.notifyUserFields({ + actorId: context.actorId.toString(), + tableId: event.tableId.toString(), + records: [ + { + id: event.recordId.toString(), + fields: changesToNewValues(event.changes), + }, + ], + }), + 'record update' + ); return ok(undefined); } } @@ -318,14 +354,19 @@ export class V2RecordsBatchUpdatedCollaboratorNotificationProjection return ok(undefined); } - await this.dispatcher.notifyUserFields({ - actorId: context.actorId.toString(), - tableId: event.tableId.toString(), - records: event.updates.map((update) => ({ - id: update.recordId, - fields: changesToNewValues(update.changes), - })), - }); + scheduleCollaboratorNotificationRun( + context, + () => + this.dispatcher.notifyUserFields({ + actorId: context.actorId.toString(), + tableId: event.tableId.toString(), + records: event.updates.map((update) => ({ + id: update.recordId, + fields: changesToNewValues(update.changes), + })), + }), + 'batch record update' + ); return ok(undefined); } } diff --git a/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts b/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts index 803238e70c..088689dbe1 100644 --- a/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts +++ b/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts @@ -51,6 +51,7 @@ export class V2ExecutionContextFactory { // Get windowId from CLS for undo/redo tracking const windowId = this.cls.get('windowId'); + const scheduleBackgroundTask = this.cls.get('scheduleV2BackgroundTask'); const t: NonNullable = (key, options) => this.i18n.t(`table.${key}` as never, { args: options, @@ -62,6 +63,7 @@ export class V2ExecutionContextFactory { tracer, requestId, windowId, + scheduleBackgroundTask, config: { ...(tableLimits ? { tableLimits } : {}), }, diff --git a/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts index ae3cc105a4..9bfa669444 100644 --- a/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts @@ -3,6 +3,7 @@ import { v2DataDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { describe, expect, it, vi } from 'vitest'; import { Events } from '../../event-emitter/events'; import { + V2RecordsBatchCreatedHistoryProjection, V2RecordsBatchUpdatedHistoryProjection, V2RecordUpdatedHistoryProjection, } from './v2-record-history.service'; @@ -41,6 +42,23 @@ const createTable = (fields: Array>) => ({ }, }); +const createScheduledContext = (actorId: string) => { + const scheduled: Array<() => Promise | void> = []; + const context = { + actorId: { toString: () => actorId }, + scheduleBackgroundTask: vi.fn((task: () => Promise | void) => { + scheduled.push(task); + }), + }; + return { context, scheduled }; +}; + +const flushScheduled = async (scheduled: Array<() => Promise | void>) => { + while (scheduled.length) { + await scheduled.shift()?.(); + } +}; + const createV2ContainerService = () => { const query = { values: vi.fn().mockReturnThis(), @@ -64,6 +82,7 @@ const createV2ContainerService = () => { query, service: { getContainer: vi.fn().mockResolvedValue(container), + getContainerForTable: vi.fn().mockResolvedValue(container), }, }; }; @@ -71,9 +90,6 @@ const createV2ContainerService = () => { describe('V2RecordUpdatedHistoryProjection', () => { it('writes record history entries through the v2 db container', async () => { const { db, query, service: v2ContainerService } = createV2ContainerService(); - const cls = { - get: vi.fn().mockReturnValue('usrHistWriter00000001'), - }; const tableQueryService = { getById: vi .fn() @@ -84,14 +100,14 @@ describe('V2RecordUpdatedHistoryProjection', () => { }; const projection = new V2RecordUpdatedHistoryProjection( v2ContainerService as never, - cls as never, { recordHistoryDisabled: false } as never, tableQueryService as never, eventEmitterService as never ); + const { context, scheduled } = createScheduledContext('usrHistWriter00000001'); const result = await projection.handle( - {} as never, + context as never, { source: 'user', tableId: { toString: () => 'tblHistTable0000001' }, @@ -107,6 +123,10 @@ describe('V2RecordUpdatedHistoryProjection', () => { ); expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.insertInto).not.toHaveBeenCalled(); + + await flushScheduled(scheduled); + expect(db.insertInto).toHaveBeenCalledWith('record_history'); const [rows] = query.values.mock.calls[0] as [Array>]; expect(rows).toHaveLength(1); @@ -141,12 +161,84 @@ describe('V2RecordUpdatedHistoryProjection', () => { }); }); +describe('V2RecordsBatchCreatedHistoryProjection', () => { + it('writes created record history entries through the routed v2 data DB', async () => { + const { db, query, service: v2ContainerService } = createV2ContainerService(); + const tableQueryService = { + getById: vi + .fn() + .mockResolvedValue(okResult(createTable([createTextField('fldHistField0000001', 'Name')]))), + }; + const eventEmitterService = { + emit: vi.fn(), + }; + const projection = new V2RecordsBatchCreatedHistoryProjection( + v2ContainerService as never, + { recordHistoryDisabled: false } as never, + tableQueryService as never, + eventEmitterService as never + ); + const { context, scheduled } = createScheduledContext('usrBatchCreator00001'); + + const result = await projection.handle( + context as never, + { + tableId: { toString: () => 'tblHistTable0000001' }, + records: [ + { + recordId: 'recHistRecord000001', + fields: [{ fieldId: 'fldHistField0000001', value: 'created-1' }], + }, + { + recordId: 'recHistRecord000002', + fields: [{ fieldId: 'fldHistField0000001', value: 'created-2' }], + }, + ], + } as never + ); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.insertInto).not.toHaveBeenCalled(); + + await flushScheduled(scheduled); + + expect(v2ContainerService.getContainerForTable).toHaveBeenCalledWith('tblHistTable0000001'); + expect(db.insertInto).toHaveBeenCalledWith('record_history'); + const [rows] = query.values.mock.calls[0] as [Array>]; + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + table_id: 'tblHistTable0000001', + record_id: 'recHistRecord000001', + field_id: 'fldHistField0000001', + created_by: 'usrBatchCreator00001', + }); + expect(JSON.parse(rows[0].before)).toEqual({ + meta: { + type: CoreFieldType.SingleLineText, + name: 'Name', + options: null, + cellValueType: 'string', + }, + data: null, + }); + expect(JSON.parse(rows[0].after)).toEqual({ + meta: { + type: CoreFieldType.SingleLineText, + name: 'Name', + options: null, + cellValueType: 'string', + }, + data: 'created-1', + }); + expect(eventEmitterService.emit).toHaveBeenCalledWith(Events.RECORD_HISTORY_CREATE, { + recordIds: ['recHistRecord000001', 'recHistRecord000002'], + }); + }); +}); + describe('V2RecordsBatchUpdatedHistoryProjection', () => { it('writes batch record history entries through the v2 db container', async () => { const { db, query, service: v2ContainerService } = createV2ContainerService(); - const cls = { - get: vi.fn().mockReturnValue('usrBatchWriter0000001'), - }; const tableQueryService = { getById: vi .fn() @@ -157,14 +249,14 @@ describe('V2RecordsBatchUpdatedHistoryProjection', () => { }; const projection = new V2RecordsBatchUpdatedHistoryProjection( v2ContainerService as never, - cls as never, { recordHistoryDisabled: false } as never, tableQueryService as never, eventEmitterService as never ); + const { context, scheduled } = createScheduledContext('usrBatchWriter0000001'); const result = await projection.handle( - {} as never, + context as never, { source: 'user', tableId: { toString: () => 'tblHistTable0000001' }, @@ -194,6 +286,10 @@ describe('V2RecordsBatchUpdatedHistoryProjection', () => { ); expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.insertInto).not.toHaveBeenCalled(); + + await flushScheduled(scheduled); + expect(db.insertInto).toHaveBeenCalledWith('record_history'); const [rows] = query.values.mock.calls[0] as [Array>]; expect(rows).toHaveLength(2); diff --git a/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts b/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts index 577aac3b0e..6646ee6bb9 100644 --- a/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts @@ -10,10 +10,13 @@ import { FieldValueTypeVisitor, ProjectionHandler, RecordUpdated, + RecordsBatchCreated, RecordsBatchUpdated, TableQueryService, ok, + scheduleExecutionContextBackgroundTask, v2CoreTokens, + withoutTransaction, } from '@teable/v2-core'; import type { DomainError, @@ -29,15 +32,34 @@ import type { DependencyContainer } from '@teable/v2-di'; import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; import type { ColumnType, Kysely } from 'kysely'; import { isEqual, isString } from 'lodash'; -import { ClsService } from 'nestjs-cls'; import { BaseConfig, IBaseConfig } from '../../configs/base.config'; import { EventEmitterService } from '../../event-emitter/event-emitter.service'; import { Events } from '../../event-emitter/events'; -import type { IClsStore } from '../../types/cls'; import { V2ContainerService } from './v2-container.service'; import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from './v2-projection-registrar'; const SELECT_FIELD_TYPE_SET = new Set([CoreFieldType.SingleSelect, CoreFieldType.MultipleSelect]); +const recordHistoryProjectionLogger = new Logger('V2RecordHistoryProjection'); + +const scheduleRecordHistoryRun = ( + context: IExecutionContext, + task: (backgroundContext: IExecutionContext) => Promise, + eventType: string +): void => { + const backgroundContext = withoutTransaction(context); + scheduleExecutionContextBackgroundTask(backgroundContext, async () => { + try { + await task(backgroundContext); + } catch (error) { + recordHistoryProjectionLogger.error( + `Error handling ${eventType} record history projection: ${ + error instanceof Error ? error.message : String(error) + }`, + error instanceof Error ? error.stack : undefined + ); + } + }); +}; interface IRecordHistoryEntry { id: string; @@ -243,7 +265,6 @@ const buildHistoryValue = ( export class V2RecordUpdatedHistoryProjection implements IEventHandler { constructor( private readonly v2ContainerService: V2ContainerService, - private readonly cls: ClsService, private readonly baseConfig: IBaseConfig, private readonly tableQueryService: TableQueryService, private readonly eventEmitterService: EventEmitterService @@ -253,29 +274,42 @@ export class V2RecordUpdatedHistoryProjection implements IEventHandler> { - // Check if record history is disabled if (this.baseConfig.recordHistoryDisabled) { return ok(undefined); } - // Skip computed updates - we only track user-initiated changes if (event.source === 'computed') { return ok(undefined); } - const tableIdStr = event.tableId.toString(); - const recordId = event.recordId.toString(); - const userId = this.cls.get('user.id'); - - // Get field IDs from changes if (event.changes.length === 0) { return ok(undefined); } + scheduleRecordHistoryRun( + context, + (backgroundContext) => this.writeRecordUpdatedHistory(backgroundContext, event), + 'record update' + ); + + return ok(undefined); + } + + private async writeRecordUpdatedHistory( + context: IExecutionContext, + event: RecordUpdated + ): Promise { + const tableIdStr = event.tableId.toString(); + const recordId = event.recordId.toString(); + // Use the actor captured in the event's execution snapshot, not CLS. + // CLS (AsyncLocalStorage) is read at drain time, which runs in an unrelated + // request's async context, so it would attribute history to the wrong user. + const userId = context.actorId.toString(); + // Load table from V2 domain const tableResult = await this.tableQueryService.getById(context, event.tableId); if (tableResult.isErr()) { - return ok(undefined); // Silently skip if table not found + return; // Silently skip if table not found } const table = tableResult.value; @@ -323,9 +357,117 @@ export class V2RecordUpdatedHistoryProjection implements IEventHandler { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly baseConfig: IBaseConfig, + private readonly tableQueryService: TableQueryService, + private readonly eventEmitterService: EventEmitterService + ) {} + + async handle( + context: IExecutionContext, + event: RecordsBatchCreated + ): Promise> { + if (this.baseConfig.recordHistoryDisabled) { + return ok(undefined); + } + + const fieldIdSet = new Set(); + for (const record of event.records) { + for (const field of record.fields) { + fieldIdSet.add(field.fieldId); + } + } + + if (fieldIdSet.size === 0) { + return ok(undefined); + } + + scheduleRecordHistoryRun( + context, + (backgroundContext) => + this.writeRecordsBatchCreatedHistory(backgroundContext, event, fieldIdSet), + 'batch record create' + ); return ok(undefined); } + + private async writeRecordsBatchCreatedHistory( + context: IExecutionContext, + event: RecordsBatchCreated, + fieldIdSet: Set + ): Promise { + const tableIdStr = event.tableId.toString(); + // Use the actor captured in the event's execution snapshot, not CLS. + // CLS (AsyncLocalStorage) is read at drain time, which runs in an unrelated + // request's async context, so it would attribute history to the wrong user. + const userId = context.actorId.toString(); + + const tableResult = await this.tableQueryService.getById(context, event.tableId); + if (tableResult.isErr()) { + return; + } + const table = tableResult.value; + + const fieldMetaMap = new Map(); + for (const fieldIdStr of fieldIdSet) { + const fieldIdResult = FieldId.create(fieldIdStr); + if (fieldIdResult.isErr()) continue; + + const fieldResult = table.getField((f) => f.id().equals(fieldIdResult.value)); + if (fieldResult.isOk()) { + fieldMetaMap.set(fieldIdStr, extractFieldMeta(fieldResult.value)); + } + } + + const recordHistoryList: IRecordHistoryEntry[] = []; + const recordIds: string[] = []; + const batchSize = 5000; + + for (const record of event.records) { + recordIds.push(record.recordId); + + for (const field of record.fields) { + const value = field.value; + if (value === '' || value == null) continue; + + const meta = fieldMetaMap.get(field.fieldId); + if (!meta || meta.isComputed) continue; + + recordHistoryList.push({ + id: generateRecordHistoryId(), + table_id: tableIdStr, + record_id: record.recordId, + field_id: field.fieldId, + before: JSON.stringify(buildHistoryValue(null, meta)), + after: JSON.stringify(buildHistoryValue(value, meta)), + created_by: userId as string, + }); + } + } + + const db = await getRecordHistoryDb(this.v2ContainerService, tableIdStr); + for (let i = 0; i < recordHistoryList.length; i += batchSize) { + const batch = recordHistoryList.slice(i, i + batchSize); + await insertRecordHistoryEntries(db, batch); + } + + if (recordIds.length > 0) { + this.eventEmitterService.emit(Events.RECORD_HISTORY_CREATE, { + recordIds, + }); + } + } } /** @@ -336,7 +478,6 @@ export class V2RecordUpdatedHistoryProjection implements IEventHandler { constructor( private readonly v2ContainerService: V2ContainerService, - private readonly cls: ClsService, private readonly baseConfig: IBaseConfig, private readonly tableQueryService: TableQueryService, private readonly eventEmitterService: EventEmitterService @@ -346,20 +487,14 @@ export class V2RecordsBatchUpdatedHistoryProjection implements IEventHandler> { - // Check if record history is disabled if (this.baseConfig.recordHistoryDisabled) { return ok(undefined); } - // Skip computed updates if (event.source === 'computed') { return ok(undefined); } - const tableIdStr = event.tableId.toString(); - const userId = this.cls.get('user.id'); - - // Collect all field IDs from all updates const fieldIdSet = new Set(); for (const update of event.updates) { for (const change of update.changes) { @@ -371,10 +506,31 @@ export class V2RecordsBatchUpdatedHistoryProjection implements IEventHandler + this.writeRecordsBatchUpdatedHistory(backgroundContext, event, fieldIdSet), + 'batch record update' + ); + + return ok(undefined); + } + + private async writeRecordsBatchUpdatedHistory( + context: IExecutionContext, + event: RecordsBatchUpdated, + fieldIdSet: Set + ): Promise { + const tableIdStr = event.tableId.toString(); + // Use the actor captured in the event's execution snapshot, not CLS. + // CLS (AsyncLocalStorage) is read at drain time, which runs in an unrelated + // request's async context, so it would attribute history to the wrong user. + const userId = context.actorId.toString(); + // Load table from V2 domain const tableResult = await this.tableQueryService.getById(context, event.tableId); if (tableResult.isErr()) { - return ok(undefined); // Silently skip if table not found + return; // Silently skip if table not found } const table = tableResult.value; @@ -435,8 +591,6 @@ export class V2RecordsBatchUpdatedHistoryProjection implements IEventHandler, @BaseConfig() private readonly baseConfig: IBaseConfig, private readonly eventEmitterService: EventEmitterService ) {} @@ -470,7 +623,16 @@ export class V2RecordHistoryService implements IV2ProjectionRegistrar { V2RecordUpdatedHistoryProjection, new V2RecordUpdatedHistoryProjection( this.v2ContainerService, - this.cls, + this.baseConfig, + tableQueryService, + this.eventEmitterService + ) + ); + + container.registerInstance( + V2RecordsBatchCreatedHistoryProjection, + new V2RecordsBatchCreatedHistoryProjection( + this.v2ContainerService, this.baseConfig, tableQueryService, this.eventEmitterService @@ -481,7 +643,6 @@ export class V2RecordHistoryService implements IV2ProjectionRegistrar { V2RecordsBatchUpdatedHistoryProjection, new V2RecordsBatchUpdatedHistoryProjection( this.v2ContainerService, - this.cls, this.baseConfig, tableQueryService, this.eventEmitterService diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts index 6dc9fff327..0da64cdf1d 100644 --- a/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import type { IOtOperation, IViewRo, @@ -33,14 +33,13 @@ import { HttpErrorCode, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { PluginPosition, PluginStatus } from '@teable/openapi'; +import { PluginPosition, PluginStatus, IViewShareMetaRo } from '@teable/openapi'; import type { IViewPluginUpdateStorageRo, IGetViewFilterLinkRecordsVo, IUpdateOrderRo, IUpdateRecordOrdersRo, IViewInstallPluginRo, - IViewShareMetaRo, } from '@teable/openapi'; import { Knex } from 'knex'; import { keyBy, pick } from 'lodash'; @@ -57,11 +56,14 @@ import { DATA_KNEX } from '../../../global/knex/knex.module'; import type { IClsStore } from '../../../types/cls'; import { Timing } from '../../../utils/timing'; import { updateMultipleOrders, updateOrder } from '../../../utils/update-order'; +import { AuditScope } from '../../audit/audit-scope'; +import { Audit } from '../../audit/audit.decorator'; import { FieldViewSyncService } from '../../field/field-calculate/field-view-sync.service'; import { FieldService } from '../../field/field.service'; import type { IFieldInstance } from '../../field/model/factory'; import { createFieldInstanceByRaw, createFieldInstanceByVo } from '../../field/model/factory'; import { RecordService } from '../../record/record.service'; +import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { createViewInstanceByRaw } from '../model/factory'; import { ViewService } from '../view.service'; @@ -78,12 +80,30 @@ export class ViewOpenApiService { private readonly fieldViewSyncService: FieldViewSyncService, private readonly eventEmitterService: EventEmitterService, private readonly cls: ClsService, + private readonly audit: AuditScope, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(DATA_KNEX) private readonly knex: Knex, - @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + + private async assertViewWritable(viewId: string) { + const view = await this.prismaService.view.findUnique({ + where: { id: viewId }, + select: { tableId: true }, + }); + if (view) { + await this.assertTableWritable(view.tableId); + } + } + async createView(tableId: string, viewRo: IViewRo) { + await this.assertTableWritable(tableId); if (viewRo.type === ViewType.Plugin) { const res = await this.pluginInstall(tableId, { name: viewRo.name, @@ -100,6 +120,7 @@ export class ViewOpenApiService { } async deleteView(tableId: string, viewId: string, windowId?: string) { + await this.assertTableWritable(tableId); const result = await this.prismaService.$tx(async () => { await this.fieldViewSyncService.deleteLinkOptionsDependenciesByViewId(tableId, viewId); return await this.deleteViewInner(tableId, viewId); @@ -194,6 +215,7 @@ export class ViewOpenApiService { columnMetaRo: IColumnMetaRo, windowId?: string ) { + await this.assertTableWritable(tableId); const view = await this.prismaService.view .findFirstOrThrow({ where: { tableId, id: viewId }, @@ -289,6 +311,21 @@ export class ViewOpenApiService { } } + @Audit({ + action: Events.SHARED_VIEW_UPDATE, + resourceId: (_tableId: string, viewId: string) => viewId, + params: (tableId: string, viewId: string, viewShareMetaRo: IViewShareMetaRo) => ({ + tableId, + viewId, + // Mirror base-share masking: never log the plaintext share password — record only whether + // one was set. (IViewShareMetaRo.password is optional and was previously stored verbatim.) + shareMeta: { + ...viewShareMetaRo, + password: viewShareMetaRo.password !== undefined ? '[set]' : undefined, + }, + }), + emit: true, + }) async updateShareMeta(tableId: string, viewId: string, viewShareMetaRo: IViewShareMetaRo) { return this.setViewProperty(tableId, viewId, 'shareMeta', viewShareMetaRo); } @@ -394,6 +431,7 @@ export class ViewOpenApiService { newValue: unknown, windowId?: string ) { + await this.assertTableWritable(tableId); const curView = await this.prismaService.view .findFirstOrThrow({ select: { [key]: true }, @@ -451,6 +489,7 @@ export class ViewOpenApiService { } async updateViewByOps(tableId: string, viewId: string, ops: IOtOperation[]) { + await this.assertTableWritable(tableId); return await this.prismaService.$tx(async () => { return await this.viewService.updateViewByOps(tableId, viewId, ops); }); @@ -462,6 +501,7 @@ export class ViewOpenApiService { viewOptions: IViewOptions, windowId?: string ) { + await this.assertTableWritable(tableId); const curView = await this.prismaService.view .findFirstOrThrow({ select: { options: true, type: true }, @@ -521,6 +561,7 @@ export class ViewOpenApiService { * shuffle view order */ async shuffle(tableId: string) { + await this.assertTableWritable(tableId); const views = await this.prismaService.view.findMany({ where: { tableId, deletedTime: null }, select: { id: true, order: true }, @@ -551,6 +592,7 @@ export class ViewOpenApiService { orderRo: IUpdateOrderRo, windowId?: string ) { + await this.assertTableWritable(tableId); const { anchorId, position } = orderRo; const view = await this.prismaService.view @@ -736,6 +778,7 @@ export class ViewOpenApiService { order?: Record; }[] ) { + await this.assertTableWritable(tableId); await this.recordService.updateRecordIndexes(tableId, recordsWithOrder); const ops = ViewOpBuilder.editor.setViewProperty.build({ key: 'lastModifiedTime', @@ -750,6 +793,7 @@ export class ViewOpenApiService { orderRo: IUpdateRecordOrdersRo, windowId?: string ) { + await this.assertTableWritable(table.id); const recordIds = orderRo.recordIds; const dbTableName = table.dbTableName; const orderIndexesBefore = windowId @@ -803,6 +847,12 @@ export class ViewOpenApiService { } } + @Audit({ + action: Events.SHARED_VIEW_REFRESH, + resourceId: (_tableId: string, viewId: string) => viewId, + params: (tableId: string, viewId: string) => ({ tableId, viewId }), + emit: (result: { shareId: string }) => ({ shareId: result.shareId }), + }) async refreshShareId(tableId: string, viewId: string) { const view = await this.prismaService.view.findUnique({ where: { id: viewId, tableId, deletedTime: null }, @@ -841,6 +891,12 @@ export class ViewOpenApiService { return { shareId: newShareId }; } + @Audit({ + action: Events.SHARED_VIEW_CREATE, + resourceId: (_tableId: string, viewId: string) => viewId, + params: (tableId: string, viewId: string) => ({ tableId, viewId, enabled: true }), + emit: (result: { shareId: string }) => ({ shareId: result.shareId }), + }) async enableShare(tableId: string, viewId: string) { const view = await this.prismaService.view.findUnique({ where: { id: viewId, tableId, deletedTime: null }, @@ -894,6 +950,12 @@ export class ViewOpenApiService { return { shareId: newShareId }; } + @Audit({ + action: Events.SHARED_VIEW_DELETE, + resourceId: (_tableId: string, viewId: string) => viewId, + params: (tableId: string, viewId: string) => ({ tableId, viewId, enabled: false }), + emit: true, + }) async disableShare(tableId: string, viewId: string) { const view = await this.prismaService.view.findUnique({ where: { id: viewId, tableId, deletedTime: null }, @@ -1064,6 +1126,7 @@ export class ViewOpenApiService { enableShare?: boolean; } ) { + await this.assertTableWritable(tableId); const userId = this.cls.get('user.id'); const { name, pluginId, shareId, shareMeta, enableShare } = ro; const plugin = await this.prismaService.txClient().plugin.findUnique({ @@ -1143,6 +1206,7 @@ export class ViewOpenApiService { } async updatePluginStorage(viewId: string, storage: IViewPluginUpdateStorageRo['storage']) { + await this.assertViewWritable(viewId); const pluginInstall = await this.prismaService.pluginInstall.findFirst({ where: { positionId: viewId, position: PluginPosition.View }, select: { id: true }, @@ -1203,6 +1267,7 @@ export class ViewOpenApiService { } async duplicateView(tableId: string, viewId: string) { + await this.assertTableWritable(tableId); const view = await this.viewService.getViewById(viewId); const { options: optionsRaw } = await this.prismaService.txClient().view.findUniqueOrThrow({ where: { id: viewId, deletedTime: null }, diff --git a/apps/nestjs-backend/src/features/view/view.service.ts b/apps/nestjs-backend/src/features/view/view.service.ts index 6517832f37..6a5b938d7e 100644 --- a/apps/nestjs-backend/src/features/view/view.service.ts +++ b/apps/nestjs-backend/src/features/view/view.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable } from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import type { ISnapshotBase, IViewRo, @@ -50,6 +50,7 @@ import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; import { convertViewVoAttachmentUrl } from '../../utils/convert-view-vo-attachment-url'; import { BatchService } from '../calculation/batch.service'; +import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service'; import { ROW_ORDER_FIELD_PREFIX } from './constant'; import { createViewInstanceByRaw, createViewVoByRaw } from './model/factory'; import { adjustFrozenField } from './utils/derive-frozen-fields'; @@ -67,9 +68,15 @@ export class ViewService implements IReadonlyAdapterService { @InjectModel(CUSTOM_KNEX) private readonly knex: Knex, @InjectModel(DATA_KNEX) private readonly dataKnex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, - private readonly viewDataSafetyLimitService: ViewDataSafetyLimitService + private readonly viewDataSafetyLimitService: ViewDataSafetyLimitService, + @Optional() + private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService ) {} + private async assertTableWritable(tableId: string) { + await this.spaceDataDbMigrationGuard?.assertTableWritable(tableId); + } + getRowIndexFieldName(viewId: string) { return `${ROW_ORDER_FIELD_PREFIX}_${viewId}`; } @@ -253,6 +260,7 @@ export class ViewService implements IReadonlyAdapterService { } async restoreView(tableId: string, viewId: string) { + await this.assertTableWritable(tableId); await this.prismaService.$tx(async () => { await this.prismaService.txClient().view.update({ where: { id: viewId }, @@ -270,6 +278,7 @@ export class ViewService implements IReadonlyAdapterService { async createDbView(tableId: string, viewRo: IViewRo) { const userId = this.cls.get('user.id'); + await this.assertTableWritable(tableId); await this.viewDataSafetyLimitService.ensureCanCreateView(tableId); const createViewRo = await this.viewDataCompensation(tableId, viewRo); @@ -363,6 +372,7 @@ export class ViewService implements IReadonlyAdapterService { } async deleteView(tableId: string, viewId: string) { + await this.assertTableWritable(tableId); // Use SELECT FOR UPDATE to lock all views in the table to prevent concurrent deletion // This ensures that when checking if this is the last view, no other transaction // can delete views simultaneously @@ -408,6 +418,7 @@ export class ViewService implements IReadonlyAdapterService { } async updateViewSort(tableId: string, viewId: string, sort: ISort) { + await this.assertTableWritable(tableId); this.viewDataSafetyLimitService.ensureSort(sort); const viewRaw = await this.prismaService .txClient() @@ -465,6 +476,7 @@ export class ViewService implements IReadonlyAdapterService { } async batchUpdateViewByOps(tableId: string, opsMap: { [viewId: string]: IOtOperation[] }) { + await this.assertTableWritable(tableId); const { updateViewMap, updateViewKeySet } = this.getBatchUpdateViewContext(opsMap); if (updateViewKeySet.size === 0) { return; @@ -568,6 +580,7 @@ export class ViewService implements IReadonlyAdapterService { } async del(_version: number, _tableId: string, viewId: string) { + await this.assertTableWritable(_tableId); await this.prismaService.txClient().view.update({ where: { id: viewId }, data: { diff --git a/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts b/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts index 0f126c83b9..4c819c9c9d 100644 --- a/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts +++ b/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts @@ -1,4 +1,4 @@ -import { BadRequestException, Logger } from '@nestjs/common'; +import { BadRequestException, Logger, RequestTimeoutException } from '@nestjs/common'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { GlobalExceptionFilter } from './global-exception.filter'; @@ -67,9 +67,11 @@ describe('GlobalExceptionFilter', () => { const response: { status: ReturnType; json: ReturnType; + setHeader: ReturnType; } = { status: vi.fn(), json: vi.fn(), + setHeader: vi.fn(), }; response.status.mockReturnValue(response); @@ -113,6 +115,73 @@ describe('GlobalExceptionFilter', () => { }); }); + it('captures unexpected exceptions with V2 attribution context', () => { + const cls = { + get: vi.fn((key: string) => { + const values = new Map([ + ['spaceId', spaceId], + ['useV2', true], + ['v2Reason', 'space_feature'], + ['v2Feature', 'deleteField'], + ]); + return values.get(key); + }), + }; + const exception = new Error('delete failed'); + const filter = new GlobalExceptionFilter(configService as never, cls as never); + + filter.catch(exception, host as never); + + expect(response.setHeader).toHaveBeenCalledWith('x-teable-v2', 'true'); + expect(response.setHeader).toHaveBeenCalledWith('x-teable-v2-reason', 'space_feature'); + expect(response.setHeader).toHaveBeenCalledWith('x-teable-v2-feature', 'deleteField'); + expect(sentryScope.setTag).toHaveBeenCalledWith('teable.version', 'v2'); + expect(sentryScope.setTag).toHaveBeenCalledWith('teable.v2.enabled', 'true'); + expect(sentryScope.setTag).toHaveBeenCalledWith('teable.v2.reason', 'space_feature'); + expect(sentryScope.setTag).toHaveBeenCalledWith('teable.v2.feature', 'deleteField'); + expect(sentryScope.setTag).toHaveBeenCalledWith('teable.request.attribution', 'v2'); + expect(activeSpan.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'teable.request.attribution': 'v2', + 'teable.v2.enabled': true, + 'teable.v2.reason': 'space_feature', + 'teable.v2.feature': 'deleteField', + }) + ); + }); + + it('logs V2 attribution for request timeout HTTP exceptions that are not captured', () => { + const loggerError = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const cls = { + get: vi.fn((key: string) => { + const values = new Map([ + ['useV2', false], + ['v2Reason', 'disabled'], + ['v2Feature', 'deleteField'], + ]); + return values.get(key); + }), + }; + const filter = new GlobalExceptionFilter(configService as never, cls as never); + + filter.catch(new RequestTimeoutException('timeout'), host as never); + + expect(withScope).not.toHaveBeenCalled(); + expect(response.setHeader).toHaveBeenCalledWith('x-teable-v2', 'false'); + expect(loggerError).toHaveBeenCalledWith( + expect.objectContaining({ + url: '/api/test', + v2: { + attribution: 'v1', + enabled: false, + reason: 'disabled', + feature: 'deleteField', + }, + }), + expect.any(String) + ); + }); + it('does not capture expected Nest HTTP exceptions', () => { const filter = new GlobalExceptionFilter(configService as never); diff --git a/apps/nestjs-backend/src/filter/global-exception.filter.ts b/apps/nestjs-backend/src/filter/global-exception.filter.ts index fdbf4e5520..83fc426a09 100644 --- a/apps/nestjs-backend/src/filter/global-exception.filter.ts +++ b/apps/nestjs-backend/src/filter/global-exception.filter.ts @@ -18,6 +18,13 @@ import type { Request, Response } from 'express'; import { ClsService } from 'nestjs-cls'; import type { ILoggerConfig } from '../configs/logger.config'; import { TemplateAppTokenNotAllowedException } from '../custom.exception'; +import { + getV2Attribution, + getV2AttributionLogContext, + getV2AttributionSpanAttributes, + setV2AttributionHeaders, + setV2AttributionOnSentryScope, +} from '../features/canary/v2-attribution'; import { classifyDataDbRuntimeError } from '../global/data-db-runtime-error'; import type { IDataDbRuntimeErrorClassification } from '../global/data-db-runtime-error'; import type { IClsStore } from '../types/cls'; @@ -58,6 +65,7 @@ export class GlobalExceptionFilter implements ExceptionFilter { const dataDbContext = this.getDataDbContext(); const dataDbError = dataDbContext ? classifyDataDbRuntimeError(exception) : null; + setV2AttributionHeaders(response, getV2Attribution(this.cls)); this.annotateActiveSpan(dataDbError); this.recordDataDbMetric(dataDbError); this.captureException(exception, dataDbError); @@ -139,6 +147,7 @@ export class GlobalExceptionFilter implements ExceptionFilter { if (spaceId) { scope.setTag('space.id', spaceId); } + setV2AttributionOnSentryScope(scope, getV2Attribution(this.cls)); } catch { // CLS may not be active (e.g., non-HTTP contexts) } @@ -167,12 +176,17 @@ export class GlobalExceptionFilter implements ExceptionFilter { } private annotateActiveSpan(dataDbError?: IDataDbRuntimeErrorClassification | null) { - const dataDbContext = this.getDataDbContext(); - if (!dataDbContext || !dataDbError) return; - const span = trace.getActiveSpan(); if (!span) return; + const v2Attributes = getV2AttributionSpanAttributes(getV2Attribution(this.cls)); + if (Object.keys(v2Attributes).length) { + span.setAttributes(v2Attributes); + } + + const dataDbContext = this.getDataDbContext(); + if (!dataDbContext || !dataDbError) return; + span.setAttributes({ [dataDbOtelAttribute.mode]: dataDbContext.mode, [dataDbOtelAttribute.errorCode]: dataDbError.code, @@ -216,10 +230,12 @@ export class GlobalExceptionFilter implements ExceptionFilter { protected logError(exception: Error, request: Request) { const dataDbContext = this.getDataDbContext(); const dataDbError = dataDbContext ? classifyDataDbRuntimeError(exception) : null; + const v2 = getV2AttributionLogContext(getV2Attribution(this.cls)); this.logger.error( { url: request?.url, message: exception.message, + v2, dataDb: dataDbError ? { code: dataDbError.code, diff --git a/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts b/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts index 585d1fb929..5a4b97006b 100644 --- a/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts +++ b/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts @@ -8,6 +8,13 @@ const withTxClient = (txClient: T) => ({ txClient: vi.fn(() => txClient), }); +const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; +const internalSchema = 'teable_meta_test'; +const connectionId = 'dcnxxx'; +const displayHost = 'example.com'; +const displayDatabase = 'teable_data'; +const urlFingerprint = 'fp_xxx'; + describe('DataDbClientManager', () => { it('falls back to the meta DB clients when a space has no BYODB binding', async () => { const prismaService = withTxClient({ @@ -156,8 +163,6 @@ describe('DataDbClientManager', () => { }); it('resolves BYODB connection details from a ready space binding', async () => { - const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; - const internalSchema = 'teable_meta_test'; const cls = { isActive: vi.fn().mockReturnValue(true), set: vi.fn(), @@ -168,12 +173,12 @@ describe('DataDbClientManager', () => { mode: 'byodb', state: 'ready', dataDbConnection: { - id: 'dcnxxx', + id: connectionId, status: 'ready', internalSchema, - displayHost: 'example.com', - displayDatabase: 'teable_data', - urlFingerprint: 'fp_xxx', + displayHost, + displayDatabase, + urlFingerprint, encryptedUrl: encryptDataDbUrl(dataUrl), }, }), @@ -194,27 +199,134 @@ describe('DataDbClientManager', () => { `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}` ); await expect(manager.getDataDatabaseForSpace('spcxxx')).resolves.toMatchObject({ - cacheKey: 'dcnxxx', - connectionId: 'dcnxxx', + cacheKey: connectionId, + connectionId, isMetaFallback: false, url: `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}`, }); expect(cls.set).toHaveBeenCalledWith('dataDb', { mode: 'byodb', spaceId: 'spcxxx', - connectionId: 'dcnxxx', - urlFingerprint: 'fp_xxx', - displayHost: 'example.com', - displayDatabase: 'teable_data', + connectionId, + urlFingerprint, + displayHost, + displayDatabase, internalSchema, }); await expect(manager.dataKnexForSpace('spcxxx')).resolves.not.toBe(metaFallbackDataKnex); await manager.onModuleDestroy(); }); + it('can preview a BYODB route for a space without writing a binding', async () => { + const prismaService = withTxClient({ + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue(null), + }, + }); + const manager = new DataDbClientManager( + prismaService as never, + {} as never, + {} as never, + new DataDbRuntimeCacheService() + ); + + await expect( + manager.getDataDatabaseForSpace('spcxxx', { + previewBinding: { + spaceId: 'spcxxx', + connectionId, + encryptedUrl: encryptDataDbUrl(dataUrl), + internalSchema, + urlFingerprint, + displayHost, + displayDatabase, + }, + }) + ).resolves.toMatchObject({ + cacheKey: connectionId, + connectionId, + internalSchema, + isMetaFallback: false, + url: `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}`, + }); + expect(prismaService.spaceDataDbBinding.findUnique).not.toHaveBeenCalled(); + }); + + it('can force the original source route to meta fallback during migration', async () => { + vi.stubEnv('PRISMA_DATABASE_URL', 'postgresql://meta.example/teable'); + const prismaService = withTxClient({ + dataDbConnection: { + findUnique: vi.fn(), + }, + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue({ + mode: 'byodb', + state: 'migrating', + dataDbConnection: { + id: connectionId, + status: 'migrating', + internalSchema, + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }), + }, + }); + const manager = new DataDbClientManager( + prismaService as never, + {} as never, + {} as never, + new DataDbRuntimeCacheService() + ); + + await expect( + manager.getDataDatabaseForSpace('spcxxx', { sourceConnectionId: null }) + ).resolves.toMatchObject({ + cacheKey: 'meta-fallback', + isMetaFallback: true, + }); + expect(prismaService.spaceDataDbBinding.findUnique).not.toHaveBeenCalled(); + expect(prismaService.dataDbConnection.findUnique).not.toHaveBeenCalled(); + }); + + it('can resolve the original source BYODB connection during migration', async () => { + const sourceConnectionId = 'dcnsource'; + const sourceSchema = 'source_schema'; + const sourceUrl = 'postgresql://teable:secret@source.example.com:5432/teable_data'; + const prismaService = withTxClient({ + dataDbConnection: { + findUnique: vi.fn().mockResolvedValue({ + id: sourceConnectionId, + internalSchema: sourceSchema, + encryptedUrl: encryptDataDbUrl(sourceUrl), + }), + }, + spaceDataDbBinding: { + findUnique: vi.fn(), + }, + }); + const manager = new DataDbClientManager( + prismaService as never, + {} as never, + {} as never, + new DataDbRuntimeCacheService() + ); + + await expect( + manager.getDataDatabaseForSpace('spcxxx', { sourceConnectionId }) + ).resolves.toMatchObject({ + cacheKey: sourceConnectionId, + connectionId: sourceConnectionId, + internalSchema: sourceSchema, + isMetaFallback: false, + url: `${sourceUrl}?schema=${sourceSchema}&options=-c+search_path%3D${sourceSchema}`, + }); + expect(prismaService.dataDbConnection.findUnique).toHaveBeenCalledWith({ + where: { id: sourceConnectionId }, + }); + expect(prismaService.spaceDataDbBinding.findUnique).not.toHaveBeenCalled(); + }); + it('resolves BYODB connection details when no CLS context is active', async () => { - const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; - const internalSchema = 'teable_meta_test'; const cls = { isActive: vi.fn().mockReturnValue(false), set: vi.fn(() => { @@ -227,12 +339,12 @@ describe('DataDbClientManager', () => { mode: 'byodb', state: 'ready', dataDbConnection: { - id: 'dcnxxx', + id: connectionId, status: 'ready', internalSchema, - displayHost: 'example.com', - displayDatabase: 'teable_data', - urlFingerprint: 'fp_xxx', + displayHost, + displayDatabase, + urlFingerprint, encryptedUrl: encryptDataDbUrl(dataUrl), }, }), @@ -254,8 +366,6 @@ describe('DataDbClientManager', () => { }); it('ensures the BYODB internal schema is migrated before returning a scoped URL', async () => { - const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; - const internalSchema = 'teable_meta_test'; const dataDbMigrationService = { ensureConnectionMigrated: vi.fn().mockResolvedValue([]), }; @@ -265,7 +375,7 @@ describe('DataDbClientManager', () => { mode: 'byodb', state: 'migrating', dataDbConnection: { - id: 'dcnxxx', + id: connectionId, status: 'migrating', internalSchema, encryptedUrl: encryptDataDbUrl(dataUrl), @@ -282,13 +392,13 @@ describe('DataDbClientManager', () => { ); await expect(manager.getDataDatabaseForSpace('spcxxx')).resolves.toMatchObject({ - cacheKey: 'dcnxxx', - connectionId: 'dcnxxx', + cacheKey: connectionId, + connectionId, internalSchema, isMetaFallback: false, }); expect(dataDbMigrationService.ensureConnectionMigrated).toHaveBeenCalledWith({ - connectionId: 'dcnxxx', + connectionId, internalSchema, url: dataUrl, }); diff --git a/apps/nestjs-backend/src/global/data-db-client-manager.service.ts b/apps/nestjs-backend/src/global/data-db-client-manager.service.ts index d569b0db5d..4c0404cdce 100644 --- a/apps/nestjs-backend/src/global/data-db-client-manager.service.ts +++ b/apps/nestjs-backend/src/global/data-db-client-manager.service.ts @@ -27,12 +27,28 @@ export interface IResolvedDataDatabase { internalSchema?: string; } +export interface IDataDbPreviewBinding { + spaceId: string; + connectionId: string; + encryptedUrl: string; + internalSchema: string; + urlFingerprint?: string | null; + displayHost?: string | null; + displayDatabase?: string | null; +} + export interface IDataDbRoutingOptions { useTransaction?: boolean; + previewBinding?: IDataDbPreviewBinding; + sourceConnectionId?: string | null; } type IMetaRoutingClient = PrismaService | NonNullable; +type IResolvedSpaceDataDbRoute = + | { isMetaFallback: true } + | { connectionId: string; internalSchema: string; isMetaFallback: false; url: string }; + @Injectable() export class DataDbClientManager { constructor( @@ -205,10 +221,15 @@ export class DataDbClientManager { private async resolveSpaceDataDb( spaceId: string, options?: IDataDbRoutingOptions - ): Promise< - | { isMetaFallback: true } - | { connectionId: string; internalSchema: string; isMetaFallback: false; url: string } - > { + ): Promise { + if ('sourceConnectionId' in (options ?? {})) { + return await this.resolveSourceSpaceDataDb(options); + } + + if (options?.previewBinding?.spaceId === spaceId) { + return this.resolvePreviewSpaceDataDb(spaceId, options.previewBinding); + } + const binding = await this.getMetaRoutingClient(options).spaceDataDbBinding.findUnique({ where: { spaceId }, include: { dataDbConnection: true }, @@ -266,6 +287,53 @@ export class DataDbClientManager { }; } + private resolvePreviewSpaceDataDb( + spaceId: string, + preview: IDataDbPreviewBinding + ): IResolvedSpaceDataDbRoute { + if (this.cls?.isActive()) { + this.cls.set('dataDb', { + mode: 'byodb', + spaceId, + connectionId: preview.connectionId, + urlFingerprint: preview.urlFingerprint ?? null, + displayHost: preview.displayHost ?? null, + displayDatabase: preview.displayDatabase ?? null, + internalSchema: preview.internalSchema, + }); + } + + return { + connectionId: preview.connectionId, + internalSchema: preview.internalSchema, + isMetaFallback: false, + url: decryptDataDbUrl(preview.encryptedUrl), + }; + } + + private async resolveSourceSpaceDataDb( + options?: IDataDbRoutingOptions + ): Promise { + const sourceConnectionId = options?.sourceConnectionId ?? null; + if (!sourceConnectionId) { + return { isMetaFallback: true }; + } + + const connection = await this.getMetaRoutingClient(options).dataDbConnection.findUnique({ + where: { id: sourceConnectionId }, + }); + if (!connection?.encryptedUrl) { + throw new Error(`Data database source connection ${sourceConnectionId} was not found`); + } + + return { + connectionId: connection.id, + internalSchema: connection.internalSchema, + isMetaFallback: false, + url: decryptDataDbUrl(connection.encryptedUrl), + }; + } + async onModuleDestroy() { await Promise.all([ this.runtimeCache.deleteByNamespace(DATA_DB_KNEX_CACHE_NAMESPACE), diff --git a/apps/nestjs-backend/src/global/database-router.service.spec.ts b/apps/nestjs-backend/src/global/database-router.service.spec.ts index 788cddab35..a37c2eacaa 100644 --- a/apps/nestjs-backend/src/global/database-router.service.spec.ts +++ b/apps/nestjs-backend/src/global/database-router.service.spec.ts @@ -2,6 +2,29 @@ import { describe, expect, it, vi } from 'vitest'; import { DatabaseRouter } from './database-router.service'; describe('DatabaseRouter', () => { + it('exposes resolved base data DB routing metadata', async () => { + const resolved = { + cacheKey: 'dcnxxx', + connectionId: 'dcnxxx', + internalSchema: 'teable_meta_x', + isMetaFallback: false, + url: 'postgresql://readonly:secret@byodb.example.com:5432/teable?schema=teable_meta_x', + }; + const dataDbClientManager = { + getDataDatabaseForBase: vi.fn().mockResolvedValue(resolved), + }; + const router = new DatabaseRouter( + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never + ); + + await expect(router.getDataDatabaseForBase('bsexxx')).resolves.toBe(resolved); + expect(dataDbClientManager.getDataDatabaseForBase).toHaveBeenCalledWith('bsexxx', undefined); + }); + it('executes table scoped raw queries through the scoped table client', async () => { const queryRaw = vi.fn().mockResolvedValue([{ count: 1 }]); const executeRaw = vi.fn().mockResolvedValue(1); @@ -61,6 +84,74 @@ describe('DatabaseRouter', () => { expect(queryRaw).toHaveBeenCalledWith('select * from x where id = $1', 'recxxx'); }); + it('clears cached query plans and retries once after schema result type changes', async () => { + const cachedPlanError = { + code: 'P2010', + meta: { + code: '0A000', + message: 'ERROR: cached plan must not change result type', + }, + }; + const queryRaw = vi + .fn() + .mockRejectedValueOnce(cachedPlanError) + .mockResolvedValueOnce([{ id: 'recxxx' }]); + const executeRaw = vi.fn().mockResolvedValue(0); + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue({ + txClient: () => ({ + $queryRawUnsafe: queryRaw, + $executeRawUnsafe: executeRaw, + }), + }), + }; + const router = new DatabaseRouter( + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never + ); + + await expect(router.queryDataPrismaForTable('tblxxx', 'select * from x')).resolves.toEqual([ + { id: 'recxxx' }, + ]); + expect(executeRaw).toHaveBeenCalledWith('DISCARD PLANS'); + expect(queryRaw).toHaveBeenCalledTimes(2); + }); + + it('does not clear cached query plans while using an explicit transaction client', async () => { + const cachedPlanError = { + code: 'P2010', + meta: { + code: '0A000', + message: 'ERROR: cached plan must not change result type', + }, + }; + const queryRaw = vi.fn().mockRejectedValue(cachedPlanError); + const executeRaw = vi.fn(); + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue({ + txClient: () => ({ + $queryRawUnsafe: queryRaw, + $executeRawUnsafe: executeRaw, + }), + }), + }; + const router = new DatabaseRouter( + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never + ); + + await expect( + router.queryDataPrismaForTable('tblxxx', 'select * from x', { useTransaction: true }) + ).rejects.toBe(cachedPlanError); + expect(executeRaw).not.toHaveBeenCalled(); + }); + it('executes table scoped transactions with PrismaClient transaction fallback', async () => { const executeRaw = vi.fn().mockResolvedValue(1); const transaction = vi.fn(async (fn) => await fn({ $executeRawUnsafe: executeRaw })); diff --git a/apps/nestjs-backend/src/global/database-router.service.ts b/apps/nestjs-backend/src/global/database-router.service.ts index 3ffacbb7e6..ec3ee480d0 100644 --- a/apps/nestjs-backend/src/global/database-router.service.ts +++ b/apps/nestjs-backend/src/global/database-router.service.ts @@ -32,6 +32,24 @@ type IDataPrismaScopedClient = IDataPrismaQueryExecutor & { ) => Promise; }; +const isCachedPlanResultTypeError = (error: unknown): boolean => { + const err = error as { + code?: unknown; + meta?: { code?: unknown; message?: unknown }; + message?: unknown; + }; + const pgCode = typeof err.meta?.code === 'string' ? err.meta.code : undefined; + const messageParts = [err.message, err.meta?.message].filter( + (part): part is string => typeof part === 'string' + ); + + return ( + err.code === 'P2010' && + pgCode === '0A000' && + messageParts.some((message) => message.includes('cached plan must not change result type')) + ); +}; + @Injectable() export class DatabaseRouter { constructor( @@ -74,6 +92,10 @@ export class DatabaseRouter { return await this.dataDbClientManager.getDataDatabaseUrlForBase(baseId, options); } + async getDataDatabaseForBase(baseId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.getDataDatabaseForBase(baseId, options); + } + async dataKnexForSpace(spaceId: string, options?: IDataDbRoutingOptions) { return await this.dataDbClientManager.dataKnexForSpace(spaceId, options); } @@ -102,6 +124,24 @@ export class DatabaseRouter { return prisma.txClient?.() ?? prisma; } + private async queryWithCachedPlanRetry( + prisma: IDataPrismaQueryExecutor, + query: string, + queryValues: unknown[], + shouldRetry: boolean + ): Promise { + try { + return await prisma.$queryRawUnsafe(query, ...queryValues); + } catch (error) { + if (!shouldRetry || !isCachedPlanResultTypeError(error)) { + throw error; + } + + await prisma.$executeRawUnsafe('DISCARD PLANS'); + return await prisma.$queryRawUnsafe(query, ...queryValues); + } + } + async dataPrismaExecutorForTable( tableId: string, options?: IDataDbRoutingOptions @@ -126,7 +166,12 @@ export class DatabaseRouter { ): Promise { const { options, queryValues } = this.normalizeRoutingOptions(optionsOrFirstValue, values); const prisma = await this.dataPrismaExecutorForTable(tableId, options); - return await prisma.$queryRawUnsafe(query, ...queryValues); + return await this.queryWithCachedPlanRetry( + prisma, + query, + queryValues, + !options?.useTransaction + ); } async executeDataPrismaForTable( @@ -148,7 +193,12 @@ export class DatabaseRouter { ): Promise { const { options, queryValues } = this.normalizeRoutingOptions(optionsOrFirstValue, values); const prisma = await this.dataPrismaExecutorForBase(baseId, options); - return await prisma.$queryRawUnsafe(query, ...queryValues); + return await this.queryWithCachedPlanRetry( + prisma, + query, + queryValues, + !options?.useTransaction + ); } async executeDataPrismaForBase( @@ -215,11 +265,12 @@ export class DatabaseRouter { } private isRoutingOptions(value: unknown): value is IDataDbRoutingOptions { + const routingOptionKeys = new Set(['useTransaction', 'previewBinding']); return ( Boolean(value) && typeof value === 'object' && Object.keys(value as Record).length > 0 && - Object.keys(value as Record).every((key) => key === 'useTransaction') + Object.keys(value as Record).every((key) => routingOptionKeys.has(key)) ); } diff --git a/apps/nestjs-backend/src/global/global.module.ts b/apps/nestjs-backend/src/global/global.module.ts index e269ebd3a9..e4321aca15 100644 --- a/apps/nestjs-backend/src/global/global.module.ts +++ b/apps/nestjs-backend/src/global/global.module.ts @@ -26,6 +26,7 @@ import { PermissionModule } from '../features/auth/permission.module'; import { DataLoaderModule } from '../features/data-loader/data-loader.module'; import { ModelModule } from '../features/model/model.module'; import { DataDbMigrationService } from '../features/space/data-db-migration.service'; +import { SpaceDataDbMigrationGuardService } from '../features/space/space-data-db-migration-guard.service'; import { RequestInfoMiddleware } from '../middleware/request-info.middleware'; import { SessionCsrfMiddleware } from '../middleware/session-csrf.middleware'; import { PerformanceCacheModule } from '../performance-cache'; @@ -100,6 +101,7 @@ const globalModules = { DataDbRuntimeCacheService, DataDbClientManager, DataDbMigrationService, + SpaceDataDbMigrationGuardService, DatabaseRouter, RequestInfoMiddleware, SessionCsrfMiddleware, @@ -121,6 +123,7 @@ const globalModules = { DataDbRuntimeCacheService, DataDbClientManager, DataDbMigrationService, + SpaceDataDbMigrationGuardService, DatabaseRouter, KnexModule, PrismaModule, diff --git a/apps/nestjs-backend/src/middleware/request-info.middleware.spec.ts b/apps/nestjs-backend/src/middleware/request-info.middleware.spec.ts new file mode 100644 index 0000000000..2dbc1ed1c5 --- /dev/null +++ b/apps/nestjs-backend/src/middleware/request-info.middleware.spec.ts @@ -0,0 +1,128 @@ +import type { Request, Response } from 'express'; +import { describe, expect, it, vi } from 'vitest'; +import type { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../types/cls'; +import { RequestInfoMiddleware } from './request-info.middleware'; + +const createRequest = (overrides: Partial = {}): Request => + ({ + headers: {}, + ip: '127.0.0.1', + socket: { remoteAddress: '127.0.0.1' }, + ...overrides, + }) as Request; + +describe('RequestInfoMiddleware', () => { + it('runs v2 background tasks only after the HTTP response finishes', () => { + const globalWithTimeout = globalThis as { + setTimeout: typeof setTimeout; + }; + const originalSetTimeout = globalWithTimeout.setTimeout; + const timers: Array<() => void> = []; + globalWithTimeout.setTimeout = ((callback: () => void) => { + timers.push(callback); + return { unref: vi.fn() }; + }) as unknown as typeof setTimeout; + + try { + const clsValues = new Map(); + const cls = { + get: vi.fn(() => undefined), + runWith: vi.fn((_store: IClsStore, callback: () => void) => callback()), + set: vi.fn((key: string, value: unknown) => { + clsValues.set(key, value); + }), + } as unknown as ClsService; + const listeners = new Map void>(); + const res = { + once: vi.fn((event: string, listener: () => void) => { + listeners.set(event, listener); + return res; + }), + writableEnded: false, + destroyed: false, + } as unknown as Response; + const next = vi.fn(); + const middleware = new RequestInfoMiddleware(cls); + + middleware.use(createRequest(), res, next); + + const schedule = clsValues.get('scheduleV2BackgroundTask') as NonNullable< + IClsStore['scheduleV2BackgroundTask'] + >; + const task = vi.fn(); + + schedule(task); + + expect(next).toHaveBeenCalledWith(); + expect(task).not.toHaveBeenCalled(); + expect(timers).toHaveLength(0); + + listeners.get('finish')?.(); + + expect(task).not.toHaveBeenCalled(); + expect(timers).toHaveLength(1); + + timers.shift()?.(); + + expect(task).toHaveBeenCalledTimes(1); + } finally { + globalWithTimeout.setTimeout = originalSetTimeout; + } + }); + + it('runs v2 background tasks with the CLS store captured when scheduled', () => { + const globalWithTimeout = globalThis as { + setTimeout: typeof setTimeout; + }; + const originalSetTimeout = globalWithTimeout.setTimeout; + const timers: Array<() => void> = []; + globalWithTimeout.setTimeout = ((callback: () => void) => { + timers.push(callback); + return { unref: vi.fn() }; + }) as unknown as typeof setTimeout; + + try { + const clsValues = new Map(); + const scheduledStore = { + audit: { + rootAction: 'table.duplicate', + operationId: 'op_1', + }, + } as IClsStore; + const cls = { + get: vi.fn(() => scheduledStore), + runWith: vi.fn((_store: IClsStore, callback: () => void) => callback()), + set: vi.fn((key: string, value: unknown) => { + clsValues.set(key, value); + }), + } as unknown as ClsService; + const listeners = new Map void>(); + const res = { + once: vi.fn((event: string, listener: () => void) => { + listeners.set(event, listener); + return res; + }), + writableEnded: false, + destroyed: false, + } as unknown as Response; + const middleware = new RequestInfoMiddleware(cls); + + middleware.use(createRequest(), res, vi.fn()); + + const schedule = clsValues.get('scheduleV2BackgroundTask') as NonNullable< + IClsStore['scheduleV2BackgroundTask'] + >; + const task = vi.fn(); + + schedule(task); + listeners.get('finish')?.(); + timers.shift()?.(); + + expect(cls.runWith).toHaveBeenCalledWith(scheduledStore, expect.any(Function)); + expect(task).toHaveBeenCalledTimes(1); + } finally { + globalWithTimeout.setTimeout = originalSetTimeout; + } + }); +}); diff --git a/apps/nestjs-backend/src/middleware/request-info.middleware.ts b/apps/nestjs-backend/src/middleware/request-info.middleware.ts index f2944d3846..19605d18c2 100644 --- a/apps/nestjs-backend/src/middleware/request-info.middleware.ts +++ b/apps/nestjs-backend/src/middleware/request-info.middleware.ts @@ -6,6 +6,57 @@ import { ClsService } from 'nestjs-cls'; import type { IClsStore } from '../types/cls'; const automationRobotUserId = 'automationRobot'; +const fallbackScheduleV2BackgroundTask: NonNullable = ( + task +) => { + const handle = setTimeout(() => void task(), 0); + handle.unref?.(); +}; + +const createAfterResponseScheduler = ( + cls: ClsService, + res: Response +): NonNullable => { + const pendingTasks: Array<() => Promise | void> = []; + let responseFinished = res.writableEnded || res.destroyed; + let flushScheduled = false; + + const scheduleFlush = () => { + if (flushScheduled) { + return; + } + flushScheduled = true; + const handle = setTimeout(() => { + flushScheduled = false; + const tasks = pendingTasks.splice(0); + for (const task of tasks) { + void task(); + } + }, 0); + handle.unref?.(); + }; + + const markResponseFinished = () => { + responseFinished = true; + scheduleFlush(); + }; + + res.once('finish', markResponseFinished); + res.once('close', markResponseFinished); + + return (task) => { + const store = cls.get(); + pendingTasks.push(() => { + if (store) { + return cls.runWith(store, task); + } + return task(); + }); + if (responseFinished) { + scheduleFlush(); + } + }; +}; @Injectable() export class RequestInfoMiddleware implements NestMiddleware { @@ -51,6 +102,11 @@ export class RequestInfoMiddleware implements NestMiddleware { this.cls.set('canaryHeader', canaryHeader); } + this.cls.set( + 'scheduleV2BackgroundTask', + res ? createAfterResponseScheduler(this.cls, res) : fallbackScheduleV2BackgroundTask + ); + next(); } } diff --git a/apps/nestjs-backend/src/share-db/share-db.adapter.spec.ts b/apps/nestjs-backend/src/share-db/share-db.adapter.spec.ts new file mode 100644 index 0000000000..230d6bddfa --- /dev/null +++ b/apps/nestjs-backend/src/share-db/share-db.adapter.spec.ts @@ -0,0 +1,84 @@ +import type { EditOp } from 'sharedb'; +import { describe, expect, it, vi } from 'vitest'; +import { ShareDbAdapter } from './share-db.adapter'; + +const createAdapter = () => { + const adapter = Object.create(ShareDbAdapter.prototype) as ShareDbAdapter; + Object.assign(adapter, { logger: { log: vi.fn() } }); + return adapter; +}; + +const editOp = (subOps: unknown[]) => ({ op: subOps }) as unknown as EditOp; + +const collection = 'rec_tblTest0000000001'; + +const filteredQuery = { + ignoreViewQuery: true, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldFiltered00000001', operator: 'is', value: '1' }], + }, +}; + +describe('ShareDbAdapter skipPoll', () => { + it('always polls for create and delete ops', () => { + const adapter = createAdapter(); + expect(adapter.skipPoll(collection, 'rec1', { create: {} } as never, filteredQuery)).toBe( + false + ); + expect(adapter.skipPoll(collection, 'rec1', { del: true } as never, filteredQuery)).toBe(false); + }); + + it('polls when the view row order pseudo column changes', () => { + const adapter = createAdapter(); + const op = editOp([{ p: ['fields', '__row_viwaTuM2nkzlPWclDH9'], oi: 3.375, od: 3 }]); + expect(adapter.skipPoll(collection, 'rec1', op, filteredQuery)).toBe(false); + }); + + it('polls when an op mixes field values with pseudo columns', () => { + const adapter = createAdapter(); + const op = editOp([ + { p: ['fields', 'fldUnrelated0000001'], oi: 'x' }, + { p: ['fields', '__row_viwaTuM2nkzlPWclDH9'], oi: 3.375, od: 3 }, + ]); + expect(adapter.skipPoll(collection, 'rec1', op, filteredQuery)).toBe(false); + }); + + it('skips polling when modified fields do not affect the query', () => { + const adapter = createAdapter(); + const op = editOp([{ p: ['fields', 'fldUnrelated0000001'], oi: 'x' }]); + expect(adapter.skipPoll(collection, 'rec1', op, filteredQuery)).toBe(true); + }); + + it('polls when a modified field is referenced by the query filter', () => { + const adapter = createAdapter(); + const op = editOp([{ p: ['fields', 'fldFiltered00000001'], oi: 'x' }]); + expect(adapter.skipPoll(collection, 'rec1', op, filteredQuery)).toBe(false); + }); + + it('polls when a modified field is referenced by the record read filter', () => { + const adapter = createAdapter(); + const op = editOp([{ p: ['fields', 'fldAuthority0000001'], oi: 'x' }]); + const query = { + ...filteredQuery, + recordReadFilter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldAuthority0000001', operator: 'is', value: 'me' }], + }, + }; + expect(adapter.skipPoll(collection, 'rec1', op, query)).toBe(false); + }); + + it('always polls for plain viewId queries whose view config lives server side', () => { + const adapter = createAdapter(); + const op = editOp([{ p: ['fields', 'fldUnrelated0000001'], oi: 'x' }]); + expect(adapter.skipPoll(collection, 'rec1', op, { viewId: 'viwaTuM2nkzlPWclDH9' })).toBe(false); + }); + + it('polls for a global filtering search (unbounded field scope)', () => { + const adapter = createAdapter(); + const op = editOp([{ p: ['fields', 'fldUnrelated0000001'], oi: 'x' }]); + const query = { ignoreViewQuery: true, search: ['hello', '', true] }; + expect(adapter.skipPoll(collection, 'rec1', op, query)).toBe(false); + }); +}); diff --git a/apps/nestjs-backend/src/share-db/share-db.adapter.ts b/apps/nestjs-backend/src/share-db/share-db.adapter.ts index 3e2519a5cb..eb82006d35 100644 --- a/apps/nestjs-backend/src/share-db/share-db.adapter.ts +++ b/apps/nestjs-backend/src/share-db/share-db.adapter.ts @@ -8,19 +8,22 @@ import { import type { IFieldPropertyKey, IFieldVo, + IFilter, IOtOperation, IRecord, ISnapshotBase, ITablePropertyKey, } from '@teable/core'; import { + collectQueryFieldIds, + extractFieldIdsFromFilter, FieldOpBuilder, getRandomString, IdPrefix, RecordOpBuilder, TableOpBuilder, } from '@teable/core'; -import type { ITableVo } from '@teable/openapi'; +import type { IGetRecordsRo, ITableVo } from '@teable/openapi'; import { omit } from 'lodash'; import { ClsService } from 'nestjs-cls'; import type { CreateOp, DeleteOp, EditOp } from 'sharedb'; @@ -53,6 +56,11 @@ type IProjection = { [fieldNameOrId: string]: boolean }; export class ShareDbAdapter extends ShareDb.DB { private logger = new Logger(ShareDbAdapter.name); + // Read by sharedb QueryEmitter (lib/query-emitter.js): ops arriving while a + // poll is in flight or within this window are coalesced into a single + // trailing poll, instead of one poll per op. + pollDebounce = Number(process.env.SHAREDB_QUERY_POLL_DEBOUNCE_MS ?? 200); + closed: boolean; constructor( @@ -167,15 +175,61 @@ export class ShareDbAdapter extends ShareDb.DB { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore skipPoll( - _collection: string, + collection: string, _id: string, op: CreateOp | DeleteOp | EditOp, - _query: unknown + query: unknown ): boolean { - // ShareDB is in charge of doing the validation of ops, so at this point we - // should be able to assume that the op is structured validly if (op.create || op.del) return false; - return !op.op; + if (!op.op) return true; + const [docType] = collection.split('_'); + if (docType !== IdPrefix.Record) return false; + + const modifiedFieldIds = this.extractModifiedFieldIds(op.op); + if (!modifiedFieldIds || modifiedFieldIds.size === 0) return false; + + // subscribers may attach their authority-matrix read filter; rows enter or + // leave their visible set when its fields change, so those fields are + // relevant too. Advisory only: read enforcement never relies on it + const recordQuery = query as + | (IGetRecordsRo & { recordReadFilter?: IFilter }) + | null + | undefined; + if (!recordQuery) return false; + // a viewId query depends on the view's server-side filter/sort, whose + // fields are not visible here, so it must always poll + if (recordQuery.viewId && !recordQuery.ignoreViewQuery) return false; + + const queryFieldIds = collectQueryFieldIds(recordQuery); + if (!queryFieldIds) return false; + for (const fieldId of extractFieldIdsFromFilter(recordQuery.recordReadFilter, true)) { + queryFieldIds.add(fieldId); + } + + for (const fieldId of modifiedFieldIds) { + if (queryFieldIds.has(fieldId)) return false; + } + this.logger.log( + `skipping poll for op on ${collection} ${_id} because modified fields do not affect the query` + ); + return true; + } + + // Returns null when the op touches anything that is not a plain field value + // (e.g. the view row order pseudo column fields.__row_), meaning the + // op may affect query order/membership in ways field analysis cannot see. + private extractModifiedFieldIds(ops: IOtOperation[]): Set | null { + const fieldIds = new Set(); + for (const subOp of ops) { + if (subOp.p?.[0] !== 'fields') continue; + // core's SetRecordBuilder owns the ['fields', fieldId] op path schema + const fieldId = RecordOpBuilder.editor.setRecord.detect(subOp)?.fieldId; + if (typeof fieldId !== 'string' || !fieldId.startsWith(IdPrefix.Field)) { + return null; + } + fieldIds.add(fieldId); + } + return fieldIds; } close(callback: () => void) { diff --git a/apps/nestjs-backend/src/tracing.ts b/apps/nestjs-backend/src/tracing.ts index c323c5431e..0f2e793fb6 100644 --- a/apps/nestjs-backend/src/tracing.ts +++ b/apps/nestjs-backend/src/tracing.ts @@ -65,7 +65,8 @@ const nativeRequire: NodeRequire = typeof __non_webpack_require__ !== 'undefined' ? __non_webpack_require__ : require; const { BatchLogRecordProcessor } = opentelemetry.logs; -const { PeriodicExportingMetricReader, AggregationType } = opentelemetry.metrics; +const { PeriodicExportingMetricReader, AggregationType, createAllowListAttributesProcessor } = + opentelemetry.metrics; const { AlwaysOnSampler } = opentelemetry.node; const otelLogger = new Logger('OpenTelemetry'); @@ -145,7 +146,6 @@ const metricExportIntervalMs = Math.max( 1000, parseNumber(getConfig('OTEL_METRIC_EXPORT_INTERVAL_MS'), 60000) ); - // Exporters const createExporterOptions = (url?: string) => ({ url, @@ -164,7 +164,9 @@ const metricsExporter = metricsEndpoint // Strip high-cardinality resource attributes from metrics only. // Traces and logs keep these for debugging; metrics drop them to prevent -// cardinality explosion in ephemeral containers (each restart = new host.name + pid). +// cardinality explosion (each restart = new host.name + pid; each deploy = +// new service.version build tag, so the unique metric series count would grow +// unbounded over time as releases accumulate). if (metricsExporter) { const dropFromMetricResource = new Set([ 'host.name', @@ -178,6 +180,7 @@ if (metricsExporter) { 'process.executable.path', 'process.owner', 'service.instance.id', + 'service.version', ]); const origExport = metricsExporter.export.bind(metricsExporter); metricsExporter.export = (metrics, cb) => { @@ -200,6 +203,19 @@ const getTraceDecision = (traceId: string): boolean => { return hash % 10000 < exportRatio * 10000; }; +// Prisma emits ~11 spans per query (operation, client:middleware/serialize, +// engine:query/connection/db_query/serialize/...). Keep only the spine +// (operation -> engine:query -> db_query) and drop the rest. The full spine is kept +// so the surviving db_query is never orphaned (which renders as a "Missing Span"). +const PRISMA_SPINE_SPANS = new Set([ + 'prisma:client:operation', + 'prisma:engine:query', + 'prisma:engine:db_query', +]); + +const isDroppedPrismaSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => + span.name.startsWith('prisma:') && !PRISMA_SPINE_SPANS.has(span.name); + const shouldExportSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => { if (exportRatio >= 1.0) return true; @@ -214,6 +230,7 @@ const shouldExportSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => const durationMs = span.duration[0] * 1000 + span.duration[1] / 1_000_000; if (durationMs > latencyThresholdMs) return true; + if (isDroppedPrismaSpan(span)) return false; // Consistent export decision based on traceId - all spans in same trace have same fate return getTraceDecision(span.spanContext().traceId); }; @@ -318,9 +335,9 @@ const ignorePaths = [ // ───────────────────────────────────────────────────────────────────────────── const drop = { type: AggregationType.DROP } as const; const buckets = (boundaries: number[]) => - ({ type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, boundaries }) as const; + ({ type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, options: { boundaries } }) as const; -const metricViews = [ +const metricViews: opentelemetry.metrics.ViewOptions[] = [ // Drop inbound HTTP metrics — 200+ routes cause cardinality explosion; // traces already provide per-request latency/status via SigNoz APM. { instrumentName: 'http.server.duration', aggregation: drop }, @@ -329,19 +346,22 @@ const metricViews = [ // Outbound HTTP — keep but drop server.address to cap cardinality // (50+ webhook hosts and growing). Only method + status remain. + // Boundaries are in seconds (instrumentation-http records seconds). { instrumentName: 'http.client.request.duration', aggregation: buckets([0.05, 0.25, 1, 5, 30]), - attributeKeys: ['http.request.method', 'http.response.status_code'], + attributesProcessors: [ + createAllowListAttributesProcessor(['http.request.method', 'http.response.status_code']), + ], }, - // Reduce high-cardinality auto-instrumented histograms from 14 → 5 buckets - // 1ms=cached, 5ms=indexed, 25ms=scan, 100ms=slow, 1s=very-slow - // Keep only operation name + system; drop db.namespace, server.address/port, etc. + // Reduce high-cardinality auto-instrumented histograms from 16 → 6 series per label set. + // Boundaries are in seconds: 1ms=cached, 5ms=indexed, 25ms=scan, 100ms=slow, 1s=very-slow. + // Keep only operation name + system; drop db.namespace, server.address/port, error.type. { instrumentName: 'db.client.operation.duration', aggregation: buckets([0.001, 0.005, 0.025, 0.1, 1]), - attributeKeys: ['db.operation.name', 'db.system'], + attributesProcessors: [createAllowListAttributesProcessor(['db.operation.name', 'db.system'])], }, ]; diff --git a/apps/nestjs-backend/src/types/cls.ts b/apps/nestjs-backend/src/types/cls.ts index da8c6f11c1..9c220e7adb 100644 --- a/apps/nestjs-backend/src/types/cls.ts +++ b/apps/nestjs-backend/src/types/cls.ts @@ -1,6 +1,7 @@ import type { Action, IFieldVo } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import type { V2Feature } from '@teable/openapi'; +import type { ExecutionContextBackgroundTaskScheduler } from '@teable/v2-core'; import type { ClsStore } from 'nestjs-cls'; import type { IAuditOperation } from '../features/audit/audit-scope'; import type { IWorkflowContext } from '../features/auth/strategies/types'; @@ -101,6 +102,7 @@ export interface IClsStore extends ClsStore { dataLoaderCache?: IDataLoaderCache; clearCacheKeys?: (keyof IPerformanceCacheStore)[]; canaryHeader?: string; // x-canary header value for canary release override + scheduleV2BackgroundTask?: ExecutionContextBackgroundTaskScheduler; useV2?: boolean; // Flag to indicate if V2 implementation should be used (set by V2FeatureGuard) v2Reason?: IV2Reason; // Reason why V2 was enabled or disabled v2Feature?: V2Feature; // The feature name that triggered V2 check diff --git a/apps/nestjs-backend/src/types/i18n.generated.ts b/apps/nestjs-backend/src/types/i18n.generated.ts index 5b53740884..8d741e188c 100644 --- a/apps/nestjs-backend/src/types/i18n.generated.ts +++ b/apps/nestjs-backend/src/types/i18n.generated.ts @@ -257,6 +257,7 @@ export type I18nTranslations = { "switchBase": string; "getMore": string; "copySuccess": string; + "openLink": string; "share": string; "clear": string; "retry": string; @@ -697,6 +698,7 @@ export type I18nTranslations = { }; "addOrgCollaborator": { "title": string; + "button": string; "placeholder": string; }; "sendInvitationSuccess": string; @@ -1334,6 +1336,7 @@ export type I18nTranslations = { "unsubscribeTime": string; "source": string; "sourceAutomationDeleted": string; + "sourceApiSend": string; "processing": string; "unsubscribeH1": string; "unsubscribeH2": string; @@ -1719,9 +1722,12 @@ export type I18nTranslations = { "sandboxCapacityFull": string; "sandboxTransient": string; "sandboxSnapshotNotFound": string; + "sandboxProviderError": string; + "sandboxProviderErrorDescription": string; "agentStartFailed": string; "idleTimeout": string; "danglingToolUse": string; + "contextImportFailed": string; }; "clickToCopyTooltip": string; "copiedTooltip": string; @@ -2930,6 +2936,11 @@ export type I18nTranslations = { "nameMaxLength": string; "descriptionMaxLength": string; }; + "validation": { + "field": { + "unique": string; + }; + }; "custom": { "fieldValueNotNull": string; "fieldValueDuplicate": string; @@ -4309,6 +4320,9 @@ export type I18nTranslations = { "medium": string; "high": string; }; + "tip": { + "gptImageResolution": string; + }; "autoFill": { "title": string; "tip": string; @@ -4979,6 +4993,7 @@ export type I18nTranslations = { "noPermission": string; "connectionCountTip": string; "createFailed": string; + "readonlyUnavailable": string; "helpLink": string; }; "view": { @@ -5205,6 +5220,11 @@ export type I18nTranslations = { "inputPlaceholder": string; "inputPlaceholderFiles": string; "thought": string; + "stage": { + "initializing": string; + "working": string; + "committing": string; + }; "meta": { "input": string; "output": string; @@ -5291,6 +5311,8 @@ export type I18nTranslations = { "fallbackName": string; "questionCount_one": string; "questionCount_other": string; + "presentFiles": string; + "download": string; }; "retry": { "interrupted": string; diff --git a/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts b/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts index 149cd03715..874baa79fc 100644 --- a/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts +++ b/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts @@ -73,6 +73,28 @@ describe('majorFieldKeysChanged', () => { ).toBe(false); }); + it('should return false if only defaultValue has changed', () => { + const singleSelectField = { + type: FieldType.SingleSelect, + name: 'Status', + dbFieldName: 'status', + options: { + choices: [{ id: 'cho1', name: 'Todo', color: 'blue' }], + defaultValue: 'Todo', + }, + } as IFieldVo; + + expect( + majorFieldKeysChanged(singleSelectField, { + ...singleSelectField, + options: { + ...singleSelectField.options, + defaultValue: undefined, + }, + }) + ).toBe(false); + }); + it('should return true if major options like expression have changed', () => { expect( majorFieldKeysChanged(formulaField, { diff --git a/apps/nestjs-backend/src/utils/major-field-keys-changed.ts b/apps/nestjs-backend/src/utils/major-field-keys-changed.ts index 359a1aa8b5..151794f9f2 100644 --- a/apps/nestjs-backend/src/utils/major-field-keys-changed.ts +++ b/apps/nestjs-backend/src/utils/major-field-keys-changed.ts @@ -11,6 +11,7 @@ export const NON_INFECT_OPTION_KEYS = new Set([ 'filter', 'sort', 'limit', + 'defaultValue', ]); export const majorOptionsKeyChanged = ( diff --git a/apps/nestjs-backend/test/action-trigger-set-record.e2e-spec.ts b/apps/nestjs-backend/test/action-trigger-set-record.e2e-spec.ts new file mode 100644 index 0000000000..686ba470ed --- /dev/null +++ b/apps/nestjs-backend/test/action-trigger-set-record.e2e-spec.ts @@ -0,0 +1,161 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { axios, X_CANARY_HEADER } from '@teable/openapi'; +import { ShareDbService } from '../src/share-db/share-db.service'; +import type { IActionTrigger } from './utils/action-trigger'; +import { collectActionTriggers } from './utils/action-trigger'; +import { createRecords, createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +const v2ResponseHeader = 'x-teable-v2'; + +// field-aware listeners (use-field-aware-table-listener) and the sharedb +// skipPoll contract rely on every setRecord presence event carrying the +// changed cell fieldIds, for both the v1 listener and the v2 projection +describe('Action trigger setRecord presence (e2e)', () => { + let app: INestApplication; + let cookie: string; + let port: string; + let shareDbService: ShareDbService; + const tableIds = new Set(); + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + cookie = appCtx.cookie; + port = process.env.PORT!; + shareDbService = app.get(ShareDbService); + }); + + afterAll(async () => { + for (const tableId of [...tableIds].reverse()) { + await permanentDeleteTable(baseId, tableId); + } + await app.close(); + }); + + const setRecordFieldIds = (actions: ReadonlyArray): string[] => + actions + .filter( + (action) => action.actionKey === 'setRecord' && Array.isArray(action.payload?.fieldIds) + ) + .flatMap((action) => action.payload?.fieldIds as string[]); + + const prepareTable = async (name: string) => { + const table = await createTable(baseId, { + name, + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Note', type: FieldType.SingleLineText }, + ], + }); + tableIds.add(table.id); + + const titleFieldId = table.fields[0].id; + const noteFieldId = table.fields[1].id; + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }, { fields: {} }], + }); + return { table, titleFieldId, noteFieldId, records }; + }; + + const updateSingleRecord = async (params: { + tableId: string; + recordId: string; + fieldId: string; + value: string; + useV2: boolean; + }) => { + const { tableId, recordId, fieldId, value, useV2 } = params; + const response = await axios.patch( + `/table/${tableId}/record/${recordId}`, + { record: { fields: { [fieldId]: value } }, fieldKeyType: FieldKeyType.Id }, + { headers: useV2 ? { [X_CANARY_HEADER]: 'true' } : {} } + ); + expect(response.status).toBe(200); + if (useV2) { + expect(response.headers[v2ResponseHeader]).toBe('true'); + } else { + expect(response.headers[v2ResponseHeader]).toBeUndefined(); + } + }; + + const updateRecordsBatch = async (params: { + tableId: string; + updates: { id: string; fields: Record }[]; + useV2: boolean; + }) => { + const { tableId, updates, useV2 } = params; + const response = await axios.patch( + `/table/${tableId}/record`, + { records: updates, fieldKeyType: FieldKeyType.Id }, + { headers: useV2 ? { [X_CANARY_HEADER]: 'true' } : {} } + ); + expect(response.status).toBe(200); + if (useV2) { + expect(response.headers[v2ResponseHeader]).toBe('true'); + } else { + expect(response.headers[v2ResponseHeader]).toBeUndefined(); + } + }; + + describe.each([ + { engine: 'v1', useV2: false }, + { engine: 'v2', useV2: true }, + ])('$engine engine', ({ engine, useV2 }) => { + it('emits setRecord with the changed fieldIds for a single record update', async () => { + const { table, titleFieldId, noteFieldId, records } = await prepareTable( + `set-record-presence-single-${engine}` + ); + + const actions = await collectActionTriggers({ + shareDbService, + cookie, + port, + tableId: table.id, + until: (received) => setRecordFieldIds(received).includes(noteFieldId), + act: () => + updateSingleRecord({ + tableId: table.id, + recordId: records[0].id, + fieldId: noteFieldId, + value: `${engine} single`, + useV2, + }), + }); + + expect(setRecordFieldIds(actions)).toEqual([noteFieldId]); + expect(setRecordFieldIds(actions)).not.toContain(titleFieldId); + }); + + it('emits setRecord with the union of changed fieldIds for a batch update', async () => { + const { table, titleFieldId, noteFieldId, records } = await prepareTable( + `set-record-presence-batch-${engine}` + ); + + const expected = new Set([titleFieldId, noteFieldId]); + const actions = await collectActionTriggers({ + shareDbService, + cookie, + port, + tableId: table.id, + until: (received) => { + const ids = new Set(setRecordFieldIds(received)); + return [...expected].every((id) => ids.has(id)); + }, + act: () => + updateRecordsBatch({ + tableId: table.id, + updates: [ + { id: records[0].id, fields: { [titleFieldId]: `${engine} batch title` } }, + { id: records[1].id, fields: { [noteFieldId]: `${engine} batch note` } }, + ], + useV2, + }), + }); + + expect(new Set(setRecordFieldIds(actions))).toEqual(expected); + }); + }); +}); diff --git a/apps/nestjs-backend/test/aggregation.e2e-spec.ts b/apps/nestjs-backend/test/aggregation.e2e-spec.ts index cd0378bfcc..dd2c445d0a 100644 --- a/apps/nestjs-backend/test/aggregation.e2e-spec.ts +++ b/apps/nestjs-backend/test/aggregation.e2e-spec.ts @@ -1466,6 +1466,48 @@ describe('OpenAPI AggregationController (e2e)', () => { }); }); + describe('row count with search projection', () => { + let table: ITableFullVo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'agg_row_count_projection', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Note', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Title: 'apple', Note: 'banana' } }, + { fields: { Title: 'banana', Note: 'cherry' } }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should exclude search hits on fields outside the projection', async () => { + const viewId = table.views[0].id; + const search: [string, string?, boolean?] = ['banana', '', true]; + + const { rowCount: withoutProjection } = (await getRowCount(table.id, { viewId, search })) + .data; + expect(withoutProjection).toEqual(2); + + // simulate a personal view that hides the Note field: only Title is searched + const { rowCount } = ( + await getRowCount(table.id, { + viewId, + ignoreViewQuery: true, + search, + projection: [table.fields[0].id], + }) + ).data; + expect(rowCount).toEqual(1); + }); + }); + describe('attachment total size aggregation with groupBy', () => { let tableId: string; let groupFieldId: string; diff --git a/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts b/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts index ebaf7841ab..d56ca01951 100644 --- a/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts +++ b/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts @@ -912,6 +912,157 @@ describe('OpenAPI Base Duplicate (e2e)', () => { ); }); + it('duplicates bidirectional link records through v2 stream copy', async () => { + const sourceTable = await createTable(base.id, { name: 'V2 Source', records: [] }); + const linkedTable = await createTable(base.id, { name: 'V2 Linked', records: [] }); + const linkField = ( + await createField(sourceTable.id, { + name: 'Links', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: linkedTable.id, + }, + }) + ).data; + const symmetricField = ( + await getField( + linkedTable.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ) + ).data; + const linkedRecords = await createRecords(linkedTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }, { fields: {} }], + }); + await createRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { fields: { [linkField.name]: [{ id: linkedRecords.records[0].id }] } }, + { fields: { [linkField.name]: [{ id: linkedRecords.records[1].id }] } }, + ], + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'v2 stream link copy', + withRecords: true, + }); + duplicateBaseId = dupResult.data.id; + + const duplicatedTables = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedSourceTable = duplicatedTables.find(({ name }) => name === sourceTable.name)!; + const duplicatedLinkedTable = duplicatedTables.find(({ name }) => name === linkedTable.name)!; + const duplicatedSourceRecords = await getRecords(duplicatedSourceTable.id); + const duplicatedLinkedRecords = await getRecords(duplicatedLinkedTable.id); + + expect(duplicatedSourceRecords.records[0].fields[linkField.name]).toMatchObject([ + { id: duplicatedLinkedRecords.records[0].id }, + ]); + expect(duplicatedSourceRecords.records[1].fields[linkField.name]).toMatchObject([ + { id: duplicatedLinkedRecords.records[1].id }, + ]); + expect(duplicatedLinkedRecords.records[0].fields[symmetricField.name]).toMatchObject([ + { id: duplicatedSourceRecords.records[0].id }, + ]); + expect(duplicatedLinkedRecords.records[1].fields[symmetricField.name]).toMatchObject([ + { id: duplicatedSourceRecords.records[1].id }, + ]); + }); + + it('downgrades cross-space cross-base link records through v2 stream copy', async () => { + const externalBase = (await createBase({ spaceId, name: 'V2 External Base' })).data; + const destSpace = (await createSpace({ name: 'v2 duplicate dest space' })).data; + + try { + const externalTable = await createTable(externalBase.id, { + name: 'V2 Vendors', + records: [], + }); + const externalPrimaryField = externalTable.fields.find(({ isPrimary }) => isPrimary)!; + const vendorRecord = ( + await createRecords(externalTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [externalPrimaryField.id]: 'Vendor A' } }], + }) + ).records[0]; + + const sourceTable = await createTable(base.id, { name: 'V2 Orders', records: [] }); + const sourcePrimaryField = sourceTable.fields.find(({ isPrimary }) => isPrimary)!; + const vendorLinkField = ( + await createField(sourceTable.id, { + name: 'Vendor', + type: FieldType.Link, + options: { + baseId: externalBase.id, + relationship: Relationship.ManyMany, + foreignTableId: externalTable.id, + }, + }) + ).data; + const vendorLookupField = ( + await createField(sourceTable.id, { + name: 'Vendor Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: externalTable.id, + linkFieldId: vendorLinkField.id, + lookupFieldId: externalPrimaryField.id, + }, + }) + ).data; + + await createRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [sourcePrimaryField.id]: 'Order 1', + [vendorLinkField.id]: [{ id: vendorRecord.id }], + }, + }, + ], + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: destSpace.id, + name: 'v2 cross base link downgrade', + withRecords: true, + }); + duplicateBaseId = dupResult.data.id; + + const duplicatedTables = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedSourceTable = duplicatedTables.find( + ({ name }) => name === sourceTable.name + )!; + const duplicatedFields = (await getFields(duplicatedSourceTable.id)).data; + const duplicatedVendorLinkField = duplicatedFields.find( + ({ name }) => name === vendorLinkField.name + )!; + const duplicatedVendorLookupField = duplicatedFields.find( + ({ name }) => name === vendorLookupField.name + )!; + expect(duplicatedVendorLinkField.type).toBe(FieldType.SingleLineText); + expect(duplicatedVendorLookupField.type).toBe(FieldType.SingleLineText); + expect(duplicatedVendorLookupField.isLookup).toBeFalsy(); + + const duplicatedRecords = await getRecords(duplicatedSourceTable.id); + const duplicatedRow = duplicatedRecords.records[0]; + expect(duplicatedRow.fields[duplicatedVendorLinkField.name]).toBe('Vendor A'); + expect(duplicatedRow.fields[duplicatedVendorLookupField.name]).toContain('Vendor A'); + } finally { + await permanentDeleteBase(externalBase.id); + if (duplicateBaseId) { + await permanentDeleteBase(duplicateBaseId); + duplicateBaseId = undefined; + } + await permanentDeleteSpace(destSpace.id); + } + }); + it('duplicates formula, link, lookup, rollup, bidirectional link, and ai field config through v2', async () => { const peopleTable = await createTable(base.id, { name: 'People' }); const taskTable = await createTable(base.id, { name: 'Tasks' }); diff --git a/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts b/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts index de51fd52c4..7277bf0492 100644 --- a/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts +++ b/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts @@ -1,4 +1,6 @@ /* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { spawnSync } from 'child_process'; import fs from 'fs'; import path from 'path'; import type { INestApplication } from '@nestjs/common'; @@ -16,6 +18,7 @@ import { getRecordHistory, getRowCount, getSignature as apiGetSignature, + getSpaceDataDbMigration, getTableList, getTableActivatedIndex, GroupPointType, @@ -25,6 +28,7 @@ import { notify as apiNotify, redo, ResourceType, + restoreTrash, SettingKey, SUPPORTEDTYPE, TableIndex, @@ -37,6 +41,7 @@ import { uploadFile as apiUploadFile, X_CANARY_HEADER, type ITableFullVo, + type IDataDbMigrationJobStatusVo, } from '@teable/openapi'; import Knex from 'knex'; import type { Knex as KnexType } from 'knex'; @@ -52,6 +57,8 @@ import { X_TEABLE_V2_REASON_HEADER, } from '../src/features/canary/interceptors/v2-indicator.interceptor'; import { CsvImporter } from '../src/features/import/open-api/import.class'; +import { SpaceDataDbMigrationWorkerService } from '../src/features/space/space-data-db-migration-worker.service'; +import { SpaceDataDbMigrationService } from '../src/features/space/space-data-db-migration.service'; import { createAwaitWithEventWithResult } from './utils/event-promise'; import { createBase, @@ -84,12 +91,18 @@ const metaDatabaseUrl = process.env.PRISMA_META_DATABASE_URL ?? process.env.PRISMA_DATABASE_URL ?? process.env.DATABASE_URL; +const defaultDataDatabaseUrl = + process.env.PRISMA_DATABASE_URL ?? process.env.DATABASE_URL ?? metaDatabaseUrl; const byodbDataDatabaseUrl = process.env.BYODB_E2E_DATA_DATABASE_URL; const isIndependentByodbDataDb = databaseIdentity(metaDatabaseUrl) != null && databaseIdentity(byodbDataDatabaseUrl) != null && databaseIdentity(metaDatabaseUrl) !== databaseIdentity(byodbDataDatabaseUrl); const describeByodbStorage = isIndependentByodbDataDb ? describe : describe.skip; +const hasPostgresMigrationTools = ['pg_dump', 'pg_restore', 'psql'].every( + (command) => spawnSync('which', [command], { stdio: 'ignore' }).status === 0 +); +const itWithMigrationTools = hasPostgresMigrationTools ? it : it.skip; const dataPlaneSystemTables = [ '__teable_data_schema_migrations', @@ -175,6 +188,29 @@ const tableExists = async (client: KnexType, dbTableName: string) => { return relationExists(client, schemaName, tableName); }; +const undoCaptureTriggerFunctionSchema = async (client: KnexType, dbTableName: string) => { + const { schemaName, tableName } = parseDbTableName(dbTableName); + const rows = await rawRows<{ function_schema: string | null }>( + client, + ` + SELECT pn.nspname AS function_schema + FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_proc p ON p.oid = tg.tgfoid + JOIN pg_namespace pn ON pn.oid = p.pronamespace + WHERE NOT tg.tgisinternal + AND tg.tgname = '__teable_undo_capture' + AND n.nspname = ? + AND c.relname = ? + LIMIT 1 + `, + [schemaName, tableName] + ); + + return rows[0]?.function_schema ?? null; +}; + const columnExists = async (client: KnexType, dbTableName: string, columnName: string) => { const { schemaName, tableName } = parseDbTableName(dbTableName); const rows = await rawRows<{ exists: boolean }>( @@ -332,6 +368,33 @@ const waitForImportCompleted = async (tableId: string, expectedSuccessCount: num throw new Error(`BYODB import timed out with latest status: ${data.status}`); }; +type IGetMigrationStatus = (spaceId: string, jobId: string) => Promise; + +const waitForMigrationSucceeded = async ( + spaceId: string, + jobId: string, + getStatus: IGetMigrationStatus = async (statusSpaceId, statusJobId) => + (await getSpaceDataDbMigration(statusSpaceId, statusJobId)).data +) => { + const maxRetries = 240; + let latest: IDataDbMigrationJobStatusVo | undefined; + + for (let i = 0; i < maxRetries; i++) { + latest = await getStatus(spaceId, jobId); + if (latest.state === 'succeeded') { + return latest; + } + if (['failed', 'canceled', 'rolled_back'].includes(latest.state)) { + throw new Error( + `BYODB migration ${jobId} ended as ${latest.state}: ${latest.lastError ?? 'unknown error'}` + ); + } + await sleep(500); + } + + throw new Error(`BYODB migration ${jobId} timed out at state: ${latest?.state ?? 'unknown'}`); +}; + const createPgClient = (url: string) => Knex({ client: 'pg', @@ -373,6 +436,18 @@ const uploadImportCsv = async () => { } }; +const expectRequestStatus = async (request: () => Promise, status: number) => { + try { + await request(); + throw new Error(`Expected request to fail with ${status}`); + } catch (error) { + const actualStatus = + (error as { status?: number; response?: { status?: number } }).status ?? + (error as { response?: { status?: number } }).response?.status; + expect(actualStatus).toBe(status); + } +}; + const safeDropSchema = async (client: KnexType | undefined, schemaName: string | undefined) => { if (!client || !schemaName) { return; @@ -386,22 +461,55 @@ const safeDropSchema = async (client: KnexType | undefined, schemaName: string | describeByodbStorage('BYODB space storage placement (e2e)', () => { let app: INestApplication; let metaDb: KnexType; + let defaultDataDb: KnexType; let dataDb: KnexType; let baseConfigService: IBaseConfig; + let migrationService: SpaceDataDbMigrationService; + let migrationWorkerService: SpaceDataDbMigrationWorkerService; let recordHistoryDisabled: boolean | undefined; let spaceId: string | undefined; let baseId: string | undefined; const userId = globalThis.testConfig.userId; const internalSchema = `byodb_e2e_${Date.now().toString(36)}`; + const waitForTrashId = async (resourceId: string, resourceType: ResourceType) => { + const maxRetries = 60; + + for (let i = 0; i < maxRetries; i++) { + const trashRows = await rawRows<{ id: string }>( + metaDb, + ` + SELECT ${quoteIdent('id')} + FROM ${quoteIdent('trash')} + WHERE ${quoteIdent('resource_id')} = ? + AND ${quoteIdent('resource_type')} = ? + ORDER BY ${quoteIdent('deleted_time')} DESC + LIMIT 1 + `, + [resourceId, resourceType] + ); + const trashId = trashRows[0]?.id; + + if (trashId) { + return trashId; + } + + await sleep(100); + } + + throw new Error(`Timed out waiting for trash item ${resourceType}:${resourceId}`); + }; beforeAll(async () => { metaDb = createPgClient(metaDatabaseUrl!); + defaultDataDb = createPgClient(defaultDataDatabaseUrl!); dataDb = createPgClient(byodbDataDatabaseUrl!); const appCtx = await initApp(); app = appCtx.app; baseConfigService = app.get(baseConfig.KEY) as IBaseConfig; + migrationService = app.get(SpaceDataDbMigrationService); + migrationWorkerService = app.get(SpaceDataDbMigrationWorkerService); recordHistoryDisabled = baseConfigService.recordHistoryDisabled; baseConfigService.recordHistoryDisabled = false; ensureUndoRedoWindowIdHeader(`win_byodb_storage_${Date.now()}`); @@ -425,6 +533,7 @@ describeByodbStorage('BYODB space storage placement (e2e)', () => { } await dataDb?.destroy().catch(() => undefined); + await defaultDataDb?.destroy().catch(() => undefined); await metaDb?.destroy().catch(() => undefined); await app?.close(); }, 60_000); @@ -743,6 +852,355 @@ describeByodbStorage('BYODB space storage placement (e2e)', () => { await assertComputedSideEffectsStayOutOfMetaDb(base.id, mainTable.id, recordId); }, 240_000); + itWithMigrationTools( + 'migrates an existing default space and routes post-switch writes to BYODB only', + async () => { + const migrationInternalSchema = `byodb_existing_migrate_${Date.now().toString(36)}`; + let migrationSpaceId: string | undefined; + let migrationBaseId: string | undefined; + let migratedTable: ITableFullVo | undefined; + let otherSpaceId: string | undefined; + let otherBaseId: string | undefined; + let otherTable: ITableFullVo | undefined; + + try { + const space = await createSpace({ name: 'BYODB existing-space migration e2e' }); + migrationSpaceId = space.id; + const base = await createBase({ + spaceId: space.id, + name: 'BYODB existing-space migration base', + }); + migrationBaseId = base.id; + migratedTable = await createTable(base.id, { + name: 'BYODB existing-space migration table', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Before migration' } }], + }); + const primaryFieldId = migratedTable.fields.find((field) => field.isPrimary)?.id; + const sourceRecordId = migratedTable.records[0]?.id; + expect(primaryFieldId).toBeTruthy(); + expect(sourceRecordId).toBeTruthy(); + + const otherSpace = await createSpace({ + name: 'BYODB other default-space during migration e2e', + }); + otherSpaceId = otherSpace.id; + const otherBase = await createBase({ + spaceId: otherSpace.id, + name: 'BYODB other default-space migration base', + }); + otherBaseId = otherBase.id; + otherTable = await createTable(otherBase.id, { + name: 'BYODB other default-space migration table', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Other before migration' } }], + }); + const otherPrimaryFieldId = otherTable.fields.find((field) => field.isPrimary)?.id; + expect(otherPrimaryFieldId).toBeTruthy(); + + await updateRecord(migratedTable.id, sourceRecordId!, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [primaryFieldId!]: 'Updated before migration', + }, + }, + }); + + await expect(tableExists(defaultDataDb, migratedTable.dbTableName)).resolves.toBe(true); + await expect(tableExists(dataDb, migratedTable.dbTableName)).resolves.toBe(false); + await expect( + waitForAtLeast( + () => + countRows( + defaultDataDb, + 'public', + 'record_history', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable!.id, sourceRecordId] + ), + 1 + ) + ).resolves.toBeGreaterThan(0); + + const migration = await migrationService.startMigrationForSpace(space.id, userId, { + url: byodbDataDatabaseUrl!, + targetMode: 'migrate-space', + internalSchema: migrationInternalSchema, + confirmLargeMigration: true, + switchOnCompletion: true, + }); + const jobId = migration.jobId; + expect(jobId).toBeTruthy(); + + const otherDuringMigrationRecords = await createRecords(otherTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [otherPrimaryFieldId!]: 'Other during migration' } }], + }); + const otherDuringMigrationRecordId = otherDuringMigrationRecords.records[0].id; + await expect( + countDbTableRowsWhere( + defaultDataDb, + otherTable.dbTableName, + `${quoteIdent('__id')} = ?`, + [otherDuringMigrationRecordId] + ) + ).resolves.toBe(1); + await expect(tableExists(dataDb, otherTable.dbTableName)).resolves.toBe(false); + + const workerResult = await migrationWorkerService.runOnce(); + expect(workerResult?.jobId).toBe(jobId); + expect(workerResult?.status).toBe('succeeded'); + + const status = await waitForMigrationSucceeded( + space.id, + jobId, + (statusSpaceId, statusJobId) => + migrationService.getMigrationJobStatus(statusSpaceId, statusJobId) + ); + expect(status.targetInternalSchema).toBe(migrationInternalSchema); + await assertDataPlaneBaseline(migrationInternalSchema); + await expect(tableExists(defaultDataDb, migratedTable.dbTableName)).resolves.toBe(true); + await expect(tableExists(dataDb, migratedTable.dbTableName)).resolves.toBe(true); + await expect( + countDbTableRowsWhere(dataDb, migratedTable.dbTableName, `${quoteIdent('__id')} = ?`, [ + sourceRecordId, + ]) + ).resolves.toBe(1); + + const postSwitchRecords = await createRecords(migratedTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [primaryFieldId!]: 'After migration' } }], + }); + const postSwitchRecordId = postSwitchRecords.records[0].id; + await expect( + countDbTableRowsWhere(dataDb, migratedTable.dbTableName, `${quoteIdent('__id')} = ?`, [ + postSwitchRecordId, + ]) + ).resolves.toBe(1); + await expect( + countDbTableRowsWhere( + defaultDataDb, + migratedTable.dbTableName, + `${quoteIdent('__id')} = ?`, + [postSwitchRecordId] + ) + ).resolves.toBe(0); + + await updateRecord(migratedTable.id, postSwitchRecordId, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [primaryFieldId!]: 'Updated after migration', + }, + }, + }); + await expect( + waitForAtLeast( + () => + countRows( + dataDb, + migrationInternalSchema, + 'record_history', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable!.id, postSwitchRecordId] + ), + 1 + ) + ).resolves.toBeGreaterThan(0); + await expect( + countRows( + defaultDataDb, + 'public', + 'record_history', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable.id, postSwitchRecordId] + ) + ).resolves.toBe(0); + await expect( + undoCaptureTriggerFunctionSchema(dataDb, migratedTable.dbTableName) + ).resolves.toBe(migrationInternalSchema); + + await deleteRecord(migratedTable.id, postSwitchRecordId); + await expect( + waitForCount( + () => + countRows( + dataDb, + migrationInternalSchema, + 'record_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable!.id, postSwitchRecordId] + ), + 1 + ) + ).resolves.toBe(1); + await expect( + countRows( + defaultDataDb, + 'public', + 'record_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable.id, postSwitchRecordId] + ) + ).resolves.toBe(0); + + const undoResult = await undo(migratedTable.id); + expect(undoResult.data.status).toBe('fulfilled'); + await expect( + waitForCount( + () => + countRows( + dataDb, + migrationInternalSchema, + 'record_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable!.id, postSwitchRecordId] + ), + 0 + ) + ).resolves.toBe(0); + await expect( + countRows( + defaultDataDb, + 'public', + 'record_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [migratedTable.id, postSwitchRecordId] + ) + ).resolves.toBe(0); + } finally { + if (migrationBaseId) { + await permanentDeleteBase(migrationBaseId).catch(() => undefined); + } + if (otherBaseId) { + await permanentDeleteBase(otherBaseId).catch(() => undefined); + } + if (migrationSpaceId) { + await permanentDeleteSpace(migrationSpaceId).catch(() => undefined); + } + if (otherSpaceId) { + await permanentDeleteSpace(otherSpaceId).catch(() => undefined); + } + await safeDropSchema(dataDb, migrationBaseId); + await safeDropSchema(defaultDataDb, migrationBaseId); + await safeDropSchema(metaDb, migrationBaseId); + await safeDropSchema(dataDb, otherBaseId); + await safeDropSchema(defaultDataDb, otherBaseId); + await safeDropSchema(metaDb, otherBaseId); + await safeDropSchema(dataDb, migrationInternalSchema); + await safeDropSchema(metaDb, migrationInternalSchema); + } + }, + 240_000 + ); + + it('rejects import trash and schema restore writes while a space migration is active', async () => { + const freezeInternalSchema = `byodb_freeze_${Date.now().toString(36)}`; + const migrationJobId = `sdmjfreeze${Date.now().toString(36)}`; + let freezeSpaceId: string | undefined; + let freezeBaseId: string | undefined; + let restoreTableId: string | undefined; + + try { + const freezeSpace = await createSpace({ name: 'BYODB migration freeze e2e' }); + freezeSpaceId = freezeSpace.id; + const freezeBase = await createBase({ + spaceId: freezeSpace.id, + name: 'BYODB migration freeze base', + }); + freezeBaseId = freezeBase.id; + const activeTable = await createTable(freezeBase.id, { + name: 'BYODB migration freeze active table', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Active row' } }], + }); + const restoreTable = await createTable(freezeBase.id, { + name: 'BYODB migration freeze restore table', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Restore row' } }], + }); + restoreTableId = restoreTable.id; + + const attachmentUrl = await uploadImportCsv(); + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl, + fileType: SUPPORTEDTYPE.CSV, + }); + const columns = worksheets[CsvImporter.DEFAULT_SHEETKEY].columns.map((column, index) => ({ + ...column, + sourceColumnIndex: index, + })); + + await deleteTable(freezeBase.id, restoreTable.id, 200); + const trashId = await waitForTrashId(restoreTable.id, ResourceType.Table); + + await metaDb('space_data_db_migration_job').insert({ + id: migrationJobId, + space_id: freezeSpace.id, + target_mode: 'migrate-space', + switch_on_completion: true, + state: 'copying', + target_url_fingerprint: `dbfp_${migrationJobId}`, + target_internal_schema: freezeInternalSchema, + inventory: { + baseIds: [freezeBase.id], + tableIds: [activeTable.id, restoreTable.id], + dbTableNames: [activeTable.dbTableName, restoreTable.dbTableName], + }, + started_at: new Date(), + created_by: userId, + }); + + await expectRequestStatus( + () => + apiImportTableFromFile(freezeBase.id, { + attachmentUrl, + fileType: SUPPORTEDTYPE.CSV, + worksheets: { + [CsvImporter.DEFAULT_SHEETKEY]: { + name: 'BYODB migration freeze imported table', + columns, + useFirstRowAsHeader: true, + importData: true, + }, + }, + tz: 'Asia/Shanghai', + }), + 409 + ); + await deleteTable(freezeBase.id, activeTable.id, 409); + await expectRequestStatus(() => restoreTrash(trashId!), 409); + await permanentDeleteTable(freezeBase.id, restoreTable.id, 409); + + await expect(tableExists(defaultDataDb, activeTable.dbTableName)).resolves.toBe(true); + await expect( + countRows(metaDb, 'public', 'trash', `${quoteIdent('id')} = ?`, [trashId]) + ).resolves.toBe(1); + } finally { + await metaDb('space_data_db_migration_job') + .where({ id: migrationJobId }) + .delete() + .catch(() => undefined); + if (freezeBaseId && restoreTableId) { + await permanentDeleteTable(freezeBaseId, restoreTableId).catch(() => undefined); + } + if (freezeBaseId) { + await permanentDeleteBase(freezeBaseId).catch(() => undefined); + } + if (freezeSpaceId) { + await permanentDeleteSpace(freezeSpaceId).catch(() => undefined); + } + await safeDropSchema(dataDb, freezeBaseId); + await safeDropSchema(defaultDataDb, freezeBaseId); + await safeDropSchema(metaDb, freezeBaseId); + await safeDropSchema(dataDb, freezeInternalSchema); + await safeDropSchema(metaDb, freezeInternalSchema); + } + }, 180_000); + const assertMetaPlaneRows = async ( targetSpaceId: string, targetBaseId: string, @@ -1134,7 +1592,10 @@ describeByodbStorage('BYODB space storage placement (e2e)', () => { { name: 'order_count', type: FieldType.Number }, ]); - await waitForImportCompleted(importedTable.id, 2); + const isV2Import = importResult.headers[X_TEABLE_V2_HEADER] === 'true'; + if (!isV2Import) { + await waitForImportCompleted(importedTable.id, 2); + } const importedRecords = await getRecords(importedTable.id, { fieldKeyType: FieldKeyType.Name, @@ -1180,6 +1641,7 @@ describeByodbStorage('BYODB space storage placement (e2e)', () => { await expect(countDbTableRows(dataDb, importedTable.dbTableName)).resolves.toBe(2); await expect(countDbTableRows(metaDb, importedTable.dbTableName)).resolves.toBe(0); + const expectedSchemaOperationType = isV2Import ? 'table.import' : 'table.create'; await expect( countRows( metaDb, @@ -1188,7 +1650,7 @@ describeByodbStorage('BYODB space storage placement (e2e)', () => { `${quoteIdent('base_id')} = ? AND ${quoteIdent('table_id')} = ? AND ${quoteIdent( 'type' )} = ? AND ${quoteIdent('status')} = ?`, - [targetBaseId, importedTable.id, 'table.create', 'ready'] + [targetBaseId, importedTable.id, expectedSchemaOperationType, 'ready'] ) ).resolves.toBe(1); await expect( diff --git a/apps/nestjs-backend/test/cls-attribution-crossing.e2e-spec.ts b/apps/nestjs-backend/test/cls-attribution-crossing.e2e-spec.ts new file mode 100644 index 0000000000..f1e35fc447 --- /dev/null +++ b/apps/nestjs-backend/test/cls-attribution-crossing.e2e-spec.ts @@ -0,0 +1,132 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Role } from '@teable/core'; +import { + deleteSpaceCollaborator, + emailSpaceInvitation, + getRecordHistory, + PrincipalType, + UPDATE_RECORD, + urlBuilder, + USER_ME, +} from '@teable/openapi'; +import type { IUserMeVo } from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import type { IBaseConfig } from '../src/configs/base.config'; +import { baseConfig } from '../src/configs/base.config'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { + createField, + createRecords, + createTable, + initApp, + permanentDeleteTable, + updateRecord, +} from './utils/init-app'; + +/** + * Regression for T4966: v2 record-history projections must attribute history to the + * user who actually made the change, not to whoever happened to trigger the async + * drain. v2 record events are drained outside the originating request (setImmediate + * on a base-shared queue), so a concurrent request's drain ran in an unrelated user's + * CLS context and stamped `record_history.created_by` with the wrong user. The fix + * reads `context.actorId` (snapshotted per event at publish time) instead of CLS at + * drain time. + */ +describe('Record history actor attribution under concurrency (e2e)', () => { + let app: INestApplication; + let secondUserAxios: AxiosInstance; + let secondUser: IUserMeVo; + + const baseId = globalThis.testConfig.baseId; + const spaceId = globalThis.testConfig.spaceId; + const firstUserId = globalThis.testConfig.userId; + const parallelWriters = 8; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + const baseConfigService = app.get(baseConfig.KEY) as IBaseConfig; + baseConfigService.recordHistoryDisabled = false; + + const secondUserEmail = `cls-attribution-${Date.now()}@example.com`; + secondUserAxios = await createNewUserAxios({ email: secondUserEmail, password: '12345678' }); + secondUser = (await secondUserAxios.get(USER_ME)).data; + await emailSpaceInvitation({ + spaceId, + emailSpaceInvitationRo: { role: Role.Editor, emails: [secondUserEmail] }, + }); + }); + + afterAll(async () => { + await deleteSpaceCollaborator({ + spaceId, + deleteSpaceCollaboratorRo: { + principalId: secondUser.id, + principalType: PrincipalType.User, + }, + }).catch(() => undefined); + await app.close(); + }); + + const waitForHistoryCreatedBy = async ( + tableId: string, + recordId: string + ): Promise => { + const deadline = Date.now() + 15000; + while (Date.now() < deadline) { + const { data } = await getRecordHistory(tableId, recordId, {}); + if (data.historyList.length > 0) { + return data.historyList[0].createdBy; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return null; + }; + + it('attributes each record history to its acting user, not the drain-triggering user', async () => { + const table = await createTable(baseId, { name: 'cls attribution crossing' }); + try { + const noteField = await createField(table.id, { type: FieldType.SingleLineText }); + + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: Array.from({ length: parallelWriters * 2 }, () => ({ fields: {} })), + }); + + // Alternate ownership so the two users' RecordUpdated events interleave inside + // the same shared async drain batch. + const firstUserRecordIds = records.filter((_, i) => i % 2 === 0).map((r) => r.id); + const secondUserRecordIds = records.filter((_, i) => i % 2 === 1).map((r) => r.id); + + // Fire all updates concurrently: first user via the global owner session, + // second user via their own session. Distinct values guarantee a real change. + await Promise.all([ + ...firstUserRecordIds.map((recordId, i) => + updateRecord(table.id, recordId, { + record: { fields: { [noteField.id]: `first-${i}` } }, + fieldKeyType: FieldKeyType.Id, + }) + ), + ...secondUserRecordIds.map((recordId, i) => + secondUserAxios.patch(urlBuilder(UPDATE_RECORD, { tableId: table.id, recordId }), { + record: { fields: { [noteField.id]: `second-${i}` } }, + fieldKeyType: FieldKeyType.Id, + }) + ), + ]); + + const firstUserAttribution = await Promise.all( + firstUserRecordIds.map((id) => waitForHistoryCreatedBy(table.id, id)) + ); + const secondUserAttribution = await Promise.all( + secondUserRecordIds.map((id) => waitForHistoryCreatedBy(table.id, id)) + ); + + expect(firstUserAttribution).toEqual(firstUserRecordIds.map(() => firstUserId)); + expect(secondUserAttribution).toEqual(secondUserRecordIds.map(() => secondUser.id)); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts b/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts index 5d15663950..fbe624e122 100644 --- a/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts +++ b/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts @@ -4,6 +4,7 @@ import { FieldKeyType, FieldType, Relationship, Role } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { deleteSpaceCollaborator, + deleteBaseCollaborator, emailSpaceInvitation, getRecord, getRecords, @@ -30,8 +31,10 @@ import { createField, createRecords, createTable, + convertField, deleteBase, deleteField, + getField, initApp, permanentDeleteTable, } from './utils/init-app'; @@ -39,6 +42,7 @@ import { describe('Computed user field (e2e)', () => { let app: INestApplication; let v2ContainerService: V2ContainerService; + let eventEmitterService: EventEmitterService; let prisma: PrismaService; const spaceId = globalThis.testConfig.spaceId; const userName = globalThis.testConfig.userName; @@ -49,6 +53,7 @@ describe('Computed user field (e2e)', () => { const appCtx = await initApp(); app = appCtx.app; v2ContainerService = app.get(V2ContainerService); + eventEmitterService = app.get(EventEmitterService); prisma = app.get(PrismaService); const base = await createBase({ name: 'base1', spaceId }); baseId = base.id; @@ -326,14 +331,7 @@ describe('Computed user field (e2e)', () => { }); let record = await getRecord(table1.id, recordId, { fieldKeyType: FieldKeyType.Id }); - if (isV2Mode) { - expect(record.data.fields[lastModifiedByField.id]).toMatchObject({ - id: globalThis.testConfig.userId, - title: globalThis.testConfig.userName, - }); - } else { - expect(record.data.fields[lastModifiedByField.id]).toBeUndefined(); - } + expect(record.data.fields[lastModifiedByField.id]).toBeUndefined(); await updateRecord(table1.id, recordId, { record: { @@ -380,14 +378,7 @@ describe('Computed user field (e2e)', () => { }); let record = await getRecord(table1.id, recordId, { fieldKeyType: FieldKeyType.Id }); - if (isV2Mode) { - expect(record.data.fields[lastModifiedByField.id]).toMatchObject({ - id: globalThis.testConfig.userId, - title: globalThis.testConfig.userName, - }); - } else { - expect(record.data.fields[lastModifiedByField.id]).toBeUndefined(); - } + expect(record.data.fields[lastModifiedByField.id]).toBeUndefined(); await deleteField(table1.id, textField.id); @@ -446,6 +437,314 @@ describe('Computed user field (e2e)', () => { ]); expect(updatedRecord.data.fields[formulaField.id]).toContain(globalThis.testConfig.userName); }); + + it('should refresh linked lookup user field after source multi-user field changes', async () => { + const hostTable = await createTable(baseId, { name: 'lookup-user-host' }); + const secondaryUserEmail = `lookup-refresh-user-${Date.now()}@example.com`; + const secondaryUserRequest = await createNewUserAxios({ + email: secondaryUserEmail, + password: '12345678', + }); + const secondaryUser = (await secondaryUserRequest.get(USER_ME)).data; + await emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { role: Role.Creator, emails: [secondaryUserEmail] }, + }); + + try { + const sourceUserField = await createField(table1.id, { + name: 'Members', + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }); + + const linkField = await createField(hostTable.id, { + name: 'Source', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + } as ILinkFieldOptionsRo, + }); + + const lookupField = await createField(hostTable.id, { + name: 'Lookup Members', + type: FieldType.User, + isLookup: true, + lookupOptions: { + linkFieldId: linkField.id, + foreignTableId: table1.id, + lookupFieldId: sourceUserField.id, + } as ILookupOptionsRo, + }); + + const sourceRecordId = table1.records[0].id; + const hostRecordId = hostTable.records[0].id; + + await updateRecord(table1.id, sourceRecordId, { + record: { + fields: { + [sourceUserField.id]: [globalThis.testConfig.userId], + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + await updateRecord(hostTable.id, hostRecordId, { + record: { + fields: { + [linkField.id]: { id: sourceRecordId }, + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + await processV2Outbox(); + + const initialHostRecord = await getRecord(hostTable.id, hostRecordId, { + fieldKeyType: FieldKeyType.Id, + }); + expect(initialHostRecord.data.fields[lookupField.id]).toEqual([ + expect.objectContaining({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }), + ]); + + await updateRecord(table1.id, sourceRecordId, { + record: { + fields: { + [sourceUserField.id]: [globalThis.testConfig.userId, secondaryUser.id], + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + await processV2Outbox(); + + const refreshedHostRecord = await getRecord(hostTable.id, hostRecordId, { + fieldKeyType: FieldKeyType.Id, + }); + expect(refreshedHostRecord.data.fields[lookupField.id]).toEqual([ + expect.objectContaining({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }), + expect.objectContaining({ id: secondaryUser.id, title: secondaryUser.name }), + ]); + } finally { + await deleteTable(baseId, hostTable.id); + await deleteBaseCollaborator({ + baseId, + deleteBaseCollaboratorRo: { + principalId: secondaryUser.id, + principalType: PrincipalType.User, + }, + }); + } + }); + + it('should emit a legacy host lookup update event when linked multi-user source changes', async () => { + const v1Base = await createBase({ name: 'lookup-user-v1-base', spaceId }); + await prisma.base.update({ where: { id: v1Base.id }, data: { v2Enabled: false } }); + const sourceTable = await createTable(v1Base.id, { name: 'lookup-user-source-v1' }); + const hostTable = await createTable(v1Base.id, { name: 'lookup-user-host-v1' }); + const secondaryUserEmail = `lookup-refresh-v1-user-${Date.now()}@example.com`; + const secondaryUserRequest = await createNewUserAxios({ + email: secondaryUserEmail, + password: '12345678', + }); + const secondaryUser = (await secondaryUserRequest.get(USER_ME)).data; + await emailBaseInvitation({ + baseId: v1Base.id, + emailBaseInvitationRo: { role: Role.Creator, emails: [secondaryUserEmail] }, + }); + + try { + const sourceUserField = await createField(sourceTable.id, { + name: 'Members', + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }); + + const linkField = await createField(hostTable.id, { + name: 'Source', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: sourceTable.id, + lookupFieldId: sourceTable.fields[0].id, + } as ILinkFieldOptionsRo, + }); + + const lookupField = await createField(hostTable.id, { + name: 'Lookup Members', + type: FieldType.User, + isLookup: true, + lookupOptions: { + linkFieldId: linkField.id, + foreignTableId: sourceTable.id, + lookupFieldId: sourceUserField.id, + } as ILookupOptionsRo, + }); + + const sourceRecordId = sourceTable.records[0].id; + const hostRecordId = hostTable.records[0].id; + + await updateRecord(sourceTable.id, sourceRecordId, { + record: { + fields: { + [sourceUserField.id]: [globalThis.testConfig.userId], + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + await updateRecord(hostTable.id, hostRecordId, { + record: { + fields: { + [linkField.id]: { id: sourceRecordId }, + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const events: unknown[] = []; + const handler = (event: unknown) => events.push(event); + eventEmitterService.eventEmitter.on(Events.TABLE_RECORD_UPDATE, handler); + try { + await updateRecord(sourceTable.id, sourceRecordId, { + record: { + fields: { + [sourceUserField.id]: [globalThis.testConfig.userId, secondaryUser.id], + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + const deadline = Date.now() + 2000; + while ( + Date.now() < deadline && + !events.some( + (event) => + (event as { payload?: { tableId?: string } }).payload?.tableId === hostTable.id + ) + ) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } finally { + eventEmitterService.eventEmitter.off(Events.TABLE_RECORD_UPDATE, handler); + } + + const refreshedHostRecord = await getRecord(hostTable.id, hostRecordId, { + fieldKeyType: FieldKeyType.Id, + }); + expect(refreshedHostRecord.data.fields[lookupField.id]).toEqual([ + expect.objectContaining({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }), + expect.objectContaining({ id: secondaryUser.id, title: secondaryUser.name }), + ]); + + const hostEvent = events.find( + (event) => (event as { payload?: { tableId?: string } }).payload?.tableId === hostTable.id + ) as { payload?: { record?: { fields?: Record } } }; + expect(hostEvent?.payload?.record?.fields?.[lookupField.id]?.newValue).toEqual([ + expect.objectContaining({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }), + expect.objectContaining({ id: secondaryUser.id, title: secondaryUser.name }), + ]); + } finally { + await deleteTable(v1Base.id, hostTable.id); + await deleteTable(v1Base.id, sourceTable.id); + await deleteBaseCollaborator({ + baseId: v1Base.id, + deleteBaseCollaboratorRo: { + principalId: secondaryUser.id, + principalType: PrincipalType.User, + }, + }); + await deleteBase(v1Base.id); + } + }); + + it('should refresh dependent formula after converting a user field to multiple users', async () => { + if (!isV2Mode) { + return; + } + + const userField = await createField(table1.id, { + name: 'single-user-source', + type: FieldType.User, + options: { + isMultiple: false, + shouldNotify: false, + }, + }); + + const formulaField = await createField(table1.id, { + name: 'user-formula', + type: FieldType.Formula, + options: { + expression: `{${userField.id}}`, + }, + }); + + expect(formulaField.isMultipleCellValue).not.toBe(true); + + const recordId = table1.records[0].id; + await updateRecord(table1.id, recordId, { + record: { + fields: { + [userField.id]: globalThis.testConfig.userId, + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + await processV2Outbox(); + + const singleUserRecord = await getRecord(table1.id, recordId, { + fieldKeyType: FieldKeyType.Id, + }); + expect(singleUserRecord.data.fields[userField.id]).toMatchObject({ + title: globalThis.testConfig.userName, + }); + expect(singleUserRecord.data.fields[formulaField.id]).toBe(globalThis.testConfig.userName); + + await convertField(table1.id, userField.id, { + name: userField.name, + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }); + await processV2Outbox(); + + const refreshedFormulaField = await getField(table1.id, formulaField.id); + expect(refreshedFormulaField.isMultipleCellValue).toBe(true); + + const multipleUserRecord = await getRecord(table1.id, recordId, { + fieldKeyType: FieldKeyType.Id, + }); + expect(multipleUserRecord.data.fields[userField.id]).toEqual([ + expect.objectContaining({ title: globalThis.testConfig.userName }), + ]); + expect(multipleUserRecord.data.fields[formulaField.id]).toEqual([ + globalThis.testConfig.userName, + ]); + }); }); describe('rename', () => { diff --git a/apps/nestjs-backend/test/delete-field.e2e-spec.ts b/apps/nestjs-backend/test/delete-field.e2e-spec.ts index 47d6633fd3..da3f58b95c 100644 --- a/apps/nestjs-backend/test/delete-field.e2e-spec.ts +++ b/apps/nestjs-backend/test/delete-field.e2e-spec.ts @@ -6,9 +6,12 @@ import type { INestApplication } from '@nestjs/common'; import { FieldKeyType, FieldType } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; +import { DataPrismaService } from '@teable/db-data-prisma'; +import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; import type { ITableFullVo } from '@teable/openapi'; import { convertField } from '@teable/openapi'; +import { type ITableSchemaRepository, v2CoreTokens } from '@teable/v2-core'; +import { V2ContainerService } from '../src/features/v2/v2-container.service'; import { createField, createTable, @@ -18,15 +21,34 @@ import { permanentDeleteTable, } from './utils/init-app'; +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; +const describeV2 = isForceV2 ? describe : describe.skip; + +const quoteIdent = (identifier: string) => `"${identifier.replace(/"/g, '""')}"`; + +const parseDbTableName = (dbTableName: string) => { + const [schemaName, tableName] = dbTableName.split('.'); + + if (!schemaName || !tableName) { + throw new Error(`Invalid dbTableName: ${dbTableName}`); + } + + return { schemaName, tableName }; +}; + describe('OpenAPI delete field (e2e)', () => { let app: INestApplication; const baseId = globalThis.testConfig.baseId; let prisma: PrismaService; + let dataPrisma: DataPrismaService; + let v2ContainerService: V2ContainerService; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; prisma = app.get(PrismaService); + dataPrisma = app.get(DataPrismaService); + v2ContainerService = app.get(V2ContainerService); }); afterAll(async () => { @@ -124,6 +146,87 @@ describe('OpenAPI delete field (e2e)', () => { }); }); + describeV2('V2 delete field schema transition', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Delete Field Pending Schema Test Table', + fields: [ + { + name: 'Primary Field', + type: FieldType.SingleLineText, + }, + { + name: 'Column To Delete', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + 'Primary Field': 'Record 1', + 'Column To Delete': 'value 1', + }, + }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('should hide the table from reads before dropping a physical column', async () => { + const field = table.fields.find((f) => f.name === 'Column To Delete')!; + const dbFieldName = field.dbFieldName!; + const container = await v2ContainerService.getContainerForTable(table.id); + const tableSchemaRepository = container.resolve( + v2CoreTokens.tableSchemaRepository + ); + const originalUpdate = tableSchemaRepository.update.bind(tableSchemaRepository); + const { schemaName, tableName } = parseDbTableName(table.dbTableName); + let observedProvisionState: ProvisionState | undefined; + let readDuringSchemaUpdateError: unknown; + + tableSchemaRepository.update = (async (...args: Parameters) => { + await dataPrisma.$executeRawUnsafe( + `ALTER TABLE ${quoteIdent(schemaName)}.${quoteIdent(tableName)} DROP COLUMN IF EXISTS ${quoteIdent(dbFieldName)} CASCADE` + ); + const tableRaw = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { provisionState: true }, + }); + observedProvisionState = tableRaw.provisionState; + + try { + await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + } catch (error) { + readDuringSchemaUpdateError = error; + } + + return originalUpdate(...args); + }) as typeof tableSchemaRepository.update; + + try { + await deleteField(table.id, field.id); + } finally { + tableSchemaRepository.update = originalUpdate; + } + + const readErrorMessage = JSON.stringify(readDuringSchemaUpdateError); + expect(observedProvisionState).toBe(ProvisionState.pending); + expect(readDuringSchemaUpdateError).toMatchObject({ status: 404 }); + expect(readErrorMessage).not.toContain(dbFieldName); + + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records.records).toHaveLength(1); + expect(records.records[0].fields[field.id]).toBeUndefined(); + }); + }); + describe('delete field with formula dependencies', () => { let table: ITableFullVo; diff --git a/apps/nestjs-backend/test/field-converting.e2e-spec.ts b/apps/nestjs-backend/test/field-converting.e2e-spec.ts index 7916738bba..ad74444859 100644 --- a/apps/nestjs-backend/test/field-converting.e2e-spec.ts +++ b/apps/nestjs-backend/test/field-converting.e2e-spec.ts @@ -47,9 +47,12 @@ import { X_CANARY_HEADER, } from '@teable/openapi'; import type { Knex } from 'knex'; +import type { MockInstance } from 'vitest'; +import { vi } from 'vitest'; import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; import type { IDbProvider } from '../src/db-provider/db.provider.interface'; import { FieldService } from '../src/features/field/field.service'; +import { ComputedOrchestratorService } from '../src/features/record/computed/services/computed-orchestrator.service'; import { createNewUserAxios } from './utils/axios-instance/new-user'; import { getRecords, @@ -78,6 +81,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { let prisma: PrismaService; let fieldService: FieldService; let knex: Knex; + let computedOrchestrator: ComputedOrchestratorService; beforeAll(async () => { const appCtx = await initApp(); @@ -86,6 +90,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { prisma = appCtx.app.get(PrismaService); fieldService = appCtx.app.get(FieldService); knex = appCtx.app.get('CUSTOM_KNEX'); + computedOrchestrator = appCtx.app.get(ComputedOrchestratorService); }); afterAll(async () => { @@ -262,6 +267,57 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { ); }); + it('should not recompute dependent fields when only defaultValue changes', async () => { + const statusField = await createField(table1.id, { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choTodo', name: 'Todo', color: Colors.Cyan }, + { id: 'choDone', name: 'Done', color: Colors.Blue }, + ], + defaultValue: 'Todo', + }, + }); + + const formulaField = await createField(table1.id, { + name: 'Status formula', + type: FieldType.Formula, + options: { + expression: `{${statusField.id}}`, + }, + }); + + await updateRecordByApi(table1.id, table1.records[0].id, statusField.id, 'Todo'); + + const computeSpy: MockInstance = vi.spyOn( + computedOrchestrator, + 'computeCellChangesForFields' + ); + + try { + const updatedField = await convertField(table1.id, statusField.id, { + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choTodo', name: 'Todo', color: Colors.Cyan }, + { id: 'choDone', name: 'Done', color: Colors.Blue }, + ], + defaultValue: 'Done', + }, + }); + + expect((updatedField.options as ISelectFieldOptions).defaultValue).toEqual('Done'); + expect(computeSpy).not.toHaveBeenCalled(); + + const record = await getRecord(table1.id, table1.records[0].id); + expect(record.fields[statusField.id]).toEqual('Todo'); + expect(record.fields[formulaField.id]).toEqual('Todo'); + } finally { + computeSpy.mockRestore(); + } + }); + it('should modify field validation', async () => { const sourceFieldRo: IFieldRo = { name: 'TextField', diff --git a/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts b/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts index eba323db6e..875627ddd1 100644 --- a/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts +++ b/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts @@ -1,7 +1,7 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; import type { IFieldRo, IFieldVo, ILinkFieldOptionsRo, ILookupOptionsRo } from '@teable/core'; -import { FieldType as FT, Relationship, StatisticsFunc } from '@teable/core'; +import { FieldType as FT, Relationship, SortFunc, StatisticsFunc } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { format as formatSql } from 'sql-formatter'; import type { IRecordQueryBuilder } from '../src/features/record/query-builder'; @@ -183,6 +183,105 @@ describe('RecordQueryBuilder (e2e)', () => { expect(formatted).toMatch(/from\s+"BASE_TBL_ALIAS"\s+as\s+"TBL_ALIAS"/i); }); + it('limits aggregate field selections to the requested projection', async () => { + const { selectionMap } = await rqb.createRecordAggregateBuilder(dbTableName, { + tableId: table.id, + aggregationFields: [ + { + fieldId: '*', + statisticFunc: StatisticsFunc.Count, + alias: 'row_count', + }, + ], + groupBy: [{ fieldId: f1.id, order: SortFunc.Asc }], + projection: [f1.id], + }); + + expect(Array.from(selectionMap.keys())).toEqual([f1.id]); + }); + + it('builds CreatedBy/LastModifiedBy SQL from data-table snapshots without users joins', async () => { + let createdByField: IFieldVo | undefined; + let lastModifiedByField: IFieldVo | undefined; + + try { + createdByField = await createField(table.id, { name: 'RQ Created By', type: FT.CreatedBy }); + lastModifiedByField = await createField(table.id, { + name: 'RQ Last Modified By', + type: FT.LastModifiedBy, + }); + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [createdByField.id, lastModifiedByField.id], + filter: { + conjunction: 'and', + filterSet: [{ fieldId: lastModifiedByField.id, operator: 'is', value: 'usrAudit' }], + }, + sort: [{ fieldId: createdByField.id, order: SortFunc.Asc }], + }); + + qb.from({ [alias]: 'db_table' }); + const sql = qb.limit(1).toQuery(); + + expect(sql).not.toContain('users'); + expect(sql).not.toContain('public.users'); + expect(sql).toContain(`"${alias}"."${createdByField.dbFieldName}"`); + expect(sql).toContain(`"${alias}"."__created_by"`); + expect(sql).toContain(`"${alias}"."${lastModifiedByField.dbFieldName}"`); + expect(sql).toContain(`"${alias}"."__last_modified_by"`); + expect(sql).toContain('jsonb_extract_path_text'); + expect(sql).toContain("->>'title'"); + } finally { + if (lastModifiedByField) { + await deleteField(table.id, lastModifiedByField.id); + } + if (createdByField) { + await deleteField(table.id, createdByField.id); + } + } + }); + + it('builds formulas referencing audit user fields without users joins', async () => { + let createdByField: IFieldVo | undefined; + let formulaField: IFieldVo | undefined; + + try { + createdByField = await createField(table.id, { + name: 'Formula Created By', + type: FT.CreatedBy, + }); + formulaField = await createField(table.id, { + name: 'Formula Created By Name', + type: FT.Formula, + options: { + expression: `{${createdByField.id}}`, + }, + } as IFieldRo); + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [formulaField.id], + }); + + qb.from({ [alias]: 'db_table' }); + const sql = qb.limit(1).toQuery(); + + expect(sql).not.toContain('users'); + expect(sql).not.toContain('public.users'); + expect(sql).toContain(`"${alias}"."${createdByField.dbFieldName}"`); + expect(sql).toContain(`"${alias}"."__created_by"`); + expect(sql).toContain("->>'title'"); + } finally { + if (formulaField) { + await deleteField(table.id, formulaField.id); + } + if (createdByField) { + await deleteField(table.id, createdByField.id); + } + } + }); + it('qualifies system columns inside lookup CTE formulas', async () => { const foreignTable = await createTable(baseId, { name: 'rqb_lookup_src' }); const foreignFormulaRo: IFieldRo = { @@ -280,6 +379,7 @@ describe('RecordQueryBuilder (e2e)', () => { const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { tableId: table.id, projection: [conditionalRollup.id], + preferStoredLookupFields: false, }); qb.from({ [alias]: 'db_table' }); @@ -363,6 +463,7 @@ describe('RecordQueryBuilder (e2e)', () => { const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { tableId: table.id, projection: [conditionalRollup.id], + preferStoredLookupFields: false, }); qb.from({ [alias]: 'db_table' }); @@ -451,6 +552,7 @@ describe('RecordQueryBuilder (e2e)', () => { const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { tableId: table.id, projection: [conditionalRollup.id], + preferStoredLookupFields: false, }); qb.from({ [alias]: 'db_table' }); @@ -511,16 +613,52 @@ describe('RecordQueryBuilder (e2e)', () => { } as ILookupOptionsRo, } as IFieldRo); - const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + const { qb, alias, selectionMap } = await rqb.createRecordQueryBuilder(dbTableName, { tableId: table.id, projection: [conditionalLookup.id], }); qb.from({ [alias]: 'db_table' }); const sql = qb.limit(1).toQuery(); - expect(sql).toContain(`__cl_${conditionalLookup.id}`); - expect(sql).toContain('ROW_NUMBER() OVER (PARTITION BY'); - expect(sql).toContain('jsonb_extract_path_text'); + expect(sql).not.toContain(`CTE_CONDITIONAL_LOOKUP_${conditionalLookup.id}`); + expect(selectionMap.get(conditionalLookup.id)?.toString()).toContain( + `"${conditionalLookup.dbFieldName}"` + ); + + const { qb: computedQb, alias: computedAlias } = await rqb.createRecordQueryBuilder( + dbTableName, + { + tableId: table.id, + projection: [conditionalLookup.id], + preferStoredLookupFields: false, + } + ); + computedQb.from({ [computedAlias]: 'db_table' }); + + const computedSql = computedQb.limit(1).toQuery(); + expect(computedSql).toContain(`__cl_${conditionalLookup.id}`); + expect(computedSql).toContain('ROW_NUMBER() OVER (PARTITION BY'); + expect(computedSql).toContain('jsonb_extract_path_text'); + + const { qb: aggregateQb, selectionMap: aggregateSelectionMap } = + await rqb.createRecordAggregateBuilder(dbTableName, { + tableId: table.id, + aggregationFields: [ + { + fieldId: '*', + statisticFunc: StatisticsFunc.Count, + alias: 'row_count', + }, + ], + groupBy: [{ fieldId: conditionalLookup.id, order: SortFunc.Asc }], + projection: [conditionalLookup.id], + }); + + const aggregateSql = aggregateQb.toQuery(); + expect(aggregateSql).not.toContain(`CTE_CONDITIONAL_LOOKUP_${conditionalLookup.id}`); + expect(aggregateSelectionMap.get(conditionalLookup.id)?.toString()).toContain( + `"${conditionalLookup.dbFieldName}"` + ); } finally { if (conditionalLookup) { await deleteField(table.id, conditionalLookup.id); diff --git a/apps/nestjs-backend/test/search-consistency.e2e-spec.ts b/apps/nestjs-backend/test/search-consistency.e2e-spec.ts new file mode 100644 index 0000000000..1ace971837 --- /dev/null +++ b/apps/nestjs-backend/test/search-consistency.e2e-spec.ts @@ -0,0 +1,276 @@ +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import { + Colors, + computeSearchHitIndex, + DateFormattingPreset, + FieldKeyType, + FieldType, + NumberFormattingType, + RatingIcon, + Relationship, + TimeFormatting, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createRecords, + getRecords as apiGetRecords, + getSearchIndex, + uploadAttachment, +} from '@teable/openapi'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { createFieldInstanceByVo } from '../src/features/field/model/factory'; +import { createTable, permanentDeleteTable, initApp, getFields } from './utils/init-app'; + +/** + * Reconciliation suite: for the same data set, server-side search hits + * (aggregation/search-index, the same SQL semantics that filter rows) must + * agree with client-side hits (computeSearchHitIndex from @teable/core, the + * same code that highlights cells). Field eligibility is structurally shared + * via FieldCore.isSearchable; this locks the value-level matching semantics. + */ +describe('Search consistency between server and client (e2e)', () => { + let app: INestApplication; + let table: ITableFullVo; + let subTable: ITableFullVo; + let viewId: string; + const baseId = globalThis.testConfig.baseId; + + const fieldId = (name: string) => { + const field = table.fields.find((f) => f.name === name); + if (!field) throw new Error(`field ${name} not found`); + return field.id; + }; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + subTable = await createTable(baseId, { + name: 'search-consistency-sub', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Sub Record A' } }, { fields: { Name: 'Sub Record B' } }], + fieldKeyType: FieldKeyType.Name, + }); + + table = await createTable(baseId, { + name: 'search-consistency-main', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'Amount', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'Stars', + type: FieldType.Rating, + options: { icon: RatingIcon.Star, color: Colors.YellowBright, max: 5 }, + }, + { + name: 'Due', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Singapore', + }, + }, + }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Open', color: Colors.Cyan }, + { name: 'Closed', color: Colors.Gray }, + ], + }, + }, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'alpha', color: Colors.Cyan }, + { name: 'beta', color: Colors.Blue }, + { name: 'gamma', color: Colors.Gray }, + ], + }, + }, + { name: 'Done', type: FieldType.Checkbox }, + { name: 'Files', type: FieldType.Attachment }, + ], + records: [], + }); + viewId = table.views[0].id; + + const titleId = fieldId('Title'); + const amountId = fieldId('Amount'); + const dueId = fieldId('Due'); + + await createField(table.id, { + name: 'Link', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: subTable.id, isOneWay: true }, + }); + await createField(table.id, { + name: 'TextFormula', + type: FieldType.Formula, + options: { expression: `{${titleId}}` }, + }); + await createField(table.id, { + name: 'NumberFormula', + type: FieldType.Formula, + options: { expression: `{${amountId}} * 2` }, + }); + await createField(table.id, { + name: 'DateFormula', + type: FieldType.Formula, + options: { expression: `{${dueId}}` }, + }); + await createField(table.id, { + name: 'BoolFormula', + type: FieldType.Formula, + options: { expression: `{${amountId}} > 1` }, + }); + table.fields = await getFields(table.id); + + const { records: createdRecords } = ( + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [fieldId('Title')]: 'Apple Pie', + [fieldId('Amount')]: 1.5, + [fieldId('Stars')]: 3, + [fieldId('Due')]: '2026-01-15T00:00:00.000Z', + [fieldId('Status')]: 'Open', + [fieldId('Tags')]: ['alpha', 'beta'], + [fieldId('Done')]: true, + [fieldId('Link')]: [{ id: subTable.records[0].id }], + }, + }, + { + fields: { + [fieldId('Title')]: 'banana split', + [fieldId('Amount')]: 100, + [fieldId('Tags')]: ['gamma'], + }, + }, + { + fields: { + [fieldId('Title')]: '50% off_sale', + [fieldId('Stars')]: 5, + }, + }, + { fields: { [fieldId('Title')]: 'TRUE Story' } }, + { fields: {} }, + ], + }) + ).data; + + const tmpPath = path.resolve( + path.join(StorageAdapter.TEMPORARY_DIR, 'search-consistency-invoice-report.txt') + ); + fs.writeFileSync(tmpPath, 'attachment content'); + try { + await uploadAttachment( + table.id, + createdRecords[0].id, + fieldId('Files'), + fs.createReadStream(tmpPath) + ); + } finally { + fs.unlinkSync(tmpPath); + } + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + await app.close(); + }); + + const toKey = (hit: { recordId: string; fieldId: string }) => `${hit.recordId}:${hit.fieldId}`; + + const fetchServerHits = async (search: [string, string, boolean]) => { + const { data } = await getSearchIndex(table.id, { viewId, take: 1000, search }); + // an empty hit response has no body, so data may be '' instead of null + return new Set(Array.isArray(data) ? data.map(toKey) : []); + }; + + const computeClientHits = async (search: [string, string, boolean]) => { + const { records } = ( + await apiGetRecords(table.id, { viewId, fieldKeyType: FieldKeyType.Id, take: 1000 }) + ).data; + const fields = table.fields.map(createFieldInstanceByVo); + const hits = computeSearchHitIndex( + records.map((r) => ({ id: r.id, fields: r.fields })), + fields, + search + ); + return new Set((hits ?? []).map(toKey)); + }; + + type ICase = { name: string; keyword: string; scope?: () => string }; + const cases: ICase[] = [ + { name: 'plain text, all fields', keyword: 'apple' }, + { name: 'case-insensitive text, all fields', keyword: 'APPLE' }, + { name: 'numeric keyword, all fields', keyword: '1.5' }, + { name: 'numeric fragment, all fields', keyword: '.5' }, + { name: 'integer keyword, all fields', keyword: '100' }, + { name: 'date-like keyword, all fields (datetime excluded)', keyword: '2026' }, + { name: 'like wildcard percent, all fields', keyword: '%' }, + { name: 'like wildcard underscore, all fields', keyword: '_' }, + { name: 'multi select option, all fields', keyword: 'alpha' }, + { name: 'single select lowercase, all fields', keyword: 'open' }, + { name: 'boolean-like keyword, all fields (boolean excluded)', keyword: 'true' }, + { name: 'multi-value join boundary, all fields', keyword: 'a, b' }, + { name: 'link title, all fields', keyword: 'sub record' }, + { name: 'date scoped to date field', keyword: '2026', scope: () => fieldId('Due') }, + { name: 'text scoped to text field', keyword: 'apple', scope: () => fieldId('Title') }, + { + name: 'numeric scoped to several fields', + keyword: '1.5', + scope: () => `${fieldId('Amount')},${fieldId('Stars')}`, + }, + { name: 'link scoped to link field', keyword: 'sub record', scope: () => fieldId('Link') }, + ]; + + it.each(cases)('agrees on $name', async ({ keyword, scope }) => { + const search: [string, string, boolean] = [keyword, scope?.() ?? '', false]; + const serverHits = await fetchServerHits(search); + const clientHits = await computeClientHits(search); + expect([...clientHits].sort()).toEqual([...serverHits].sort()); + }); + // attachments are excluded from search on both sides (AttachmentFieldCore + // isSearchable returns false; the server consumes the same predicate): + // server-side matching could only run over the stored JSON text, which + // produced noise hits for mimetype/path keywords like "png" or "pdf" + it('never matches attachment fields on either side', async () => { + const { records } = ( + await apiGetRecords(table.id, { viewId, fieldKeyType: FieldKeyType.Id, take: 1000 }) + ).data; + const attach = (records[0].fields[fieldId('Files')] as { name: string; token: string }[])[0]; + + const keywordsByScope: [string, string][] = [ + ['invoice', ''], + ['invoice', fieldId('Files')], + [attach.token, ''], + ['text/plain', ''], + ['mimetype', ''], + ]; + for (const [keyword, scope] of keywordsByScope) { + const search: [string, string, boolean] = [keyword, scope, false]; + const serverHits = await fetchServerHits(search); + const clientHits = await computeClientHits(search); + expect(serverHits.size).toBe(0); + expect(clientHits.size).toBe(0); + } + }); +}); diff --git a/apps/nestjs-backend/test/selection.e2e-spec.ts b/apps/nestjs-backend/test/selection.e2e-spec.ts index 78cc74e21a..13f25a92e8 100644 --- a/apps/nestjs-backend/test/selection.e2e-spec.ts +++ b/apps/nestjs-backend/test/selection.e2e-spec.ts @@ -6,6 +6,7 @@ import { Colors, FieldKeyType, FieldType, + Me, MultiNumberDisplayType, Relationship, Role, @@ -33,7 +34,9 @@ import { getFields, deleteSelection, clear, + updateRecordOrders, updateViewFilter, + updateViewGroup, updateViewSort, USER_ME, UPDATE_USER_NAME, @@ -52,6 +55,7 @@ import { initApp, createTable, createRecords, + convertField, permanentDeleteTable, permanentDeleteSpace, updateRecordByApi, @@ -2304,35 +2308,37 @@ describe('OpenAPI SelectionController (e2e)', () => { const recordOpenApiV2Service = app.get(RecordOpenApiV2Service); const deleteByRangeStreamSpy = vi .spyOn(recordOpenApiV2Service, 'deleteByRangeStream') - .mockImplementation(async function* () { - yield { - id: 'progress', - phase: 'deleting', - batchIndex: 0, - totalCount: 3, - deletedCount: 1, - batchDeletedCount: 1, - }; - yield { - id: 'error', - phase: 'deleting', - batchIndex: 1, - totalCount: 3, - deletedCount: 1, - recordIds: [streamTable.records[1]!.id], - message: 'chunk 2 failed', - code: 'unexpected', - }; - yield { - id: 'done', - totalCount: 3, - deletedCount: 2, - data: { + .mockImplementation(async () => + (async function* () { + yield { + id: 'progress', + phase: 'deleting', + batchIndex: 0, + totalCount: 3, + deletedCount: 1, + batchDeletedCount: 1, + }; + yield { + id: 'error', + phase: 'deleting', + batchIndex: 1, + totalCount: 3, + deletedCount: 1, + recordIds: [streamTable.records[1]!.id], + message: 'chunk 2 failed', + code: 'unexpected', + }; + yield { + id: 'done', + totalCount: 3, deletedCount: 2, - deletedRecordIds: [streamTable.records[0]!.id, streamTable.records[2]!.id], - }, - }; - }); + data: { + deletedCount: 2, + deletedRecordIds: [streamTable.records[0]!.id, streamTable.records[2]!.id], + }, + }; + })() + ); try { const { progressEvents, doneEvent, errorEvents } = await deleteStreamWithCanary( @@ -3188,6 +3194,98 @@ describe('OpenAPI SelectionController (e2e)', () => { }); }); + describe('paste with sort ties and manual row order (personal view)', () => { + /** + * Repro for paste row misalignment under personal views: + * the grid displays rows via the v1 read path, which always breaks sort + * ties by the view's manual row order (__row_{viewId}), while v2 range + * commands broke ties by __auto_number. With duplicate sort values and a + * manually reordered row, paste targeted a different record than the one + * displayed at the same offset. + */ + let tieTable: ITableFullVo; + + beforeEach(async () => { + // Creation order (auto number): A, B, C tie in group 1; D, E in group 2 + tieTable = await createTable(baseId, { + name: 'tie-paste-table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Group', type: FieldType.Number }, + ], + records: [ + { fields: { Name: 'RecordA', Group: 1 } }, + { fields: { Name: 'RecordB', Group: 1 } }, + { fields: { Name: 'RecordC', Group: 1 } }, + { fields: { Name: 'RecordD', Group: 2 } }, + { fields: { Name: 'RecordE', Group: 2 } }, + ], + }); + // Make the manual row order differ from creation order within the tied + // group: move C before A -> view row order: C, A, B, D, E + await updateRecordOrders(tieTable.id, tieTable.views[0].id, { + anchorId: tieTable.records[0].id, + position: 'before', + recordIds: [tieTable.records[2].id], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tieTable.id); + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true }] + : [ + { label: 'v1', useV2: false }, + { label: 'v2', useV2: true }, + ] + )('should paste into the displayed row when the sort has ties ($label)', async ({ useV2 }) => { + const nameField = tieTable.fields.find((f) => f.name === 'Name')!; + const groupField = tieTable.fields.find((f) => f.name === 'Group')!; + // Mimic a personal view: the client sends its own sort with + // ignoreViewQuery instead of relying on the saved view config. + const personalViewQuery = { + viewId: tieTable.views[0].id, + ignoreViewQuery: true, + orderBy: [{ fieldId: groupField.id, order: SortFunc.Asc }], + }; + + // Ground truth: the row order the grid displays (v1 read path breaks + // the Group=1 tie by manual row order, so RecordC is the first row). + const displayed = await getRecordsWithCanary( + tieTable.id, + { ...personalViewQuery, fieldKeyType: FieldKeyType.Id }, + false + ); + expect(displayed.data.records[0].fields[nameField.id]).toBe('RecordC'); + + // Paste a single cell into the first displayed row (Name column). + await pasteWithCanary( + tieTable.id, + { + ...personalViewQuery, + content: 'PastedTop', + ranges: [ + [0, 0], + [0, 0], + ], + }, + useV2 + ); + + const allRecords = await getRecords(tieTable.id, { fieldKeyType: FieldKeyType.Id }); + const recordC = allRecords.data.records.find((r) => r.id === tieTable.records[2].id); + const recordA = allRecords.data.records.find((r) => r.id === tieTable.records[0].id); + + // The displayed first row (RecordC) must receive the pasted value; + // RecordA (first by auto number within the tie) must stay unchanged. + expect(recordC?.fields[nameField.id]).toBe('PastedTop'); + expect(recordA?.fields[nameField.id]).toBe('RecordA'); + }); + }); + describe('paste with view-level sort and filter (no client orderBy)', () => { /** * Regression test: when the view has a saved sort/filter but the client @@ -3872,5 +3970,125 @@ describe('OpenAPI SelectionController (e2e)', () => { ).toBe(false); } ); + + it('T4992 should paste grouped tail rows in v2 when the view filter references Me', async () => { + const assigneeValue = { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }; + const requiredTable = await createTable(baseId, { + name: 'T4992 grouped paste required field', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Need more information', color: Colors.Red }, + { name: 'Added to backlog', color: Colors.Teal }, + { name: 'Entered development workflow', color: Colors.Purple }, + { name: 'Launched', color: Colors.Green }, + { name: 'Closed as completed', color: Colors.Green }, + { name: 'Closed as not planned', color: Colors.Red }, + ], + }, + }, + { name: 'Assignee', type: FieldType.User }, + { name: 'Email', type: FieldType.SingleLineText }, + ], + records: [ + ...Array.from({ length: 14 }, (_, index) => ({ + fields: { + Title: `Blank-${index + 1}`, + Assignee: assigneeValue, + Email: `blank-${index + 1}@example.com`, + }, + })), + ...Array.from({ length: 3 }, (_, index) => ({ + fields: { + Title: `Need-${index + 1}`, + Status: 'Need more information', + Assignee: assigneeValue, + Email: `need-${index + 1}@example.com`, + }, + })), + ...Array.from({ length: 7 }, (_, index) => ({ + fields: { + Title: `Backlog-${index + 1}`, + Status: 'Added to backlog', + Assignee: assigneeValue, + Email: `backlog-${index + 1}@example.com`, + }, + })), + ...Array.from({ length: 5 }, (_, index) => ({ + fields: { + Title: `Workflow-${index + 1}`, + Status: 'Entered development workflow', + Assignee: assigneeValue, + Email: `workflow-${index + 1}@example.com`, + }, + })), + ], + }); + + try { + const viewId = requiredTable.views[0].id; + const statusField = requiredTable.fields.find((field) => field.name === 'Status')!; + const assigneeField = requiredTable.fields.find((field) => field.name === 'Assignee')!; + const emailField = requiredTable.fields.find((field) => field.name === 'Email')!; + await convertField(requiredTable.id, emailField.id, { ...emailField, notNull: true }); + await updateViewFilter(requiredTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { fieldId: assigneeField.id, operator: 'is', value: Me }, + { + fieldId: statusField.id, + operator: 'isNoneOf', + value: ['Closed as not planned', 'Closed as completed', 'Launched'], + }, + ], + }, + }); + const groupBy = [{ fieldId: statusField.id, order: SortFunc.Asc }] as const; + await updateViewGroup(requiredTable.id, viewId, { group: [...groupBy] }); + + const pasteRes = await pasteWithCanary( + requiredTable.id, + { + viewId, + content: [['Launched']], + ranges: [ + [1, 24], + [1, 28], + ], + header: [statusField], + groupBy: [...groupBy], + }, + true + ); + + expect(pasteRes.status).toBe(200); + expect(pasteRes.headers['x-teable-v2']).toBe('true'); + + const allRecords = await getRecords(requiredTable.id, { + fieldKeyType: FieldKeyType.Id, + take: 100, + }); + + expect(allRecords.data.records).toHaveLength(29); + const workflowRecords = allRecords.data.records.filter((record) => + String(record.fields[requiredTable.fields[0].id] ?? '').startsWith('Workflow-') + ); + expect(workflowRecords).toHaveLength(5); + expect( + workflowRecords.every((record) => record.fields[statusField.id] === 'Launched') + ).toBe(true); + } finally { + await permanentDeleteTable(baseId, requiredTable.id); + } + }); }); }); diff --git a/apps/nestjs-backend/test/share-db-poll-debounce.e2e-spec.ts b/apps/nestjs-backend/test/share-db-poll-debounce.e2e-spec.ts new file mode 100644 index 0000000000..e2ab913710 --- /dev/null +++ b/apps/nestjs-backend/test/share-db-poll-debounce.e2e-spec.ts @@ -0,0 +1,111 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, IdPrefix, ViewType } from '@teable/core'; +import { createRecords as apiCreateRecords } from '@teable/openapi'; +import type { Connection, Query } from 'sharedb/lib/client'; +import { vi } from 'vitest'; +import { ShareDbAdapter } from '../src/share-db/share-db.adapter'; +import { ShareDbService } from '../src/share-db/share-db.service'; +import { initApp, createTable, permanentDeleteTable } from './utils/init-app'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const waitForQueryReady = (query: Query, timeout = 5000): Promise => { + return new Promise((resolve, reject) => { + if (query.ready) { + resolve(); + return; + } + const timer = setTimeout(() => reject(new Error('Query ready timeout')), timeout); + query.once('ready', () => { + clearTimeout(timer); + resolve(); + }); + }); +}; + +describe('ShareDB query pollDebounce (e2e)', () => { + let app: INestApplication; + let tableId: string; + let cookie: string; + let port: string; + let shareDbService: ShareDbService; + let adapter: ShareDbAdapter; + let originalPollDebounce: number; + const baseId = globalThis.testConfig.baseId; + + const burstSize = 8; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + cookie = appCtx.cookie; + port = process.env.PORT!; + shareDbService = app.get(ShareDbService); + adapter = app.get(ShareDbAdapter); + originalPollDebounce = adapter.pollDebounce; + + const table = await createTable(baseId, { + name: 'poll-debounce-test-table', + views: [{ type: ViewType.Grid, name: 'default-view' }], + }); + tableId = table.id; + }); + + afterAll(async () => { + adapter.pollDebounce = originalPollDebounce; + await permanentDeleteTable(baseId, tableId); + await app.close(); + }); + + it('exposes pollDebounce on the backend db read by QueryEmitter', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((shareDbService as any).db).toBe(adapter); + expect(typeof adapter.pollDebounce).toBe('number'); + expect(adapter.pollDebounce).toBeGreaterThan(0); + }); + + // Subscribes a fresh query (QueryEmitter snapshots pollDebounce at subscribe + // time), fires burstSize sequential record creations, then counts how many + // times the adapter polled this collection. + const countPollsForBurst = async (pollDebounceMs: number): Promise => { + adapter.pollDebounce = pollDebounceMs; + const connection: Connection = shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: { cookie }, + }); + const collection = `${IdPrefix.Record}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + try { + await waitForQueryReady(query); + const spy = vi.spyOn(adapter, 'queryPoll'); + + for (let i = 0; i < burstSize; i++) { + await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], + }); + } + // Wait past the debounce window so the trailing poll lands + await sleep(pollDebounceMs * 1.5 + 1500); + + const polls = spy.mock.calls.filter(([c]) => c === collection).length; + spy.mockRestore(); + return polls; + } finally { + query.destroy(); + connection.close(); + } + }; + + it('coalesces an op burst into fewer polls when pollDebounce is on', async () => { + const pollsWithoutDebounce = await countPollsForBurst(0); + const pollsWithDebounce = await countPollsForBurst(3000); + + // Without debounce each create op polls (minus natural in-flight coalescing) + expect(pollsWithoutDebounce).toBeGreaterThanOrEqual(4); + // With debounce: one leading poll + coalesced trailing polls + expect(pollsWithDebounce).toBeLessThanOrEqual(3); + expect(pollsWithDebounce).toBeLessThan(pollsWithoutDebounce); + }, 60000); +}); diff --git a/apps/nestjs-backend/test/share.e2e-spec.ts b/apps/nestjs-backend/test/share.e2e-spec.ts index 498e52f72b..5b0f620721 100644 --- a/apps/nestjs-backend/test/share.e2e-spec.ts +++ b/apps/nestjs-backend/test/share.e2e-spec.ts @@ -66,6 +66,7 @@ import { getField, deleteField, convertField, + updateRecordByApi, permanentDeleteBase, } from './utils/init-app'; @@ -90,7 +91,6 @@ describe('OpenAPI ShareController (e2e)', () => { const spaceId = globalThis.testConfig.spaceId; const userId = globalThis.testConfig.userId; const userName = globalThis.testConfig.userName; - const userEmail = globalThis.testConfig.email; let fieldIds: string[] = []; let anonymousUser: ReturnType; @@ -908,12 +908,10 @@ describe('OpenAPI ShareController (e2e)', () => { const mulResult = await apiGetShareViewCollaborators(gridViewShareId, { fieldId: multipleUserFieldId, }); - expect(result.data).toEqual([ - { userId, userName, email: userEmail, avatar: expect.any(String) }, - ]); - expect(mulResult.data).toEqual([ - { userId, userName, email: userEmail, avatar: expect.any(String) }, - ]); + // Email is intentionally omitted from share responses to avoid leaking + // the member directory to anonymous viewers. + expect(result.data).toEqual([{ userId, userName, avatar: expect.any(String) }]); + expect(mulResult.data).toEqual([{ userId, userName, avatar: expect.any(String) }]); await apiDeleteRecords( userTableRes.id, @@ -967,6 +965,24 @@ describe('OpenAPI ShareController (e2e)', () => { }, ]); }); + it('should search collaborators by name but not by email', async () => { + await apiUpdateViewColumnMeta(userTableRes.id, formViewId, [ + { fieldId: userFieldId, columnMeta: { visible: true } }, + ]); + const byName = await apiGetShareViewCollaborators(fromViewShareId, { + search: userName, + }); + expect(byName.data.map((user) => user.userId)).toContain(userId); + // Email is neither returned nor searchable for anonymous shares, so a + // full email must not surface a collaborator (membership oracle closed). + const byEmail = await apiGetShareViewCollaborators(fromViewShareId, { + search: globalThis.testConfig.email, + }); + expect(byEmail.data).toEqual([]); + await apiUpdateViewColumnMeta(userTableRes.id, formViewId, [ + { fieldId: userFieldId, columnMeta: { visible: false } }, + ]); + }); }); }); @@ -1127,6 +1143,78 @@ describe('OpenAPI ShareController (e2e)', () => { }); expect(unmatched.data.records).toHaveLength(0); }); + + // T4864: a record linked before (or outside of) the link field's filterByViewId + // scope must still appear in the selected list / detail panel. The view scope only + // limits which records can be newly linked (candidate list), not the existing links. + it('selected list ignores the link field filterByViewId scope', async () => { + const primary = table2.fields[0]; + + const foreignRecords = await apiCreateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { fields: { [primary.id]: 'in view' } }, + { fields: { [primary.id]: 'out of view' } }, + ], + }); + const inViewId = foreignRecords.data.records[0].id; + const outOfViewId = foreignRecords.data.records[1].id; + + // scope the foreign default view so only the "in view" record is visible + await updateViewFilter(table2.id, table2.defaultViewId!, { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: primary.id, operator: is.value, value: 'in view' }], + }, + }); + + const linkField = await createField(table1.id, { + name: 'scoped link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + filterByViewId: table2.defaultViewId, + }, + }); + + const hostRecords = await apiCreateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], + }); + const hostId = hostRecords.data.records[0].id; + + // link the host record to the record that is NOT in the configured view + await updateRecordByApi(table1.id, hostId, linkField.data.id, { + id: outOfViewId, + }); + + const selectedQuery = { + filterLinkCellSelected: [linkField.data.id, hostId] as [string, string], + }; + + // the already-linked record must be visible even though it fails the view filter + const selected = await apiGetShareViewRecords(linkField.data.id, selectedQuery); + expect(selected.data.records.map((r) => r.id)).toEqual([outOfViewId]); + + const selectedRowCount = await getShareViewRowCount(linkField.data.id, selectedQuery); + expect(selectedRowCount.data.rowCount).toEqual(1); + + // the expand-record card loads already-linked records by selectedRecordIds and + // must also bypass the view scope + const byIds = await apiGetShareViewRecords(linkField.data.id, { + selectedRecordIds: [outOfViewId], + }); + expect(byIds.data.records.map((r) => r.id)).toEqual([outOfViewId]); + + // the candidate list must still respect the view filter + const candidate = await apiGetShareViewRecords(linkField.data.id, { + filterLinkCellCandidate: [linkField.data.id, hostId], + }); + const candidateIds = candidate.data.records.map((r) => r.id); + expect(candidateIds).toContain(inViewId); + expect(candidateIds).not.toContain(outOfViewId); + }); }); describe('link view limit', () => { diff --git a/apps/nestjs-backend/test/sort.e2e-spec.ts b/apps/nestjs-backend/test/sort.e2e-spec.ts index b0754dce78..80f83981db 100644 --- a/apps/nestjs-backend/test/sort.e2e-spec.ts +++ b/apps/nestjs-backend/test/sort.e2e-spec.ts @@ -170,6 +170,25 @@ describe('OpenAPI ViewController view order sort (e2e)', () => { expect(viewSort).toEqual(assertSort.sort); }); + it('/api/table/{tableId}/view/{viewId}/sort clears view sort with null (PUT)', async () => { + await apiSetViewSort(tableId, viewId, { + sort: { + sortObjs: [ + { + fieldId: fields[0].id as string, + order: SortFunc.Asc, + }, + ], + manualSort: false, + }, + }); + + await apiSetViewSort(tableId, viewId, { sort: null }); + + const updatedView = await getView(tableId, viewId); + expect(updatedView.sort).toBeUndefined(); + }); + it('sort date should always use a second precision when formatting time is not none', async () => { await createRecords(tableId, { records: [ diff --git a/apps/nestjs-backend/test/table-import.e2e-spec.ts b/apps/nestjs-backend/test/table-import.e2e-spec.ts index 44e2e1ff77..234adb4a38 100644 --- a/apps/nestjs-backend/test/table-import.e2e-spec.ts +++ b/apps/nestjs-backend/test/table-import.e2e-spec.ts @@ -31,6 +31,8 @@ import { initApp, permanentDeleteTable, getTable as apiGetTableById } from './ut extend(timezone); +const importTimeZone = 'Asia/Shanghai'; + enum TestFileFormat { 'CSV' = 'csv', 'TSV' = 'tsv', @@ -39,6 +41,7 @@ enum TestFileFormat { } const defaultTestSheetKey = 'Sheet1'; +const xTeableV2Header = 'x-teable-v2'; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const testSupportTypeMap = { @@ -272,7 +275,7 @@ describe('OpenAPI ImportController (e2e)', () => { importData: true, }, }, - tz: 'Asia/Shanghai', + tz: importTimeZone, }); const { fields, id } = table.data[0]; @@ -282,9 +285,11 @@ describe('OpenAPI ImportController (e2e)', () => { name: field.name, })); - await awaitWithEvent(async () => { - noop(); - }); + if (table.headers[xTeableV2Header] !== 'true') { + await awaitWithEvent(async () => { + noop(); + }); + } const { records } = await apiGetTableById(baseId, table.data[0].id, { includeContent: true, @@ -297,13 +302,80 @@ describe('OpenAPI ImportController (e2e)', () => { } ); + it('should route CSV new-table import through V2 when V2 is forced', async () => { + const previousForceV2All = process.env.FORCE_V2_ALL; + process.env.FORCE_V2_ALL = 'true'; + + try { + const spaceRes = await apiCreateSpace({ name: 'v2-import-csv' }); + const spaceId = spaceRes?.data?.id; + const baseRes = await apiCreateBase({ spaceId }); + const baseId = baseRes.data.id; + + const format = TestFileFormat.CSV; + const fileType = testSupportTypeMap[format].fileType; + const attachmentUrl = testFiles[format].url; + const sheetKey = testSupportTypeMap[format].defaultSheetKey; + + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl, + fileType, + }); + const columns = worksheets[sheetKey].columns.map((column, index) => ({ + ...column, + sourceColumnIndex: index, + })); + + const importRes = await apiImportTableFromFile(baseId, { + attachmentUrl, + fileType, + worksheets: { + [sheetKey]: { + name: sheetKey, + columns, + useFirstRowAsHeader: true, + importData: true, + }, + }, + tz: importTimeZone, + }); + + expect(importRes.headers[xTeableV2Header]).toBe('true'); + expect(importRes.headers['x-teable-v2-feature']).toBe('importCsv'); + expect(['env_force_v2_all', 'new_base']).toContain(importRes.headers['x-teable-v2-reason']); + + const { fields, id } = importRes.data[0]; + const createdFields = fields.map((field) => ({ + type: field.type, + name: field.name, + })); + + const { records } = await apiGetTableById(baseId, id, { + includeContent: true, + }); + + bases.push([baseId, id]); + + expect(records?.length).toBe(2); + expect(createdFields).toEqual(assertHeaders); + } finally { + if (previousForceV2All === undefined) { + delete process.env.FORCE_V2_ALL; + } else { + process.env.FORCE_V2_ALL = previousForceV2All; + } + } + }); + it('should query import status until completed for imported table', async () => { const spaceRes = await apiCreateSpace({ name: 'status-check' }); const spaceId = spaceRes?.data?.id; const baseRes = await apiCreateBase({ spaceId }); const baseId = baseRes.data.id; - const format = TestFileFormat.CSV; + const format = TestFileFormat.XLSX; const fileType = testSupportTypeMap[format].fileType; const attachmentUrl = testFiles[format].url; const sheetKey = testSupportTypeMap[format].defaultSheetKey; @@ -330,7 +402,7 @@ describe('OpenAPI ImportController (e2e)', () => { importData: true, }, }, - tz: 'Asia/Shanghai', + tz: importTimeZone, }); const tableId = importRes.data[0].id; @@ -422,17 +494,20 @@ describe('OpenAPI ImportController (e2e)', () => { }); // import data into table - await awaitWithEvent(async () => { - await apiInplaceImportTableFromFile(baseId, tableId, { - attachmentUrl, - fileType, - insertConfig: { - sourceWorkSheetKey: CsvImporter.DEFAULT_SHEETKEY, - excludeFirstRow: true, - sourceColumnMap, - }, - }); + const importRes = await apiInplaceImportTableFromFile(baseId, tableId, { + attachmentUrl, + fileType, + insertConfig: { + sourceWorkSheetKey: CsvImporter.DEFAULT_SHEETKEY, + excludeFirstRow: true, + sourceColumnMap, + }, }); + if (importRes.headers[xTeableV2Header] !== 'true') { + await awaitWithEvent(async () => { + noop(); + }); + } const { records } = await apiGetTableById(baseId, tableId, { includeContent: true, diff --git a/apps/nestjs-backend/test/utils/action-trigger.ts b/apps/nestjs-backend/test/utils/action-trigger.ts new file mode 100644 index 0000000000..df6b26d761 --- /dev/null +++ b/apps/nestjs-backend/test/utils/action-trigger.ts @@ -0,0 +1,123 @@ +import { getActionTriggerChannel } from '@teable/core'; +import type { Connection } from 'sharedb/lib/client'; +import type { ShareDbService } from '../../src/share-db/share-db.service'; + +export interface IActionTrigger { + actionKey: string; + payload?: Record; +} + +const createConnection = ( + shareDbService: ShareDbService, + cookie: string, + port: string +): Connection => { + return shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: { cookie }, + }); +}; + +/** + * Subscribe to a table's action-trigger presence channel, run `act`, and + * resolve with every action received until the `until` predicate matches or + * the channel stays idle for `idleMs` after `act` completed. + */ +export const collectActionTriggers = async (params: { + shareDbService: ShareDbService; + cookie: string; + port: string; + tableId: string; + act: () => Promise; + idleMs?: number; + timeoutMs?: number; + until?: (actions: ReadonlyArray) => boolean; +}): Promise => { + const { + shareDbService, + cookie, + port, + tableId, + act, + idleMs = 300, + timeoutMs = 5000, + until, + } = params; + + return new Promise((resolve, reject) => { + const connection = createConnection(shareDbService, cookie, port); + const presence = connection.getPresence(getActionTriggerChannel(tableId)); + const received: IActionTrigger[] = []; + let capture = false; + let settled = false; + let actCompleted = false; + let idleTimer: NodeJS.Timeout | undefined; + + const cleanup = () => { + clearTimeout(timeout); + if (idleTimer) clearTimeout(idleTimer); + presence.removeListener('receive', onReceive); + try { + presence.unsubscribe(); + presence.destroy(); + } catch { + void 0; + } + connection.close(); + }; + + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + cleanup(); + if (error) { + reject(error instanceof Error ? error : new Error(String(error))); + return; + } + resolve(received); + }; + + const onReceive = (_id: string, batch: IActionTrigger[]) => { + if (!capture) { + return; + } + received.push(...batch); + if (until?.(received)) { + finish(); + return; + } + if (!actCompleted) { + return; + } + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => finish(), idleMs); + }; + + const timeout = setTimeout(() => { + finish(new Error('Action trigger timeout')); + }, timeoutMs); + + presence.subscribe(async (error: unknown) => { + if (error) { + finish(error); + return; + } + + presence.on('receive', onReceive); + + try { + capture = true; + await act(); + actCompleted = true; + if (until?.(received)) { + finish(); + return; + } + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => finish(), idleMs); + } catch (actError) { + finish(actError); + } + }); + }); +}; diff --git a/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts b/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts index 9a1913c815..d69e9543b1 100644 --- a/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts +++ b/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { INestApplication } from '@nestjs/common'; -import { FieldKeyType, FieldType, getActionTriggerChannel } from '@teable/core'; +import { FieldKeyType, FieldType } from '@teable/core'; import { axios, X_CANARY_HEADER } from '@teable/openapi'; -import type { Connection } from 'sharedb/lib/client'; import { ShareDbService } from '../src/share-db/share-db.service'; +import { collectActionTriggers } from './utils/action-trigger'; import { createField, createRecords, @@ -12,12 +12,8 @@ import { permanentDeleteTable, } from './utils/init-app'; -interface IActionTrigger { - actionKey: string; - payload?: Record; -} - const amountTextFieldName = 'Amount Text'; +const v2ResponseHeader = 'x-teable-v2'; let fieldIdCounter = 0; @@ -27,116 +23,6 @@ const createFieldId = () => { return `fld${suffix}`; }; -const createConnection = ( - shareDbService: ShareDbService, - cookie: string, - port: string -): Connection => { - return shareDbService.connect(undefined, { - url: `ws://localhost:${port}/socket`, - headers: { cookie }, - }); -}; - -const collectActionTriggers = async (params: { - shareDbService: ShareDbService; - cookie: string; - port: string; - tableId: string; - act: () => Promise; - idleMs?: number; - timeoutMs?: number; - until?: (actions: ReadonlyArray) => boolean; -}): Promise => { - const { - shareDbService, - cookie, - port, - tableId, - act, - idleMs = 300, - timeoutMs = 5000, - until, - } = params; - - return new Promise((resolve, reject) => { - const connection = createConnection(shareDbService, cookie, port); - const presence = connection.getPresence(getActionTriggerChannel(tableId)); - const received: IActionTrigger[] = []; - let capture = false; - let settled = false; - let actCompleted = false; - let idleTimer: NodeJS.Timeout | undefined; - - const cleanup = () => { - clearTimeout(timeout); - if (idleTimer) clearTimeout(idleTimer); - presence.removeListener('receive', onReceive); - try { - presence.unsubscribe(); - presence.destroy(); - } catch { - void 0; - } - connection.close(); - }; - - const finish = (error?: unknown) => { - if (settled) return; - settled = true; - cleanup(); - if (error) { - reject(error instanceof Error ? error : new Error(String(error))); - return; - } - resolve(received); - }; - - const onReceive = (_id: string, batch: IActionTrigger[]) => { - if (!capture) { - return; - } - received.push(...batch); - if (until?.(received)) { - finish(); - return; - } - if (!actCompleted) { - return; - } - if (idleTimer) clearTimeout(idleTimer); - idleTimer = setTimeout(() => finish(), idleMs); - }; - - const timeout = setTimeout(() => { - finish(new Error('Action trigger timeout')); - }, timeoutMs); - - presence.subscribe(async (error: unknown) => { - if (error) { - finish(error); - return; - } - - presence.on('receive', onReceive); - - try { - capture = true; - await act(); - actCompleted = true; - if (until?.(received)) { - finish(); - return; - } - if (idleTimer) clearTimeout(idleTimer); - idleTimer = setTimeout(() => finish(), idleMs); - } catch (actError) { - finish(actError); - } - }); - }); -}; - describe('V2 action trigger field conversion (e2e)', () => { let app: INestApplication; let cookie: string; @@ -164,7 +50,7 @@ describe('V2 action trigger field conversion (e2e)', () => { const table = await createTable(baseId, { name: 'v2-action-trigger-field-conversion', fields: [ - { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Name', type: FieldType.SingleLineText }, { name: amountTextFieldName, type: FieldType.SingleLineText }, ], }); @@ -212,7 +98,7 @@ describe('V2 action trigger field conversion (e2e)', () => { ); expect(response.status).toBe(200); - expect(response.headers['x-teable-v2']).toBe('true'); + expect(response.headers[v2ResponseHeader]).toBe('true'); }, }); @@ -253,7 +139,7 @@ describe('V2 action trigger field conversion (e2e)', () => { const table = await createTable(baseId, { name: 'v2-action-trigger-field-conversion-formula', fields: [ - { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Name', type: FieldType.SingleLineText }, { name: amountTextFieldName, type: FieldType.SingleLineText }, ], }); @@ -304,7 +190,7 @@ describe('V2 action trigger field conversion (e2e)', () => { ); expect(response.status).toBe(200); - expect(response.headers['x-teable-v2']).toBe('true'); + expect(response.headers[v2ResponseHeader]).toBe('true'); }, }); @@ -330,7 +216,7 @@ describe('V2 action trigger field conversion (e2e)', () => { const foreignTable = await createTable(baseId, { name: 'v2-action-trigger-foreign-schema-source', fields: [ - { name: 'Name', type: 'singleLineText', isPrimary: true }, + { name: 'Name', type: 'singleLineText' }, { name: 'Status', type: 'singleSelect', @@ -356,7 +242,6 @@ describe('V2 action trigger field conversion (e2e)', () => { id: hostPrimaryFieldId, name: 'Name', type: 'singleLineText', - isPrimary: true, }, { id: linkFieldId, @@ -436,7 +321,7 @@ describe('V2 action trigger field conversion (e2e)', () => { ); expect(response.status).toBe(200); - expect(response.headers['x-teable-v2']).toBe('true'); + expect(response.headers[v2ResponseHeader]).toBe('true'); }, }); @@ -461,7 +346,7 @@ describe('V2 action trigger field conversion (e2e)', () => { const table = await createTable(baseId, { name: 'v2-action-trigger-create-formula-field', fields: [ - { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Name', type: FieldType.SingleLineText }, { id: sourceFieldId, name: amountTextFieldName, type: FieldType.Number }, ], }); @@ -499,7 +384,7 @@ describe('V2 action trigger field conversion (e2e)', () => { ); expect(response.status).toBe(201); - expect(response.headers['x-teable-v2']).toBe('true'); + expect(response.headers[v2ResponseHeader]).toBe('true'); }, }); diff --git a/apps/nextjs-app/public/images/layout/app-builder-dark.png b/apps/nextjs-app/public/images/layout/app-builder-dark.png new file mode 100644 index 0000000000..e6b8bd3bd7 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/app-builder-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/app-builder-light.png b/apps/nextjs-app/public/images/layout/app-builder-light.png new file mode 100644 index 0000000000..2478012861 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/app-builder-light.png differ diff --git a/apps/nextjs-app/public/images/layout/chat-dark.png b/apps/nextjs-app/public/images/layout/chat-dark.png new file mode 100644 index 0000000000..4b01aaaf79 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/chat-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/chat-light.png b/apps/nextjs-app/public/images/layout/chat-light.png new file mode 100644 index 0000000000..5f34050d40 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/chat-light.png differ diff --git a/apps/nextjs-app/src/features/app/blocks/db-connection/Panel.tsx b/apps/nextjs-app/src/features/app/blocks/db-connection/Panel.tsx index 04241c00e8..d63bc19c32 100644 --- a/apps/nextjs-app/src/features/app/blocks/db-connection/Panel.tsx +++ b/apps/nextjs-app/src/features/app/blocks/db-connection/Panel.tsx @@ -11,6 +11,7 @@ import { CopyButton } from '../../components/CopyButton'; import { useBaseUsage } from '../../hooks/useBaseUsage'; import { useIsCloud } from '../../hooks/useIsCloud'; import { useIsCommunity } from '../../hooks/useIsCommunity'; +import { getDbConnectionCreateErrorI18nKey } from './db-connection-errors'; import { useDbConnection } from './hooks'; export const DbConnectionPanel = ({ className }: { className?: string }) => { @@ -34,6 +35,12 @@ export const DbConnectionPanel = ({ className }: { className?: string }) => { toast.error(t('table:connection.createFailed')); } }, + onError: (error) => { + toast.error(t(getDbConnectionCreateErrorI18nKey(error))); + }, + meta: { + preventGlobalError: true, + }, }); const mutationDelete = useMutation({ diff --git a/apps/nextjs-app/src/features/app/blocks/db-connection/db-connection-errors.spec.ts b/apps/nextjs-app/src/features/app/blocks/db-connection/db-connection-errors.spec.ts new file mode 100644 index 0000000000..c0d83ca4ad --- /dev/null +++ b/apps/nextjs-app/src/features/app/blocks/db-connection/db-connection-errors.spec.ts @@ -0,0 +1,27 @@ +import { HttpErrorCode } from '@teable/core'; +import { describe, expect, it } from 'vitest'; +import { + getDbConnectionCreateErrorI18nKey, + readonlyRolePrivilegeUnavailableReason, +} from './db-connection-errors'; + +describe('getDbConnectionCreateErrorI18nKey', () => { + it('uses BYODB readonly capability copy for scoped role or grant privilege failures', () => { + expect( + getDbConnectionCreateErrorI18nKey({ + code: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE, + data: { + reason: readonlyRolePrivilegeUnavailableReason, + }, + }) + ).toBe('table:connection.readonlyUnavailable'); + }); + + it('falls back to the existing create failure copy for other errors', () => { + expect( + getDbConnectionCreateErrorI18nKey({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + }) + ).toBe('table:connection.createFailed'); + }); +}); diff --git a/apps/nextjs-app/src/features/app/blocks/db-connection/db-connection-errors.ts b/apps/nextjs-app/src/features/app/blocks/db-connection/db-connection-errors.ts new file mode 100644 index 0000000000..3ed1f60df5 --- /dev/null +++ b/apps/nextjs-app/src/features/app/blocks/db-connection/db-connection-errors.ts @@ -0,0 +1,17 @@ +import { HttpErrorCode, type IHttpError } from '@teable/core'; + +export const readonlyRolePrivilegeUnavailableReason = 'readonly_role_privilege_unavailable'; + +export const getDbConnectionCreateErrorI18nKey = (error: unknown) => { + const httpError = error as Partial; + const data = httpError.data as { reason?: unknown } | undefined; + + if ( + httpError.code === HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE && + data?.reason === readonlyRolePrivilegeUnavailableReason + ) { + return 'table:connection.readonlyUnavailable'; + } + + return 'table:connection.createFailed'; +}; diff --git a/apps/nextjs-app/src/features/app/blocks/setting/components/FormItem.tsx b/apps/nextjs-app/src/features/app/blocks/setting/components/FormItem.tsx index b403a8b933..5dcd5fa7af 100644 --- a/apps/nextjs-app/src/features/app/blocks/setting/components/FormItem.tsx +++ b/apps/nextjs-app/src/features/app/blocks/setting/components/FormItem.tsx @@ -5,7 +5,8 @@ import type { z } from 'zod'; import { RequireCom } from './RequireCom'; interface IFormItemProps { - title: string; + title: React.ReactNode; + titleExtra?: React.ReactNode; description?: string; required?: boolean; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -19,7 +20,7 @@ export interface IFormItemRef { } export const FormItem = forwardRef((props, ref) => { - const { title, description, required, children, validateSchema } = props; + const { title, titleExtra, description, required, children, validateSchema } = props; const containerRef = useRef(null); const [error, setError] = useState(); @@ -44,7 +45,7 @@ export const FormItem = forwardRef((props, ref) =>
"::text + // expression) instead of a jsonb_array_elements + regex subquery that cannot use it. Trades + // negligible precision (JSON brackets/quotes become matchable) for index usage. + return ok( + sql`(${columnRef})::text ILIKE ${`%${escapeLikeWildcards(search.value)}%`} ESCAPE '\\'` + ); + } return ok(buildPlainMultipleCondition(columnRef, search.value)); } diff --git a/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.spec.ts b/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.spec.ts index 200c31bba6..cfa3e74395 100644 --- a/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.spec.ts @@ -153,6 +153,22 @@ const createVisitor = (...fields: Array>) => } ); +const createVisitorWithContext = ( + fields: Array>, + context: Partial[3]> +) => + CellValueMutateVisitor.create( + createTestDb() as never, + createTable(...fields) as never, + 'public.records', + { + recordId: mkRecordId('source'), + actorId: 'usrActor000000001', + now: '2025-01-01T00:00:00.000Z', + ...context, + } + ); + const createForeignTable = (params: { tableId?: string; dbTableName: string; @@ -553,13 +569,52 @@ describe('CellValueMutateVisitor', () => { const buildResult = visitor.build(); expect(buildResult.isOk()).toBe(true); const built = buildResult._unsafeUnwrap(); + const setClauses = visitor.getSetClausesRaw().setClauses; expect(built.changedFieldIds.map((id) => id.toString())).toEqual([ textField.id().toString(), lastModifiedTimeField.id().toString(), lastModifiedByField.id().toString(), ]); expect(normalizeSql(built.mainUpdate.sql)).toContain('"lmt_col" = $'); - expect(normalizeSql(built.mainUpdate.sql)).toContain('jsonb_build_object'); + expect(normalizeSql(built.mainUpdate.sql)).not.toContain('public.users'); + expect(JSON.parse(setClauses.lmb_col as string)).toMatchObject({ + id: 'usrActor000000001', + title: 'usrActor000000001', + email: null, + }); + }); + + it('uses actor identity context for tracked last-modified-by snapshots', () => { + const textField = createField({ + fieldId: 'trackedText', + type: 'singleLineText', + dbFieldName: 'text_col', + }); + const lastModifiedByField = createTrackedLastModifiedByField({ + fieldId: 'trackedBy', + dbFieldName: 'lmb_col', + trackedFieldIds: [textField.id()], + }); + const visitor = createVisitorWithContext([textField, lastModifiedByField as never], { + actorName: 'Nee', + actorEmail: 'nee@teable.io', + }); + + expect( + visitor + .visitSetSingleLineTextValue({ + fieldId: textField.id(), + value: CellValue.fromValidated('updated'), + } as never) + .isOk() + ).toBe(true); + + expect(visitor.build().isOk()).toBe(true); + expect(JSON.parse(visitor.getSetClausesRaw().setClauses.lmb_col as string)).toMatchObject({ + id: 'usrActor000000001', + title: 'Nee', + email: 'nee@teable.io', + }); }); it('handles row-order updates and unsupported logical combinators', () => { diff --git a/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.ts b/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.ts index 34b5a131d5..2cba624b3a 100644 --- a/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.ts +++ b/packages/v2/adapter-table-repository-postgres/src/record/visitors/CellValueMutateVisitor.ts @@ -38,7 +38,7 @@ import { sql } from 'kysely'; import { err, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; -import { resolveUserAvatarUrlPrefix } from '../../shared/userAvatarUrl'; +import { buildUserAvatarUrl } from '../../shared/userAvatarUrl'; import { buildAttachmentTableReplaceQueries } from '../attachments/attachmentTableMutations'; import { buildFilledLinkValueExpression } from '../buildFilledLinkValueExpression'; import { isPersistedAsGeneratedColumn } from '../computed/isPersistedAsGeneratedColumn'; @@ -317,30 +317,13 @@ export class CellValueMutateVisitor implements ICellValueSpecVisitor { return ok(validTracked); } - /** - * Build the value for LastModifiedBy field using COALESCE subquery. - * Fetches user info from users table, with fallback to just the actor ID. - */ - private buildLastModifiedByValue(): ReturnType { - const avatarPrefix = resolveUserAvatarUrlPrefix(); - return sql`COALESCE( - ( - SELECT jsonb_build_object( - 'id', u.id, - 'title', u.name, - 'email', u.email, - 'avatarUrl', ${avatarPrefix} || u.id - ) - FROM public.users u - WHERE u.id = ${this.ctx.actorId}::text - ), - jsonb_build_object( - 'id', ${this.ctx.actorId}::text, - 'title', ${this.ctx.actorId}::text, - 'email', NULL::text, - 'avatarUrl', ${avatarPrefix}::text || ${this.ctx.actorId}::text - ) - )`; + private buildLastModifiedByValue(): string { + return JSON.stringify({ + id: this.ctx.actorId, + title: this.ctx.actorName ?? this.ctx.actorId, + email: this.ctx.actorEmail ?? null, + avatarUrl: buildUserAvatarUrl(this.ctx.actorId), + }); } private addSimpleValue(fieldId: FieldId, rawValue: unknown): Result { diff --git a/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.spec.ts b/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.spec.ts index 7c1c1e99be..111d01294e 100644 --- a/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.spec.ts @@ -10,6 +10,8 @@ import { IncomingLinkCandidateSpec, IncomingLinkSelectedSpec, LinkFieldConfig, + LookupField, + LookupOptions, RecordByIdSpec, RecordByIdsSpec, RecordId, @@ -332,6 +334,28 @@ const createLinkTitleReferenceFields = () => { return { linkField, tagNameField }; }; +const createLookupLinkTitleReferenceFields = () => { + const { linkField, tagNameField } = createLinkTitleReferenceFields(); + const lookupOptions = LookupOptions.create({ + linkFieldId: linkField.id().toString(), + lookupFieldId: linkField.id().toString(), + foreignTableId: `tbl${'f'.repeat(16)}`, + })._unsafeUnwrap(); + const lookupField = LookupField.create({ + id: FieldId.create(`fld${'p'.repeat(16)}`)._unsafeUnwrap(), + name: FieldName.create('Lookup Tags')._unsafeUnwrap(), + innerField: linkField, + lookupOptions, + isMultipleCellValue: true, + })._unsafeUnwrap(); + + lookupField + .setDbFieldName(DbFieldName.rehydrate('col_lookup_tags')._unsafeUnwrap()) + ._unsafeUnwrap(); + + return { lookupField, tagNameField }; +}; + // ============================================================================ // Tests // ============================================================================ @@ -539,6 +563,29 @@ describe('TableRecordConditionWhereVisitor NULL handling', () => { expect(parameters).toEqual([]); }); + test('lookup of link title comparison to host text stays on the title-matching path', () => { + const { lookupField, tagNameField } = createLookupLinkTitleReferenceFields(); + const value = RecordConditionFieldReferenceValue.create(tagNameField)._unsafeUnwrap(); + const spec = lookupField.spec().create({ operator: 'is', value }); + expect(spec.isOk()).toBe(true); + if (spec.isErr()) return; + + const visitor = new TableRecordConditionWhereVisitor({ + tableAlias: 'f', + hostTableAlias: 't', + }); + const visitResult = spec.value.accept(visitor); + expect(visitResult.isOk()).toBe(true); + const where = visitor.where(); + expect(where.isOk()).toBe(true); + if (where.isErr()) return; + + const { sql, parameters } = compileWhere(db, where.value); + expect(sql).toContain(`__link->>'title' = ("t"."col_tag_name")::text`); + expect(sql).not.toBe('1 = 0'); + expect(parameters).toEqual([]); + }); + test('date isBefore with field reference uses host table alias', () => { const value = RecordConditionFieldReferenceValue.create(cutoffDateField)._unsafeUnwrap(); const spec = dueDateField.spec().create({ operator: 'isBefore', value }); diff --git a/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.ts b/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.ts index abcbf0bd0b..3628a67f1b 100644 --- a/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.ts +++ b/packages/v2/adapter-table-repository-postgres/src/record/visitors/TableRecordConditionWhereVisitor.ts @@ -52,11 +52,19 @@ const resolveLookupInnerField = (field: core.Field): core.Field | undefined => { return undefined; }; -const fieldIsLookupWithUserOrLinkInner = (field: core.Field): boolean => { +const resolveEffectiveLookupField = (field: core.Field): core.Field => { const innerField = resolveLookupInnerField(field); - return innerField ? fieldIsUserOrLink(innerField) : false; + return innerField ? resolveEffectiveLookupField(innerField) : field; }; +const fieldIsLookupWithUserOrLinkInner = (field: core.Field): boolean => { + const effectiveField = resolveEffectiveLookupField(field); + return effectiveField !== field && fieldIsUserOrLink(effectiveField); +}; + +const fieldHasLinkDisplayValue = (field: core.Field): boolean => + fieldIsLink(resolveEffectiveLookupField(field)); + const isArrayLikeOutputField = (field: core.Field, isMultiple: boolean): boolean => { // Query model currently forces lookup/conditionalLookup output to arrays for v1 parity. return isMultiple || fieldIsLookup(field) || fieldIsConditionalLookup(field); @@ -444,7 +452,7 @@ const classifyFieldReferenceComparison = ( if (rightIsUserOrLinkLike) { return ok({ kind: 'userOrLinkIds' }); } - if (fieldIsLink(field)) { + if (fieldHasLinkDisplayValue(field)) { return ok({ kind: 'linkTitle' }); } return ok(hasHostTableAlias ? { kind: 'incompatible' } : { kind: 'generic' }); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/di/register.ts b/packages/v2/adapter-table-repository-postgres/src/schema/di/register.ts index 4cbe08fe12..a8baf0bf01 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/di/register.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/di/register.ts @@ -15,7 +15,7 @@ class NoopComputedFieldBackfillService { } async backfillMany() { - return ok(undefined); + return ok({ fields: [] }); } async executeSync() { @@ -23,7 +23,7 @@ class NoopComputedFieldBackfillService { } async executeSyncMany() { - return ok(undefined); + return ok({ fields: [] }); } } @@ -58,6 +58,12 @@ export const registerV2PostgresDdlAdapter = async ( new NoopComputedFieldBackfillService() ); } + if (!c.isRegistered(v2CoreTokens.computedFieldBackfillService)) { + c.registerInstance( + v2CoreTokens.computedFieldBackfillService, + c.resolve(v2RecordRepositoryPostgresTokens.computedFieldBackfillService) + ); + } if (!c.isRegistered(v2RecordRepositoryPostgresTokens.computedFieldCascadeService)) { c.registerInstance( diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.spec.ts b/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.spec.ts index 5d4a2b9c4d..c3bf29e5f7 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.spec.ts @@ -1,4 +1,5 @@ import { PGlite } from '@electric-sql/pglite'; +import { PostgresUnitOfWorkTransaction } from '@teable/v2-adapter-db-postgres-shared'; import { ActorId, BaseId, @@ -38,13 +39,14 @@ import { PostgresTableSchemaRepository } from './PostgresTableSchemaRepository'; class PGliteDriver { #client: PGlite; + readonly queries: CompiledQuery[] = []; constructor(client: PGlite) { this.#client = client; } async acquireConnection() { - return new PGliteConnection(this.#client); + return new PGliteConnection(this.#client, this.queries); } async beginTransaction(connection: PGliteConnection) { @@ -75,11 +77,15 @@ class PGliteDriver { class PGliteConnection { #client: PGlite; - constructor(client: PGlite) { + constructor( + client: PGlite, + private readonly queries: CompiledQuery[] + ) { this.#client = client; } async executeQuery(compiledQuery: CompiledQuery): Promise> { + this.queries.push(compiledQuery); const result = await this.#client.query(compiledQuery.sql, [...compiledQuery.parameters]); return { rows: result.rows, @@ -109,14 +115,14 @@ class PGliteConnection { } class PGliteDialect implements Dialect { - #client: PGlite; + readonly driver: PGliteDriver; constructor(client: PGlite) { - this.#client = client; + this.driver = new PGliteDriver(client); } createDriver() { - return new PGliteDriver(this.#client); + return this.driver; } createAdapter() { @@ -179,7 +185,7 @@ class FakeComputedFieldBackfillService { skipDistinctFilter: input.skipDistinctFilter, includeOneManyTwoWay: input.includeOneManyTwoWay, }); - return ok(undefined); + return ok({ fields: input.fields }); } async executeSync() { @@ -187,7 +193,7 @@ class FakeComputedFieldBackfillService { } async executeSyncMany() { - return ok(undefined); + return ok({ fields: [] }); } } @@ -363,6 +369,81 @@ describe('PostgresTableSchemaRepository', () => { ).toEqual([]); }); + it('creates generated system field columns directly during table creation', async () => { + const baseId = BaseId.generate()._unsafeUnwrap(); + const tableId = TableId.generate()._unsafeUnwrap(); + const tableName = TableName.create('Generated Columns')._unsafeUnwrap(); + const actorId = ActorId.create('system')._unsafeUnwrap(); + const context: IExecutionContext = { actorId }; + + const builder = Table.builder().withBaseId(baseId).withId(tableId).withName(tableName); + builder.field().singleLineText().withName(FieldName.create('Name')._unsafeUnwrap()).done(); + builder.field().createdBy().withName(FieldName.create('Creator')._unsafeUnwrap()).done(); + builder.field().autoNumber().withName(FieldName.create('No')._unsafeUnwrap()).done(); + builder.view().defaultGrid().done(); + const table = builder.build()._unsafeUnwrap(); + + const tableRepository = new FakeTableRepository([table]); + const repository = new PostgresTableSchemaRepository( + db, + tableRepository as never, + new FakeComputedFieldBackfillService(), + new FakeComputedFieldCascadeService(), + new FakeComputedUpdatePlanner() as never, + new FakeFieldDependencyGraph() as never + ); + + (await repository.insert(context, table))._unsafeUnwrap(); + + const { schema, tableName: dbTableName } = table + .dbTableName() + .andThen((name) => name.split({ defaultSchema: null })) + ._unsafeUnwrap(); + const introspector = new PostgresSchemaIntrospector(db); + const generatedFields = table.getFields().filter((field) => { + const persistedResult = + 'isPersistedAsGeneratedColumn' in field ? field.isPersistedAsGeneratedColumn() : ok(false); + return persistedResult.isOk() && persistedResult.value; + }); + + for (const field of generatedFields) { + const columnName = field + .dbFieldName() + .andThen((name) => name.value()) + ._unsafeUnwrap(); + const column = ( + await introspector.getColumn(schema, dbTableName, columnName) + )._unsafeUnwrap(); + expect(column?.isGenerated).toBe(true); + } + + const createdByField = table + .getFields() + .find((field) => field.type().toString() === 'createdBy'); + const createdByColumnName = createdByField + ?.dbFieldName() + .andThen((name) => name.value()) + ._unsafeUnwrap(); + if (!createdByColumnName) { + throw new Error('Created By field missing'); + } + const createdByColumn = ( + await introspector.getColumn(schema, dbTableName, createdByColumnName) + )._unsafeUnwrap(); + expect(createdByColumn?.isGenerated).toBe(false); + + const checker = createSchemaChecker({ + db, + introspector, + schema: baseId.toString(), + }); + const results = await collectFinalCheckResults(checker.checkTable(table)); + + expect( + results.filter((result) => result.status === 'error' || result.status === 'warn') + ).toEqual([]); + }); + it('ensures an existing table schema without recreating it', async () => { const baseId = BaseId.generate()._unsafeUnwrap(); const tableId = TableId.generate()._unsafeUnwrap(); @@ -722,6 +803,113 @@ describe('PostgresTableSchemaRepository', () => { expect(columnResult?.column_name).toBe(fkColumnName); }); + it('does not reinstall undo capture triggers after batch table creation in a transaction', async () => { + const pglite = await PGlite.create(); + const dialect = new PGliteDialect(pglite); + const db = new Kysely({ dialect }); + + try { + await installUndoCaptureGlobals(db); + + const baseId = BaseId.generate()._unsafeUnwrap(); + const actorId = ActorId.create('system')._unsafeUnwrap(); + const context: IExecutionContext = { actorId }; + const tables = ['Batch Undo A', 'Batch Undo B'].map((name) => { + const builder = Table.builder() + .withBaseId(baseId) + .withId(TableId.generate()._unsafeUnwrap()) + .withName(TableName.create(name)._unsafeUnwrap()); + builder.field().singleLineText().withName(FieldName.create('Name')._unsafeUnwrap()).done(); + builder.view().defaultGrid().done(); + return builder.build()._unsafeUnwrap(); + }); + + const repository = new PostgresTableSchemaRepository( + db, + new FakeTableRepository(tables) as never, + new FakeComputedFieldBackfillService(), + new FakeComputedFieldCascadeService(), + new FakeComputedUpdatePlanner() as never, + new FakeFieldDependencyGraph() as never + ); + dialect.driver.queries.length = 0; + + await db.transaction().execute(async (trx) => { + const result = await repository.insertMany( + { ...context, transaction: new PostgresUnitOfWorkTransaction(trx as never) }, + tables + ); + result._unsafeUnwrap(); + }); + + const triggerInstallQueries = dialect.driver.queries.filter((query) => + query.sql.includes('CREATE OR REPLACE TRIGGER "__teable_undo_capture"') + ); + + expect(triggerInstallQueries).toHaveLength(tables.length); + } finally { + await db.destroy(); + } + }); + + it('deduplicates schema creation and can skip undo capture setup for batch table import', async () => { + const pglite = await PGlite.create(); + const dialect = new PGliteDialect(pglite); + const db = new Kysely({ dialect }); + + try { + await installUndoCaptureGlobals(db); + + const baseId = BaseId.generate()._unsafeUnwrap(); + const actorId = ActorId.create('system')._unsafeUnwrap(); + const context: IExecutionContext = { actorId }; + const tables = ['Batch Import A', 'Batch Import B'].map((name) => { + const builder = Table.builder() + .withBaseId(baseId) + .withId(TableId.generate()._unsafeUnwrap()) + .withName(TableName.create(name)._unsafeUnwrap()); + builder.field().singleLineText().withName(FieldName.create('Name')._unsafeUnwrap()).done(); + builder.view().defaultGrid().done(); + return builder.build()._unsafeUnwrap(); + }); + + const repository = new PostgresTableSchemaRepository( + db, + new FakeTableRepository(tables) as never, + new FakeComputedFieldBackfillService(), + new FakeComputedFieldCascadeService(), + new FakeComputedUpdatePlanner() as never, + new FakeFieldDependencyGraph() as never + ); + dialect.driver.queries.length = 0; + + await db.transaction().execute(async (trx) => { + const result = await repository.insertMany( + { ...context, transaction: new PostgresUnitOfWorkTransaction(trx as never) }, + tables, + { optimizeForEmptyTables: true, skipUndoCaptureSetup: true } + ); + result._unsafeUnwrap(); + }); + + const createSchemaQueries = dialect.driver.queries.filter((query) => + query.sql.includes(`create schema if not exists "${baseId.toString()}"`) + ); + const triggerProbeQueries = dialect.driver.queries.filter((query) => + query.sql.includes('FROM pg_trigger AS t') + ); + const triggerInstallQueries = dialect.driver.queries.filter((query) => + query.sql.includes('CREATE OR REPLACE TRIGGER "__teable_undo_capture"') + ); + + expect(createSchemaQueries).toHaveLength(1); + expect(triggerProbeQueries).toHaveLength(0); + expect(triggerInstallQueries).toHaveLength(0); + } finally { + await db.destroy(); + } + }); + it('routes batch reference metadata statements to the meta DB when databases are split', async () => { const dataPglite = await PGlite.create(); const metaPglite = await PGlite.create(); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.ts b/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.ts index 6732d359fc..a98b964785 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/repositories/PostgresTableSchemaRepository.ts @@ -13,6 +13,7 @@ import type { DomainError, ITracer, SpanAttributes, + TableSchemaInsertManyOptions, } from '@teable/v2-core'; import { TraceSpan, @@ -80,7 +81,7 @@ type ComputedFieldBackfillService = { includeOneManyTwoWay?: boolean; skipDistinctFilter?: boolean; } - ): Promise>; + ): Promise }, DomainError>>; }; type ComputedFieldCascadeService = { @@ -99,6 +100,7 @@ type ComputedFieldCascadeService = { type FieldValueChangeSet = { selfBackfillFieldIds: ReadonlyArray; valueChangedFieldIds: ReadonlyArray; + recordActionFieldIds: ReadonlyArray; deferredBackfillFieldIds: ReadonlyArray; hasDbStorageTypeChange: boolean; }; @@ -206,6 +208,8 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { const targetDb = currentScope === 'meta' ? this.resolveMetaDb(context) : db; await executeTableSchemaStatements(targetDb, batch, { ...trace, + dataDb: db as Kysely | Transaction, + metaDb: this.resolveMetaDb(context) as Kysely | Transaction, enforceRelationAccess: this.db !== this.metaDb, }); batch = []; @@ -225,11 +229,13 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { private async ensureDeferredForeignKeys( context: IExecutionContext, - tables: ReadonlyArray
+ tables: ReadonlyArray
, + options?: Pick ): Promise> { const repository = this; return safeTry(async function* () { const db = resolvePostgresDbOrTx(repository.db, context) as Kysely; + const metaDb = repository.resolveMetaDb(context) as Kysely; const introspector = new PostgresSchemaIntrospector(db); for (const table of tables) { @@ -249,12 +255,14 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { const ctx = createSchemaRuleContext({ db, + metaDb, introspector, schema, tableName, tableId: table.id().toString(), field, table, + optimizeForEmptyTables: options?.optimizeForEmptyTables, }); const deferredFkRules = rules.filter( @@ -267,6 +275,8 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { const statements = yield* rule.up(ctx); await executeTableSchemaStatements(db, statements, { tracer: context.tracer, + dataDb: db as Kysely | Transaction, + metaDb: metaDb as Kysely | Transaction, attributes: { [TeableSpanAttributes.TABLE_ID]: table.id().toString(), 'teable.base_id': table.baseId().toString(), @@ -332,7 +342,8 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { private async insertTableFieldSchemas( context: IExecutionContext, table: Table, - knownTables: ReadonlyArray
= [table] + knownTables: ReadonlyArray
= [table], + options?: Pick ): Promise> { const repository = this; return await safeTry(async function* () { @@ -350,6 +361,7 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { tableName, tableId: table.id().toString(), tableLocationsById, + optimizeForEmptyTables: options?.optimizeForEmptyTables, }); const statements = yield* visitor.apply(table); @@ -406,10 +418,35 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { }); } - @TraceSpan() - async insert(context: IExecutionContext, table: Table): Promise> { + private async insertTableSchema( + context: IExecutionContext, + table: Table, + knownTables: ReadonlyArray
= [table], + options?: Pick< + TableSchemaInsertManyOptions, + 'optimizeForEmptyTables' | 'skipUndoCaptureSetup' + > & { + skipSchemaCreate?: boolean; + } + ): Promise< + Result< + { + schema: string | null; + tableName: string; + fieldStatements: ReadonlyArray; + }, + DomainError + > + > { const repository = this; - return await safeTry(async function* () { + return await safeTry< + { + schema: string | null; + tableName: string; + fieldStatements: ReadonlyArray; + }, + DomainError + >(async function* () { yield* ensureDbFieldNames(table.getFields()); const { schema, tableName } = yield* table @@ -439,28 +476,19 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { schema, tableName, tableId: table.id().toString(), - tableLocationsById: yield* buildTableLocationsById([table]), + tableLocationsById: yield* buildTableLocationsById(knownTables), + optimizeForEmptyTables: options?.optimizeForEmptyTables, }); const fieldStatements = yield* visitor.apply(table); try { const compiledStatements: CompiledQuery[] = []; - if (schema && schema !== 'public') { + if (!options?.skipSchemaCreate && schema && schema !== 'public') { compiledStatements.push(db.schema.createSchema(schema).ifNotExists().compile()); } compiledStatements.push(builderRef.builder.compile()); await executeCompiledQueries(db, compiledStatements); - await repository.executeScopedTableSchemaStatements(context, db, fieldStatements, { - tracer: context.tracer, - attributes: { - [TeableSpanAttributes.TABLE_ID]: table.id().toString(), - 'teable.base_id': table.baseId().toString(), - 'teable.table_name': tableName, - 'teable.schema': schema ?? 'public', - 'teable.schema.statement.source': 'table_schema_insert', - }, - }); } catch (error) { return err( domainError.infrastructure({ @@ -469,47 +497,146 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { ); } - try { - await ensureUndoCaptureInfrastructure( - repository.db, - db, - toQualifiedIdentifierLiteral(schema, tableName), - `${schema ?? 'public'}.${tableName}` - ); - } catch { - // Snapshot capture wiring is best-effort and must not block table creation. + if (!options?.skipUndoCaptureSetup) { + try { + await ensureUndoCaptureInfrastructure( + repository.db, + db, + toQualifiedIdentifierLiteral(schema, tableName), + `${schema ?? 'public'}.${tableName}` + ); + } catch { + // Snapshot capture wiring is best-effort and must not block table creation. + } } - return ok(undefined); + return ok({ schema, tableName, fieldStatements }); }); } + private async ensureSchemas( + context: IExecutionContext, + schemas: ReadonlyArray + ): Promise> { + const schemaNames = [ + ...new Set( + schemas.filter((schema): schema is string => schema != null && schema !== 'public') + ), + ]; + if (schemaNames.length === 0) { + return ok(undefined); + } + + const db = resolvePostgresDbOrTx(this.db, context); + try { + await executeCompiledQueries( + db, + schemaNames.map((schema) => db.schema.createSchema(schema).ifNotExists().compile()) + ); + return ok(undefined); + } catch (error) { + return err( + domainError.infrastructure({ + message: `Failed to create table schemas: ${describeError(error)}`, + }) + ); + } + } + + @TraceSpan() + async insert(context: IExecutionContext, table: Table): Promise> { + const result = await this.insertTableSchema(context, table); + if (result.isErr()) { + return err(result.error); + } + + const { schema, tableName, fieldStatements } = result.value; + const db = resolvePostgresDbOrTx(this.db, context); + try { + await this.executeScopedTableSchemaStatements(context, db, fieldStatements, { + tracer: context.tracer, + attributes: { + [TeableSpanAttributes.TABLE_ID]: table.id().toString(), + 'teable.base_id': table.baseId().toString(), + 'teable.table_name': tableName, + 'teable.schema': schema ?? 'public', + 'teable.schema.statement.source': 'table_schema_insert', + }, + }); + } catch (error) { + return err( + domainError.infrastructure({ + message: `Failed to insert table schema: ${describeError(error)}`, + }) + ); + } + + return ok(undefined); + } + @TraceSpan() async insertMany( context: IExecutionContext, tables: ReadonlyArray
, - options?: { knownTables?: ReadonlyArray
} + options?: TableSchemaInsertManyOptions ): Promise> { const knownTables = options?.knownTables ?? tables; + const fieldStatementGroups: Array<{ + table: Table; + schema: string | null; + tableName: string; + fieldStatements: ReadonlyArray; + }> = []; + + const tableLocations: Array<{ schema: string | null; tableName: string }> = []; for (const table of tables) { - const result = await this.insertTableSkeleton(context, table); - if (result.isErr()) return err(result.error); + const tableLocation = table + .dbTableName() + .andThen((name) => name.split({ defaultSchema: null })); + if (tableLocation.isErr()) return err(tableLocation.error); + tableLocations.push(tableLocation.value); } + const ensureSchemasResult = await this.ensureSchemas( + context, + tableLocations.map((tableLocation) => tableLocation.schema) + ); + if (ensureSchemasResult.isErr()) return err(ensureSchemasResult.error); for (const table of tables) { - const result = await this.insertTableFieldSchemas(context, table, knownTables); + const result = await this.insertTableSchema(context, table, knownTables, { + ...options, + skipSchemaCreate: true, + }); if (result.isErr()) return err(result.error); + fieldStatementGroups.push({ table, ...result.value }); } - for (const table of tables) { - const result = await this.ensureUndoCaptureForTable(context, table); - if (result.isErr()) return err(result.error); + const db = resolvePostgresDbOrTx(this.db, context); + for (const { table, schema, tableName, fieldStatements } of fieldStatementGroups) { + try { + await this.executeScopedTableSchemaStatements(context, db, fieldStatements, { + tracer: context.tracer, + attributes: { + [TeableSpanAttributes.TABLE_ID]: table.id().toString(), + 'teable.base_id': table.baseId().toString(), + 'teable.table_name': tableName, + 'teable.schema': schema ?? 'public', + 'teable.schema.statement.source': 'table_schema_insert_fields', + }, + }); + } catch (error) { + return err( + domainError.infrastructure({ + message: `Failed to insert table schema: ${describeError(error)}`, + }) + ); + } } // Some FK constraints are conditionally created only if the target table already exists. // In batch table creation, referenced tables might be created later, so we do a second pass // to (idempotently) add any missing FK constraints once all tables exist. - const ensureFkResult = await this.ensureDeferredForeignKeys(context, tables); + const ensureFkResult = await this.ensureDeferredForeignKeys(context, tables, options); if (ensureFkResult.isErr()) return err(ensureFkResult.error); return ok(undefined); @@ -714,6 +841,7 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { return ok({ selfBackfillFieldIds: valueChangeVisitor.selfBackfillFields(), valueChangedFieldIds: valueChangeVisitor.valueChangedFields(), + recordActionFieldIds: valueChangeVisitor.recordActionFields(), deferredBackfillFieldIds: valueChangeVisitor.deferredBackfillFields(), hasDbStorageTypeChange: valueChangeVisitor.hasDbStorageTypeChange(), }); @@ -816,6 +944,22 @@ export class PostgresTableSchemaRepository implements ITableSchemaRepository { addActionTriggerTarget(table.id(), changedFieldIds, table.baseId()); + if (valueChanges.recordActionFieldIds.length > 0) { + const [firstRecordActionFieldId] = valueChanges.recordActionFieldIds; + table.requestActionTrigger({ + tableId: table.id(), + baseId: table.baseId(), + actionKey: 'setRecord', + payload: { + tableId: table.id().toString(), + fieldIds: valueChanges.recordActionFieldIds.map((fieldId) => fieldId.toString()), + ...(firstRecordActionFieldId + ? { field: { id: firstRecordActionFieldId.toString() } } + : {}), + }, + }); + } + const planResult = yield* await repository.computedUpdatePlanner.plan( { table, diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/checker/SchemaChecker.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/checker/SchemaChecker.ts index eb9a2d3b66..e420a2d8ba 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/checker/SchemaChecker.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/checker/SchemaChecker.ts @@ -1,6 +1,8 @@ import type { Table } from '@teable/v2-core'; +import type { ISchemaRule } from '../core'; import { getRuleRepairHint } from '../core/RuleRepairMetadata'; +import type { SchemaRuleContext } from '../context/SchemaRuleContext'; import { createSchemaRulePlanner, getSchemaRulePlanningStageDescription, @@ -15,11 +17,21 @@ import { warnResult, } from './SchemaCheckResult'; +const yieldToEventLoop = () => new Promise((resolve) => setImmediate(resolve)); + /** * Parameters for creating a SchemaChecker. */ export interface SchemaCheckerParams extends SchemaRulePlannerParams {} +const getRuleValidationCtx = (ctx: SchemaRuleContext, rule: ISchemaRule): SchemaRuleContext => + rule.validationScope === 'meta' + ? { + ...ctx, + db: ctx.metaDb, + } + : ctx; + /** * Checks the schema of a table field by field, rule by rule. * Yields results as an async generator for streaming. @@ -27,6 +39,10 @@ export interface SchemaCheckerParams extends SchemaRulePlannerParams {} export class SchemaChecker { constructor(private readonly params: SchemaCheckerParams) {} + private async checkRuleValidity(ctx: SchemaRuleContext, rule: ISchemaRule) { + return rule.isValid(getRuleValidationCtx(ctx, rule)); + } + /** * Check all fields in a table. * Yields check results one by one for each rule. @@ -34,6 +50,7 @@ export class SchemaChecker { async *checkTable(table: Table): AsyncGenerator { const planner = createSchemaRulePlanner(this.params); for (const planEntry of planner.planTable(table)) { + await yieldToEventLoop(); if (planEntry.type === 'error') { yield errorResult( pendingResult( @@ -89,7 +106,7 @@ export class SchemaChecker { yield runningResult(pending); try { - const validationResult = await rule.isValid(ctx); + const validationResult = await this.checkRuleValidity(ctx, rule); if (validationResult.isErr()) { yield errorResult(pending, validationResult.error.message); @@ -109,7 +126,8 @@ export class SchemaChecker { extra: validation.extra, extraItems: validation.extraItems, }; - const repairResult = getRuleRepairHint(rule, ctx, validation, { + const validationCtx = getRuleValidationCtx(ctx, rule); + const repairResult = getRuleRepairHint(rule, validationCtx, validation, { skipStatementCheck: true, }); const repair = repairResult.isOk() ? repairResult.value : undefined; @@ -135,6 +153,7 @@ export class SchemaChecker { ); validatedRules.set(rule.id, false); } + await yieldToEventLoop(); } } } @@ -149,6 +168,7 @@ export class SchemaChecker { ): AsyncGenerator { const planner = createSchemaRulePlanner(this.params); for (const planEntry of planner.planTable(table, { fieldId })) { + await yieldToEventLoop(); if (planEntry.type === 'error') { yield errorResult( pendingResult( @@ -198,7 +218,7 @@ export class SchemaChecker { yield runningResult(pending); try { - const validationResult = await rule.isValid(ctx); + const validationResult = await this.checkRuleValidity(ctx, rule); if (validationResult.isErr()) { yield errorResult(pending, validationResult.error.message); @@ -218,7 +238,8 @@ export class SchemaChecker { extra: validation.extra, extraItems: validation.extraItems, }; - const repairResult = getRuleRepairHint(rule, ctx, validation, { + const validationCtx = getRuleValidationCtx(ctx, rule); + const repairResult = getRuleRepairHint(rule, validationCtx, validation, { skipStatementCheck: true, }); const repair = repairResult.isOk() ? repairResult.value : undefined; @@ -252,6 +273,7 @@ export class SchemaChecker { ); validatedRules.set(rule.id, false); } + await yieldToEventLoop(); } } } diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/context/SchemaRuleContext.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/context/SchemaRuleContext.ts index a8d0bcb4dd..e67f79c39e 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/context/SchemaRuleContext.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/context/SchemaRuleContext.ts @@ -14,6 +14,9 @@ export interface SchemaRuleContext { /** Kysely database instance for generating SQL statements */ readonly db: Kysely; + /** Kysely database instance for metadata tables such as field/reference */ + readonly metaDb: Kysely; + /** Introspector for querying current database schema state */ readonly introspector: SchemaIntrospector; @@ -39,6 +42,13 @@ export interface SchemaRuleContext { * Defaults to 'update' when not specified. */ readonly mode?: 'delete' | 'update'; + + /** + * True when applying schema rules to brand-new empty physical tables. + * Rules may skip repair/backfill statements for existing rows while keeping + * structural DDL and metadata writes intact. + */ + readonly optimizeForEmptyTables?: boolean; } /** @@ -46,6 +56,7 @@ export interface SchemaRuleContext { */ export const createSchemaRuleContext = (params: { db: Kysely; + metaDb?: Kysely; introspector: SchemaIntrospector; schema: string | null; tableName: string; @@ -53,8 +64,10 @@ export const createSchemaRuleContext = (params: { field?: Field; table?: Table; mode?: 'delete' | 'update'; + optimizeForEmptyTables?: boolean; }): SchemaRuleContext => ({ db: params.db, + metaDb: params.metaDb ?? params.db, introspector: params.introspector, schema: params.schema, tableName: params.tableName, @@ -62,4 +75,5 @@ export const createSchemaRuleContext = (params: { field: params.field, table: params.table, mode: params.mode, + optimizeForEmptyTables: params.optimizeForEmptyTables, }); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/ISchemaRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/ISchemaRule.ts index 88d3f83fe1..7f119d5b12 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/ISchemaRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/ISchemaRule.ts @@ -1,5 +1,5 @@ import type { DomainError } from '@teable/v2-core'; -import type { CompiledQuery, QueryExecutorProvider } from 'kysely'; +import type { CompiledQuery, QueryExecutorProvider, QueryResult } from 'kysely'; import type { Result } from 'neverthrow'; import type { SchemaRuleContext } from '../context/SchemaRuleContext'; @@ -10,12 +10,23 @@ import type { SchemaRuleContext } from '../context/SchemaRuleContext'; */ export type TableSchemaStatementScope = 'data' | 'meta'; +export type TableSchemaStatementExecutorProvider = QueryExecutorProvider & { + executeQuery(compiledQuery: CompiledQuery): Promise>; +}; + +export type TableSchemaStatementExecutionContext = { + readonly scopedDb: TableSchemaStatementExecutorProvider; + readonly dataDb: TableSchemaStatementExecutorProvider; + readonly metaDb: TableSchemaStatementExecutorProvider; +}; + export type TableSchemaStatementCompiler = { compile: (executorProvider: QueryExecutorProvider) => CompiledQuery; }; export type TableSchemaStatementBuilder = TableSchemaStatementCompiler & { readonly scope: TableSchemaStatementScope; + readonly execute?: (ctx: TableSchemaStatementExecutionContext) => Promise; }; export type SchemaRuleI18nValue = string | number | boolean; @@ -127,6 +138,12 @@ export interface ISchemaRule { */ readonly required: boolean; + /** + * Which storage plane owns the relations used by isValid/manualRepair. + * Defaults to 'data'. Rules that inspect metadata tables such as field/reference use 'meta'. + */ + readonly validationScope?: TableSchemaStatementScope; + /** * Whether this rule can be auto-repaired by replaying `up()`. * Rules that need human intervention should mark themselves as `manual`. diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/SchemaStatementAccessPolicy.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/SchemaStatementAccessPolicy.ts index da127a12d1..6625b4a5af 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/SchemaStatementAccessPolicy.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/core/SchemaStatementAccessPolicy.ts @@ -17,6 +17,8 @@ const metaRelations = [ 'schema_operation', 'data_db_connection', 'space_data_db_binding', + 'users', + 'collaborator', ] as const; const dataRelations = [ diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldMetaRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldMetaRule.ts index 6421404ebb..9649b9a4d0 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldMetaRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldMetaRule.ts @@ -21,6 +21,7 @@ export class FieldMetaRule implements ISchemaRule { readonly description: string; readonly dependencies: ReadonlyArray; readonly required = true; + readonly validationScope = 'meta'; /** * @param field - The field to update metadata for @@ -129,6 +130,12 @@ export class FieldMetaRule implements ISchemaRule { } up(ctx: SchemaRuleContext): Result, DomainError> { + if (ctx.optimizeForEmptyTables) { + // Duplicate/import persists the field aggregate metadata up front. The repair statement is + // only needed when reconciling existing metadata drift. + return ok([]); + } + const fieldId = this.field.id().toString(); const patch = JSON.stringify(this.meta); const updateMeta = ctx.db diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldSchemaRulesFactory.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldSchemaRulesFactory.ts index 9cee7046d2..7308052a4b 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldSchemaRulesFactory.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FieldSchemaRulesFactory.ts @@ -59,6 +59,8 @@ export interface FieldSchemaRulesContext { tableId: string; /** Known logical table ID to physical table location mapping for batch schema creation. */ tableLocationsById?: ReadonlyMap; + /** Whether the target physical tables are newly-created and empty. */ + optimizeForEmptyTables?: boolean; } /** diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FkColumnRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FkColumnRule.ts index 58903c4552..4c2ca4038b 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FkColumnRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/FkColumnRule.ts @@ -66,6 +66,19 @@ export class FkColumnRule implements ISchemaRule { return this.targetTable ?? { schema: ctx.schema, tableName: ctx.tableName }; } + createTableColumnForTarget( + targetTable: TableIdentifier + ): { columnName: string; dataType: 'text' } | undefined { + const target = this.targetTable ?? targetTable; + if ((target.schema ?? null) !== (targetTable.schema ?? null)) { + return undefined; + } + if (target.tableName !== targetTable.tableName) { + return undefined; + } + return { columnName: this.columnName, dataType: 'text' }; + } + async isValid(ctx: SchemaRuleContext): Promise> { const columnName = this.columnName; const target = this.getTargetTable(ctx); @@ -119,7 +132,7 @@ export class FkColumnRule implements ISchemaRule { const isCurrentContextTarget = (target.schema ?? null) === (ctx.schema ?? null) && target.tableName === ctx.tableName; - if (ctx.field) { + if (!ctx.optimizeForEmptyTables && ctx.field) { const linkValueColumnName = yield* resolveColumnName(ctx.field); if (linkValueColumnName !== columnName) { if (isCurrentContextTarget) { diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ForeignKeyRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ForeignKeyRule.ts index 371be6396d..017cb6f801 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ForeignKeyRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ForeignKeyRule.ts @@ -4,6 +4,7 @@ import type { Result } from 'neverthrow'; import { sql } from 'kysely'; import { z } from 'zod'; +import { PostgresSchemaIntrospector } from '../context/PostgresSchemaIntrospector'; import type { SchemaRuleContext } from '../context/SchemaRuleContext'; import type { ISchemaRule, @@ -148,7 +149,8 @@ export class ForeignKeyRule implements ISchemaRule { return ok(this.targetTable); } - const tableMetaExists = await ctx.introspector.tableExists('public', 'table_meta'); + const metaIntrospector = new PostgresSchemaIntrospector(ctx.metaDb); + const tableMetaExists = await metaIntrospector.tableExists('public', 'table_meta'); if (tableMetaExists.isErr()) { return err(tableMetaExists.error); } @@ -163,7 +165,7 @@ export class ForeignKeyRule implements ISchemaRule { WHERE id = ${this.targetTableMetaId} AND deleted_time IS NULL LIMIT 1 - `.execute(ctx.db); + `.execute(ctx.metaDb); const dbTableName = result.rows[0]?.db_table_name; if (!dbTableName) { diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/GeneratedColumnRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/GeneratedColumnRule.ts index 6ce9c8ebc3..384c07e806 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/GeneratedColumnRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/GeneratedColumnRule.ts @@ -3,7 +3,10 @@ import { sql } from 'kysely'; import { ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; -import { resolveColumnName } from '../../visitors/PostgresTableSchemaFieldColumn'; +import { + resolveColumnName, + type TableColumnDataType, +} from '../../visitors/PostgresTableSchemaFieldColumn'; import type { SchemaRuleContext } from '../context/SchemaRuleContext'; import type { ISchemaRule, @@ -83,6 +86,12 @@ export class GeneratedColumnRule implements ISchemaRule { return new GeneratedColumnRule(field, '__auto_number', 'double precision'); } + createTableColumnType(): TableColumnDataType { + return sql`${sql.raw(this.columnType)} generated always as (${sql.ref( + this.sourceColumn + )}) stored`; + } + async isValid(ctx: SchemaRuleContext): Promise> { return safeTry(async function* () { const columnName = yield* resolveColumnName(ctx.field); @@ -130,7 +139,9 @@ export class GeneratedColumnRule implements ISchemaRule { return safeTry, DomainError>(function* () { const columnName = yield* resolveColumnName(ctx.field); - const definition = sql`${sql.raw(columnType)} generated always as (${sql.ref(sourceColumn)}) stored`; + const definition = sql`${sql.raw(columnType)} generated always as (${sql.ref( + sourceColumn + )}) stored`; const table: TableIdentifier = { schema: ctx.schema, tableName: ctx.tableName }; // If the column exists but is not generated, Postgres can't "ALTER" it into a generated column. // Drop+recreate is safe because generated columns are always derived from source columns. diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/JunctionTableRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/JunctionTableRule.ts index bbe3660973..39f1bd84d9 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/JunctionTableRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/JunctionTableRule.ts @@ -5,6 +5,7 @@ import type { Result } from 'neverthrow'; import { z } from 'zod'; import { resolveColumnName } from '../../visitors/PostgresTableSchemaFieldColumn'; +import { PostgresSchemaIntrospector } from '../context/PostgresSchemaIntrospector'; import type { SchemaRuleContext } from '../context/SchemaRuleContext'; import type { ISchemaRule, @@ -381,56 +382,60 @@ export class JunctionTableExistsRule implements ISchemaRule { } statements.push(dataStatement(createTableBuilder)); - // Also repair partially-created junction tables by adding any missing columns. - statements.push( - dataStatement( - schemaBuilder - .alterTable(config.junctionTable.tableName) - .addColumn('__id', 'serial', (col) => col.ifNotExists()) - ) - ); - statements.push( - dataStatement( - schemaBuilder - .alterTable(config.junctionTable.tableName) - .addColumn(config.selfKeyName, 'text', (col) => col.ifNotExists()) - ) - ); - statements.push( - dataStatement( - schemaBuilder - .alterTable(config.junctionTable.tableName) - .addColumn(config.foreignKeyName, 'text', (col) => col.ifNotExists()) - ) - ); - - if (config.orderColumnName) { + if (!ctx.optimizeForEmptyTables) { + // Also repair partially-created junction tables by adding any missing columns. + statements.push( + dataStatement( + schemaBuilder + .alterTable(config.junctionTable.tableName) + .addColumn('__id', 'serial', (col) => col.ifNotExists()) + ) + ); statements.push( dataStatement( schemaBuilder .alterTable(config.junctionTable.tableName) - .addColumn(config.orderColumnName, 'double precision', (col) => col.ifNotExists()) + .addColumn(config.selfKeyName, 'text', (col) => col.ifNotExists()) ) ); + statements.push( + dataStatement( + schemaBuilder + .alterTable(config.junctionTable.tableName) + .addColumn(config.foreignKeyName, 'text', (col) => col.ifNotExists()) + ) + ); + + if (config.orderColumnName) { + statements.push( + dataStatement( + schemaBuilder + .alterTable(config.junctionTable.tableName) + .addColumn(config.orderColumnName, 'double precision', (col) => col.ifNotExists()) + ) + ); + } } - const sourceLinkValueColumnName = yield* resolveColumnName(self.field); - const sameColumnLinkFieldCount = - ctx.table?.getFields().filter((field) => { - const dbFieldName = field.dbFieldName().andThen((name) => name.value()); - return dbFieldName.isOk() && dbFieldName.value === sourceLinkValueColumnName; - }).length ?? 1; - statements.push( - backfillJunctionTableFromLinkValueStatement({ - sourceTable: config.sourceTable, - sourceLinkValueColumnName, - junctionTable: config.junctionTable, - selfKeyName: config.selfKeyName, - foreignKeyName: config.foreignKeyName, - orderColumnName: config.orderColumnName, - skipBackfill: sameColumnLinkFieldCount > 1, - }) - ); + if (!ctx.optimizeForEmptyTables) { + const sourceLinkValueColumnName = yield* resolveColumnName(self.field); + const sameColumnLinkFieldCount = + ctx.table?.getFields().filter((field) => { + const dbFieldName = field.dbFieldName().andThen((name) => name.value()); + return dbFieldName.isOk() && dbFieldName.value === sourceLinkValueColumnName; + }).length ?? 1; + statements.push( + backfillJunctionTableFromLinkValueStatement({ + sourceTable: config.sourceTable, + sourceLinkValueColumnName, + junctionTable: config.junctionTable, + selfKeyName: config.selfKeyName, + foreignKeyName: config.foreignKeyName, + orderColumnName: config.orderColumnName, + skipBackfill: sameColumnLinkFieldCount > 1, + }) + ); + } return ok(statements); }); @@ -707,7 +712,8 @@ export class JunctionTableForeignKeyRule implements ISchemaRule { return ok(this.targetTable); } - const tableMetaExists = await ctx.introspector.tableExists('public', 'table_meta'); + const metaIntrospector = new PostgresSchemaIntrospector(ctx.metaDb); + const tableMetaExists = await metaIntrospector.tableExists('public', 'table_meta'); if (tableMetaExists.isErr()) { return err(tableMetaExists.error); } @@ -722,7 +728,7 @@ export class JunctionTableForeignKeyRule implements ISchemaRule { WHERE id = ${this.targetTableMetaId} AND deleted_time IS NULL LIMIT 1 - `.execute(ctx.db); + `.execute(ctx.metaDb); const dbTableName = result.rows[0]?.db_table_name; if (!dbTableName) { diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/LinkSymmetricFieldRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/LinkSymmetricFieldRule.ts index 78dd39173d..dd26e8e7ab 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/LinkSymmetricFieldRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/LinkSymmetricFieldRule.ts @@ -33,6 +33,7 @@ export class LinkSymmetricFieldRule implements ISchemaRule { readonly dependencies: ReadonlyArray = []; readonly required = true; readonly repairMode = 'manual' as const; + readonly validationScope = 'meta'; private readonly manualRepairSchema = withManualRepairFormMeta( z.object({ diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/OrderColumnRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/OrderColumnRule.ts index 39f9407b21..b370018ece 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/OrderColumnRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/OrderColumnRule.ts @@ -56,6 +56,18 @@ export class OrderColumnRule implements ISchemaRule { return new OrderColumnRule(field, columnName, targetTable, parent); } + createTableColumnForTarget( + targetTable: TableIdentifier + ): { columnName: string; dataType: 'double precision' } | undefined { + if ((this.targetTable.schema ?? null) !== (targetTable.schema ?? null)) { + return undefined; + } + if (this.targetTable.tableName !== targetTable.tableName) { + return undefined; + } + return { columnName: this.columnName, dataType: 'double precision' }; + } + async isValid(ctx: SchemaRuleContext): Promise> { const columnName = this.columnName; const targetTable = this.targetTable; diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ReferenceRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ReferenceRule.ts index 17bfde53df..2e8a63df38 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ReferenceRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/ReferenceRule.ts @@ -29,6 +29,7 @@ export class ReferenceRule implements ISchemaRule { readonly description: string; readonly dependencies: ReadonlyArray = []; readonly required: boolean; + readonly validationScope = 'meta'; /** * @param field - The field that has the dependencies diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SchemaRules.pglite.spec.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SchemaRules.pglite.spec.ts index 8564881b64..2ec713083c 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SchemaRules.pglite.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SchemaRules.pglite.spec.ts @@ -621,6 +621,7 @@ describe('Schema Rules Unit Tests with PGlite', () => { const createContext = (tableName: string, field: Field): SchemaRuleContext => ({ db, + metaDb: db, introspector, schema: TEST_SCHEMA, tableName, @@ -2641,6 +2642,29 @@ describe('Schema Rules Unit Tests with PGlite', () => { await sql`DELETE FROM field WHERE id = ${field.id().toString()}`.execute(db); } }); + + it('should skip metadata repair statements when optimizing brand-new empty tables', () => { + const fieldResult = createRealField('fmr006', 'Link Empty Table', 'link_col'); + const field = fieldResult._unsafeUnwrap(); + + const rule = FieldMetaRule.forOrderColumn(field); + const ctx = createContext(TABLE_NAME, field); + + expect( + rule + .up(ctx) + ._unsafeUnwrap() + .map((statement) => statement.scope) + ).toEqual(['meta']); + expect( + rule + .up({ + ...ctx, + optimizeForEmptyTables: true, + }) + ._unsafeUnwrap() + ).toEqual([]); + }); }); describe('SelectOptionsMetaRule', () => { @@ -2968,6 +2992,32 @@ describe('Schema Rules Unit Tests with PGlite', () => { expect(repairHint._unsafeUnwrap()?.description?.fallback).toContain('display'); }); + it('should skip repair statements when optimizing brand-new empty tables', () => { + const field = createRealSingleSelectField({ + id: 'som009', + name: 'Status Empty Table', + dbFieldName: 'status_col', + choices: expectedChoices, + })._unsafeUnwrap(); + const rule = new SelectOptionsMetaRule(field); + const ctx = createContext(TABLE_NAME, field); + + expect( + rule + .up(ctx) + ._unsafeUnwrap() + .map((statement) => statement.scope) + ).toEqual(['data', 'meta']); + expect( + rule + .up({ + ...ctx, + optimizeForEmptyTables: true, + }) + ._unsafeUnwrap() + ).toEqual([]); + }); + it('should register the select options rule for both single and multiple select fields', () => { const singleField = createRealSingleSelectField({ id: 'som003', diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SelectOptionsMetaRule.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SelectOptionsMetaRule.ts index a158648eaa..ca28361cf3 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SelectOptionsMetaRule.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/field/SelectOptionsMetaRule.ts @@ -53,6 +53,7 @@ export class SelectOptionsMetaRule implements ISchemaRule { readonly description: string; readonly dependencies: ReadonlyArray; readonly required = true; + readonly validationScope = 'meta'; constructor( private readonly field: SelectField, @@ -188,6 +189,12 @@ export class SelectOptionsMetaRule implements ISchemaRule { up(ctx: SchemaRuleContext): Result, DomainError> { const rule = this; return safeTry, DomainError>(function* () { + if (ctx.optimizeForEmptyTables) { + // Duplicate/import already persisted field.options from the field aggregate, and there + // are no rows yet whose stored select values need repair. + return ok([]); + } + const fieldId = rule.field.id().toString(); const columnName = yield* resolveColumnName(ctx.field); const patch = JSON.stringify({ choices: rule.expectedChoices() }); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/planner/SchemaRulePlanner.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/planner/SchemaRulePlanner.ts index cb69abcd5d..d00d132550 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/planner/SchemaRulePlanner.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/planner/SchemaRulePlanner.ts @@ -17,6 +17,7 @@ import { export interface SchemaRulePlannerParams { db: Kysely; + metaDb?: Kysely; introspector: SchemaIntrospector; schema: string | null; tableLocationsById?: ReadonlyMap; @@ -244,6 +245,7 @@ export class SchemaRulePlanner { } else { const systemCtx = createSchemaRuleContext({ db: this.params.db, + metaDb: this.params.metaDb, introspector: this.params.introspector, schema: tableLocation.schema, tableName: tableLocation.tableName, @@ -287,6 +289,7 @@ export class SchemaRulePlanner { const ctx = createSchemaRuleContext({ db: this.params.db, + metaDb: this.params.metaDb, introspector: this.params.introspector, schema: tableLocation.schema, tableName: tableLocation.tableName, diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/rules/repairer/SchemaRepairer.ts b/packages/v2/adapter-table-repository-postgres/src/schema/rules/repairer/SchemaRepairer.ts index 5762b2ac93..cef83b707a 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/rules/repairer/SchemaRepairer.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/rules/repairer/SchemaRepairer.ts @@ -1,7 +1,14 @@ +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; import type { Table } from '@teable/v2-core'; +import type { Kysely, Transaction } from 'kysely'; import { executeTableSchemaStatements } from '../../../shared/db'; -import type { SchemaRuleManualRepairValues, TableSchemaStatementBuilder } from '../core'; +import type { + ISchemaRule, + SchemaRuleManualRepairValues, + TableSchemaStatementBuilder, +} from '../core'; +import type { SchemaRuleContext } from '../context/SchemaRuleContext'; import { getRuleRepairHint } from '../core/RuleRepairMetadata'; import { createSchemaRulePlanner, @@ -28,11 +35,12 @@ export interface SchemaRepairOptions { } const compileRepairStatements = ( - db: SchemaRepairerParams['db'], + dataDb: SchemaRepairerParams['db'], + metaDb: SchemaRepairerParams['db'], statements: ReadonlyArray ): ReadonlyArray => statements.map((statement) => { - const compiled = statement.compile(db); + const compiled = statement.compile(statement.scope === 'meta' ? metaDb : dataDb); return { sql: compiled.sql, @@ -40,6 +48,45 @@ const compileRepairStatements = ( }; }); +const getRuleValidationCtx = (ctx: SchemaRuleContext, rule: ISchemaRule): SchemaRuleContext => + rule.validationScope === 'meta' + ? { + ...ctx, + db: ctx.metaDb, + } + : ctx; + +const executeScopedRepairStatements = async ( + dataDb: Kysely | Transaction, + metaDb: Kysely | Transaction, + statements: ReadonlyArray +): Promise => { + let currentScope: 'data' | 'meta' | undefined; + let batch: TableSchemaStatementBuilder[] = []; + + const flush = async () => { + if (!currentScope || batch.length === 0) { + return; + } + const targetDb = currentScope === 'meta' ? metaDb : dataDb; + await executeTableSchemaStatements(targetDb, batch, { + dataDb: dataDb as Kysely | Transaction, + metaDb: metaDb as Kysely | Transaction, + }); + batch = []; + }; + + for (const statement of statements) { + if (currentScope && currentScope !== statement.scope) { + await flush(); + } + currentScope = statement.scope; + batch.push(statement); + } + + await flush(); +}; + export class SchemaRepairer { constructor(private readonly params: SchemaRepairerParams) {} @@ -136,7 +183,8 @@ export class SchemaRepairer { continue; } - const validationResult = await rule.isValid(ctx); + const validationCtx = getRuleValidationCtx(ctx, rule); + const validationResult = await rule.isValid(validationCtx); if (validationResult.isErr()) { yield errorResult(pending, validationResult.error.message); repairedRules.set(rule.id, false); @@ -156,7 +204,7 @@ export class SchemaRepairer { extra: validation.extra, extraItems: validation.extraItems, }; - const repairResult = getRuleRepairHint(rule, ctx, validation); + const repairResult = getRuleRepairHint(rule, validationCtx, validation); const repair = repairResult.isOk() ? repairResult.value : undefined; const shouldUseManualRepair = rule.repairMode === 'manual' || @@ -181,9 +229,13 @@ export class SchemaRepairer { continue; } - const manualRepairResult = await rule.manualRepair(ctx, options.manualRepairValues, { - dryRun: options.dryRun, - }); + const manualRepairResult = await rule.manualRepair( + validationCtx, + options.manualRepairValues, + { + dryRun: options.dryRun, + } + ); if (manualRepairResult.isErr()) { yield { @@ -203,7 +255,7 @@ export class SchemaRepairer { continue; } - const revalidationResult = await rule.isValid(ctx); + const revalidationResult = await rule.isValid(validationCtx); if (revalidationResult.isErr()) { yield { ...errorResult(pending, revalidationResult.error.message, details), @@ -244,7 +296,7 @@ export class SchemaRepairer { continue; } - const statementsResult = rule.up(ctx); + const statementsResult = rule.up(validationCtx); if (statementsResult.isErr()) { yield { ...errorResult(pending, statementsResult.error.message, details), @@ -265,7 +317,7 @@ export class SchemaRepairer { } if (options?.dryRun) { - const compiledStatements = compileRepairStatements(ctx.db, statements); + const compiledStatements = compileRepairStatements(ctx.db, ctx.metaDb, statements); yield { ...successResult( pending, @@ -283,9 +335,9 @@ export class SchemaRepairer { continue; } - await executeTableSchemaStatements(ctx.db, statements); + await executeScopedRepairStatements(ctx.db, ctx.metaDb, statements); - const revalidationResult = await rule.isValid(ctx); + const revalidationResult = await rule.isValid(validationCtx); if (revalidationResult.isErr()) { yield { ...errorResult(pending, revalidationResult.error.message, { diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldTypeConversionVisitor.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldTypeConversionVisitor.ts index cd30cadda4..d62e548909 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldTypeConversionVisitor.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldTypeConversionVisitor.ts @@ -4,6 +4,7 @@ import { DateTimeFormatting, DEFAULT_TABLE_DATA_SAFETY_LIMITS, NumberFormatting, + SelectOptionId, domainError, } from '@teable/v2-core'; import { @@ -41,6 +42,7 @@ import type { Result } from 'neverthrow'; import { resolveUserAvatarUrlPrefix } from '../../shared/userAvatarUrl'; import type { TableSchemaStatementBuilder } from '../rules/core'; +import type { TableIdentifier } from '../rules/helpers'; import { PostgresTableSchemaFieldCreateVisitor } from './PostgresTableSchemaFieldCreateVisitor'; import { PostgresTableSchemaFieldDeleteVisitor } from './PostgresTableSchemaFieldDeleteVisitor'; @@ -52,6 +54,8 @@ export type FieldConversionParams = { dbFieldName: string; /** Field ID for updating field metadata (e.g., auto-generating select options) */ fieldId?: string; + tableLocationsById?: ReadonlyMap; + fieldsById?: ReadonlyMap; }; const createCompiledStatementBuilder = ( @@ -74,6 +78,409 @@ const SELECT_CHOICE_NAME_MAX_LENGTH = const selectChoiceNameLengthError = `Select choice name exceeds ${SELECT_CHOICE_NAME_MAX_LENGTH} characters`; +type SelectChoiceDto = { + id: string; + name: string; + color: string; +}; + +type FieldConversionFieldMetadata = { + dbFieldName: string; + tableId?: string; + options?: unknown; +}; + +type LinkMappingMetadata = { + lookupColumnName: string | null; + foreignTable: TableIdentifier | null; + symmetricColumnName?: string | null; +}; + +type TextUserMapping = { + lookupValue: string; + id: string; + title: string; + email: string | null; +}; + +type TableSchemaStatementExecuteContext = Parameters< + NonNullable +>[0]; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const tryParseJson = (value: string): unknown => { + try { + return JSON.parse(value); + } catch { + return undefined; + } +}; + +const normalizeChoice = (value: unknown): SelectChoiceDto | undefined => { + if (!isRecord(value)) { + return undefined; + } + + const { id, name, color } = value; + if (typeof id !== 'string' || typeof name !== 'string' || typeof color !== 'string') { + return undefined; + } + + return { id, name, color }; +}; + +const normalizeChoices = (options: unknown): SelectChoiceDto[] => { + const rawOptions = typeof options === 'string' ? tryParseJson(options) : options; + if (!isRecord(rawOptions) || !Array.isArray(rawOptions.choices)) { + return []; + } + + return rawOptions.choices + .map(normalizeChoice) + .filter((choice): choice is SelectChoiceDto => Boolean(choice)); +}; + +const buildMergedOptions = ( + currentOptions: unknown, + generatedChoices: ReadonlyArray +): Record => { + const parsedOptions = + typeof currentOptions === 'string' ? tryParseJson(currentOptions) : currentOptions; + const baseOptions = isRecord(parsedOptions) ? parsedOptions : {}; + return { + ...baseOptions, + choices: [...normalizeChoices(parsedOptions), ...generatedChoices], + }; +}; + +const sqlNullableLiteral = (value: string | null | undefined): string => + value == null ? 'NULL' : quoteLiteral(value); + +const parseTableIdentifier = (dbTableName: string): TableIdentifier => { + const [schema, ...rest] = dbTableName.split('.'); + if (rest.length === 0) { + return { schema: null, tableName: schema ?? dbTableName }; + } + + return { schema: schema ?? null, tableName: rest.join('.') }; +}; + +const buildTableIdentifierDeclarations = (table: TableIdentifier | null): string => { + const schema = table?.schema ?? (table ? 'public' : null); + return [ + ` foreign_schema text := ${sqlNullableLiteral(schema)};`, + ` foreign_name text := ${sqlNullableLiteral(table?.tableName)};`, + ].join('\n'); +}; + +const createCustomDataStatementBuilder = ( + db: Kysely, + previewSql: string, + resolveSql: (ctx: TableSchemaStatementExecuteContext) => Promise +): TableSchemaStatementBuilder => ({ + scope: 'data', + compile: () => sql.raw(previewSql).compile(db), + execute: async (ctx) => { + const sqlText = await resolveSql(ctx); + if (!sqlText) { + return; + } + await sql.raw(sqlText).execute(ctx.dataDb); + }, +}); + +const normalizeFieldOptions = (options: unknown): Record => { + const parsed = typeof options === 'string' ? tryParseJson(options) : options; + return isRecord(parsed) ? parsed : {}; +}; + +const getKnownFieldMetadata = ( + params: FieldConversionParams, + fieldId: string +): FieldConversionFieldMetadata | undefined => params.fieldsById?.get(fieldId); + +const getKnownFieldDbName = (params: FieldConversionParams, fieldId: string): string | null => + getKnownFieldMetadata(params, fieldId)?.dbFieldName ?? null; + +const getKnownTableIdentifier = ( + params: FieldConversionParams, + tableId: string +): TableIdentifier | null => params.tableLocationsById?.get(tableId) ?? null; + +const getPreviewFieldDbName = (params: FieldConversionParams, fieldId: string): string => + getKnownFieldDbName(params, fieldId) ?? fieldId; + +const getPreviewTableIdentifier = ( + params: FieldConversionParams, + tableId: string +): TableIdentifier => + getKnownTableIdentifier(params, tableId) ?? { schema: params.schema, tableName: tableId }; + +const getKnownSymmetricFieldDbName = ( + params: FieldConversionParams, + foreignTableId: string, + currentFieldId: string, + explicitSymmetricFieldId?: string +): string | null => { + if (explicitSymmetricFieldId) { + const explicitField = getKnownFieldMetadata(params, explicitSymmetricFieldId); + if (explicitField?.dbFieldName) { + return explicitField.dbFieldName; + } + } + + for (const [fieldId, field] of params.fieldsById ?? + new Map()) { + if (explicitSymmetricFieldId && fieldId === explicitSymmetricFieldId) { + return field.dbFieldName; + } + if (field.tableId !== foreignTableId) { + continue; + } + const options = normalizeFieldOptions(field.options); + if (options.symmetricFieldId === currentFieldId) { + return field.dbFieldName; + } + } + + return null; +}; + +const getPreviewSymmetricFieldDbName = ( + params: FieldConversionParams, + foreignTableId: string, + currentFieldId: string, + explicitSymmetricFieldId?: string +): string | null => + getKnownSymmetricFieldDbName(params, foreignTableId, currentFieldId, explicitSymmetricFieldId) ?? + explicitSymmetricFieldId ?? + null; + +const getPreviewLinkMappingMetadata = ( + params: FieldConversionParams, + linkField: LinkField, + options?: { includeSymmetric?: boolean } +): LinkMappingMetadata => { + const foreignTableId = linkField.foreignTableId().toString(); + const currentFieldId = linkField.id().toString(); + return { + lookupColumnName: getPreviewFieldDbName(params, linkField.lookupFieldId().toString()), + foreignTable: getPreviewTableIdentifier(params, foreignTableId), + ...(options?.includeSymmetric + ? { + symmetricColumnName: getPreviewSymmetricFieldDbName( + params, + foreignTableId, + currentFieldId, + linkField.symmetricFieldId()?.toString() + ), + } + : {}), + }; +}; + +const fetchFieldDbName = async ( + ctx: TableSchemaStatementExecuteContext, + fieldId: string +): Promise => { + const rows = await sql<{ db_field_name: string | null }>` + SELECT db_field_name + FROM field + WHERE id = ${fieldId} AND deleted_time IS NULL + LIMIT 1 + `.execute(ctx.metaDb); + + return rows.rows[0]?.db_field_name ?? null; +}; + +const fetchTableIdentifier = async ( + ctx: TableSchemaStatementExecuteContext, + tableId: string +): Promise => { + const rows = await sql<{ db_table_name: string | null }>` + SELECT db_table_name + FROM table_meta + WHERE id = ${tableId} AND deleted_time IS NULL + LIMIT 1 + `.execute(ctx.metaDb); + const dbTableName = rows.rows[0]?.db_table_name; + return dbTableName ? parseTableIdentifier(dbTableName) : null; +}; + +const fetchSymmetricFieldDbName = async ( + ctx: TableSchemaStatementExecuteContext, + foreignTableId: string, + currentFieldId: string, + explicitSymmetricFieldId?: string +): Promise => { + const explicitClause = explicitSymmetricFieldId + ? ` OR id = ${quoteLiteral(explicitSymmetricFieldId)}` + : ''; + const rows = await sql<{ db_field_name: string | null }>` + SELECT db_field_name + FROM field + WHERE table_id = ${foreignTableId} + AND type = 'link' + AND deleted_time IS NULL + AND ( + options::jsonb->>'symmetricFieldId' = ${currentFieldId} + ${sql.raw(explicitClause)} + ) + ORDER BY id + LIMIT 1 + `.execute(ctx.metaDb); + + return rows.rows[0]?.db_field_name ?? null; +}; + +const fetchLinkMappingMetadata = async ( + ctx: TableSchemaStatementExecuteContext, + linkField: LinkField, + options?: { includeSymmetric?: boolean } +): Promise => { + const foreignTableId = linkField.foreignTableId().toString(); + const currentFieldId = linkField.id().toString(); + const [lookupColumnName, foreignTable, symmetricColumnName] = await Promise.all([ + fetchFieldDbName(ctx, linkField.lookupFieldId().toString()), + fetchTableIdentifier(ctx, foreignTableId), + options?.includeSymmetric + ? fetchSymmetricFieldDbName( + ctx, + foreignTableId, + currentFieldId, + linkField.symmetricFieldId()?.toString() + ) + : Promise.resolve(undefined), + ]); + + return { + lookupColumnName, + foreignTable, + ...(options?.includeSymmetric ? { symmetricColumnName } : {}), + }; +}; + +const buildTextUserPartsSql = (fullTableName: string, col: string): string => ` + SELECT DISTINCT parts.uid + FROM ${fullTableName} AS t + CROSS JOIN LATERAL regexp_split_to_table(t.${col}, ${quoteLiteral(',')}) + WITH ORDINALITY AS raw_parts(part, part_idx) + CROSS JOIN LATERAL (SELECT trim(raw_parts.part) AS uid) AS parts + WHERE t.${col} IS NOT NULL AND parts.uid <> ''`; + +const fetchTextUserMappings = async ( + ctx: TableSchemaStatementExecuteContext, + fullTableName: string, + col: string +): Promise> => { + const identifierRows = await sql<{ uid: string }>` + ${sql.raw(buildTextUserPartsSql(fullTableName, col))} + `.execute(ctx.dataDb); + const identifiers = [ + ...new Set(identifierRows.rows.map((row) => row.uid).filter((uid) => uid.length > 0)), + ]; + if (identifiers.length === 0) { + return []; + } + + const identifierValues = identifiers + .map((identifier) => `(${quoteLiteral(identifier)})`) + .join(', '); + const userRows = await sql<{ id: string; name: string | null; email: string | null }>` + WITH input(uid) AS (VALUES ${sql.raw(identifierValues)}) + SELECT DISTINCT users.id, users.name, users.email + FROM users + JOIN input ON users.id = input.uid OR users.email = input.uid OR users.name = input.uid + ORDER BY users.id + `.execute(ctx.metaDb); + + const byId = new Map>(); + const byEmail = new Map>(); + const byName = new Map>(); + + for (const row of userRows.rows) { + const user = { + id: row.id, + title: row.name ?? row.id, + email: row.email ?? null, + }; + if (!byId.has(row.id)) { + byId.set(row.id, user); + } + if (row.email && !byEmail.has(row.email)) { + byEmail.set(row.email, user); + } + if (row.name && !byName.has(row.name)) { + byName.set(row.name, user); + } + } + + return identifiers + .map((identifier) => { + const user = byId.get(identifier) ?? byEmail.get(identifier) ?? byName.get(identifier); + return user ? { lookupValue: identifier, ...user } : null; + }) + .filter((mapping): mapping is TextUserMapping => Boolean(mapping)); +}; + +const buildTextToUserTransformSql = ( + fullTableName: string, + col: string, + isMultiple: boolean, + mappings: ReadonlyArray +): string | null => { + if (mappings.length === 0) { + return null; + } + + const avatarPrefix = resolveUserAvatarUrlPrefix(); + const mappingValues = mappings + .map( + (mapping) => + `(${quoteLiteral(mapping.lookupValue)}, ${quoteLiteral(mapping.id)}, ${quoteLiteral( + mapping.title + )}, ${sqlNullableLiteral(mapping.email)})` + ) + .join(', '); + const userJsonBuild = (alias: string) => + `jsonb_build_object('id', ${alias}.id, 'title', ${alias}.title, 'email', ${alias}.email, 'avatarUrl', ${quoteLiteral( + avatarPrefix + )} || ${alias}.id)`; + const aggregationSql = isMultiple + ? `SELECT rid, jsonb_agg(${userJsonBuild('matched_users')} ORDER BY part_idx)::text AS user_json + FROM matched_users + GROUP BY rid` + : `SELECT DISTINCT ON (rid) rid, ${userJsonBuild('matched_users')}::text AS user_json + FROM matched_users + ORDER BY rid, part_idx`; + + return `WITH user_mapping(uid, id, title, email) AS ( + VALUES ${mappingValues} + ), + text_parts AS ( + SELECT t.__id AS rid, parts.uid, raw_parts.part_idx::integer AS part_idx + FROM ${fullTableName} AS t + CROSS JOIN LATERAL regexp_split_to_table(t.${col}, ${quoteLiteral(',')}) + WITH ORDINALITY AS raw_parts(part, part_idx) + CROSS JOIN LATERAL (SELECT trim(raw_parts.part) AS uid) AS parts + WHERE t.${col} IS NOT NULL AND parts.uid <> '' + ), + matched_users AS ( + SELECT p.rid, p.part_idx, u.id, u.title, u.email + FROM text_parts AS p + JOIN user_mapping AS u ON u.uid = p.uid + ), + aggregated AS ( + ${aggregationSql} + ) + UPDATE ${fullTableName} AS t + SET ${col} = aggregated.user_json + FROM aggregated + WHERE t.__id = aggregated.rid`; +}; + const buildSelectOptionsFromValuesStatement = ( params: FieldConversionParams, distinctValuesSql: string @@ -83,69 +490,66 @@ const buildSelectOptionsFromValuesStatement = ( } const colors = SELECT_OPTION_COLORS; - const mergeSql = ` + const previewSql = ` WITH distinct_values AS ( ${distinctValuesSql} ), -existing_choices AS ( - SELECT jsonb_array_elements(COALESCE(options::jsonb->'choices', '[]'::jsonb)) AS choice - FROM field - WHERE id = ${quoteLiteral(params.fieldId)} -), -existing_names AS ( - SELECT choice->>'name' AS name FROM existing_choices -), oversized_values AS ( SELECT name FROM distinct_values WHERE char_length(name) > ${SELECT_CHOICE_NAME_MAX_LENGTH} -), -oversized_guard AS ( - SELECT CASE - WHEN EXISTS (SELECT 1 FROM oversized_values) - -- Keep the invalid cast runtime-bound; Postgres folds constant casts before CASE short-circuits. - THEN CAST( - ${quoteLiteral(selectChoiceNameLengthError)} - || COALESCE((SELECT '' FROM oversized_values LIMIT 1), '') - AS integer - ) - ELSE 1 - END AS ok -), -new_values AS ( - SELECT name, ROW_NUMBER() OVER () AS rn - FROM distinct_values - CROSS JOIN oversized_guard - WHERE name NOT IN (SELECT name FROM existing_names WHERE name IS NOT NULL) - AND oversized_guard.ok = 1 -), -new_choices AS ( - SELECT jsonb_agg( - jsonb_build_object( - 'id', 'cho' || substr(md5(random()::text || clock_timestamp()::text), 1, 16), - 'name', name, - 'color', (ARRAY[${colors.map((color) => quoteLiteral(color)).join(', ')}])[(rn % ${colors.length}) + 1] - ) - ) AS choices - FROM new_values -), -merged_choices AS ( - SELECT COALESCE( - (SELECT jsonb_agg(choice) FROM existing_choices), - '[]'::jsonb - ) || COALESCE( - (SELECT choices FROM new_choices), - '[]'::jsonb - ) AS choices -) -UPDATE field -SET options = jsonb_set( - COALESCE(options::jsonb, '{}'::jsonb), - '{choices}', - (SELECT choices FROM merged_choices) ) -WHERE id = ${quoteLiteral(params.fieldId)};`; +SELECT name FROM distinct_values +WHERE name IS NOT NULL AND name <> '' + AND NOT EXISTS (SELECT 1 FROM oversized_values);`; + + return { + scope: 'meta', + compile: () => sql.raw(previewSql).compile(params.db), + execute: async ({ dataDb, metaDb }) => { + const distinctRows = await sql<{ name: string | null }>` + WITH distinct_values AS (${sql.raw(distinctValuesSql)}) + SELECT DISTINCT name + FROM distinct_values + WHERE name IS NOT NULL AND name <> '' + `.execute(dataDb); + + const distinctNames = distinctRows.rows + .map((row) => row.name) + .filter((name): name is string => Boolean(name)); + const oversizedName = distinctNames.find( + (name) => name.length > SELECT_CHOICE_NAME_MAX_LENGTH + ); + if (oversizedName) { + throw new Error(`${selectChoiceNameLengthError}${oversizedName}`); + } - return createCompiledStatementBuilder(params.db, mergeSql); + const currentRows = await sql<{ options: unknown }>` + SELECT options + FROM field + WHERE id = ${params.fieldId} + LIMIT 1 + `.execute(metaDb); + const currentOptions = currentRows.rows[0]?.options; + const existingNames = new Set(normalizeChoices(currentOptions).map((choice) => choice.name)); + const newNames = distinctNames.filter((name) => !existingNames.has(name)); + if (!newNames.length) { + return; + } + + const generatedChoices = newNames.map((name, index) => ({ + id: SelectOptionId.generate()._unsafeUnwrap().toString(), + name, + color: colors[index % colors.length], + })); + const mergedOptions = buildMergedOptions(currentOptions, generatedChoices); + + await sql` + UPDATE field + SET options = ${JSON.stringify(mergedOptions)} + WHERE id = ${params.fieldId} + `.execute(metaDb); + }, + }; }; const buildScalarToLinkMigrationStatements = ( @@ -186,8 +590,7 @@ const buildScalarToLinkMigrationStatements = ( let sourceFkColumnName: string | null = null; let sourceOrderColumnName: string | null = null; let symmetricDeclareSql = ''; - let symmetricResolveSql = ''; - let symmetricBackfillSql = ''; + let buildSymmetricBackfillSql: (metadata: LinkMappingMetadata) => string = () => ''; if (relationship === 'manyOne' || relationship === 'oneOne') { const fkColumnName = yield* newLinkField.foreignKeyNameString(); @@ -279,25 +682,8 @@ const buildScalarToLinkMigrationStatements = ( domainError.unexpected({ message: 'Missing source FK column for link mapping' }) ); } - const foreignTableId = newLinkField.foreignTableId().toString(); - const currentFieldId = newLinkField.id().toString(); - const explicitSymmetricFieldId = newLinkField.symmetricFieldId()?.toString(); - const explicitSymmetricClause = explicitSymmetricFieldId - ? ` OR id = ${quoteLiteral(explicitSymmetricFieldId)}` - : ''; symmetricDeclareSql = ' symmetric_col text;'; - symmetricResolveSql = ` - SELECT db_field_name INTO symmetric_col - FROM field - WHERE table_id = ${quoteLiteral(foreignTableId)} - AND type = 'link' - AND deleted_time IS NULL - AND ( - options::jsonb->>'symmetricFieldId' = ${quoteLiteral(currentFieldId)}${explicitSymmetricClause} - ) - ORDER BY id - LIMIT 1;`; const sourceQualifiedName = `${quoteIdent(sourceSchema)}.${quoteIdent(sourceTableName)}`; const sourceFkIdent = quoteIdent(sourceFkColumnName); @@ -306,7 +692,8 @@ const buildScalarToLinkMigrationStatements = ( : '"__id" ASC'; if (relationship === 'manyOne') { - symmetricBackfillSql = ` + buildSymmetricBackfillSql = (metadata) => ` + symmetric_col := ${sqlNullableLiteral(metadata.symmetricColumnName)}; IF symmetric_col IS NOT NULL THEN EXECUTE format( 'UPDATE %I.%I AS f @@ -325,7 +712,8 @@ const buildScalarToLinkMigrationStatements = ( ); END IF;`; } else { - symmetricBackfillSql = ` + buildSymmetricBackfillSql = (metadata) => ` + symmetric_col := ${sqlNullableLiteral(metadata.symmetricColumnName)}; IF symmetric_col IS NOT NULL THEN EXECUTE format( 'UPDATE %I.%I AS f @@ -346,27 +734,10 @@ const buildScalarToLinkMigrationStatements = ( END IF;`; } } else if (!isOneWay && relationship === 'oneMany') { - const foreignTableId = newLinkField.foreignTableId().toString(); - const currentFieldId = newLinkField.id().toString(); - const explicitSymmetricFieldId = newLinkField.symmetricFieldId()?.toString(); - const explicitSymmetricClause = explicitSymmetricFieldId - ? ` OR id = ${quoteLiteral(explicitSymmetricFieldId)}` - : ''; - symmetricDeclareSql = ' symmetric_col text;'; - symmetricResolveSql = ` - SELECT db_field_name INTO symmetric_col - FROM field - WHERE table_id = ${quoteLiteral(foreignTableId)} - AND type = 'link' - AND deleted_time IS NULL - AND ( - options::jsonb->>'symmetricFieldId' = ${quoteLiteral(currentFieldId)}${explicitSymmetricClause} - ) - ORDER BY id - LIMIT 1;`; - symmetricBackfillSql = ` + buildSymmetricBackfillSql = (metadata) => ` + symmetric_col := ${sqlNullableLiteral(metadata.symmetricColumnName)}; IF symmetric_col IS NOT NULL THEN EXECUTE format( 'WITH candidates AS ( @@ -403,39 +774,23 @@ const buildScalarToLinkMigrationStatements = ( END IF;`; } - const mapFkSql = ` + const includeSymmetric = symmetricDeclareSql.length > 0; + const buildMapFkSql = (metadata: LinkMappingMetadata): string | null => { + if (!metadata.lookupColumnName || !metadata.foreignTable) { + return null; + } + + return ` DO $v2_link_map$ DECLARE - lookup_col text; - foreign_tbl text; - foreign_schema text; - foreign_name text; + lookup_col text := ${quoteLiteral(metadata.lookupColumnName)}; +${buildTableIdentifierDeclarations(metadata.foreignTable)} ${symmetricDeclareSql} BEGIN - SELECT db_field_name INTO lookup_col - FROM field - WHERE id = ${quoteLiteral(newLinkField.lookupFieldId().toString())} AND deleted_time IS NULL - LIMIT 1; - - SELECT db_table_name INTO foreign_tbl - FROM table_meta - WHERE id = ${quoteLiteral(newLinkField.foreignTableId().toString())} AND deleted_time IS NULL - LIMIT 1; - - IF lookup_col IS NULL OR foreign_tbl IS NULL THEN + IF lookup_col IS NULL OR foreign_schema IS NULL OR foreign_name IS NULL THEN RETURN; END IF; -${symmetricResolveSql} - - IF strpos(foreign_tbl, '.') > 0 THEN - foreign_schema := split_part(foreign_tbl, '.', 1); - foreign_name := split_part(foreign_tbl, '.', 2); - ELSE - foreign_schema := 'public'; - foreign_name := foreign_tbl; - END IF; - ${ensureForeignFkSql} ${ensureForeignOrderSql} @@ -463,16 +818,28 @@ ${symmetricResolveSql} ${quoteLiteral(tmpColumnName)}${applyFormatArgs.length > 0 ? `, ${applyFormatArgs.join(', ')}` : ''} ); -${symmetricBackfillSql} +${buildSymmetricBackfillSql(metadata)} END $v2_link_map$;`; + }; + const previewMetadata = getPreviewLinkMappingMetadata(params, newLinkField, { + includeSymmetric, + }); + const previewMapFkSql = + buildMapFkSql(previewMetadata) ?? `DO $v2_link_map$ BEGIN RETURN; END $v2_link_map$;`; const dropTmpSql = `ALTER TABLE ${quoteIdent(sourceSchema)}.${quoteIdent(sourceTableName)} DROP COLUMN IF EXISTS ${quoteIdent(tmpColumnName)}`; return ok([ createCompiledStatementBuilder(params.db, renameSql), ...dropStatements, ...createStatements, - createCompiledStatementBuilder(params.db, mapFkSql), + createCustomDataStatementBuilder(params.db, previewMapFkSql, async (ctx) => + buildMapFkSql( + await fetchLinkMappingMetadata(ctx, newLinkField, { + includeSymmetric, + }) + ) + ), createCompiledStatementBuilder(params.db, dropTmpSql), ]); }); @@ -897,36 +1264,21 @@ const buildLinkToTextMigrationStatements = ( const createStatements = yield* newField.accept(createVisitor); const renameSql = `ALTER TABLE ${quoteIdent(sourceSchema)}.${quoteIdent(sourceTableName)} RENAME COLUMN ${quoteIdent(dbFieldName)} TO ${quoteIdent(tmpColumnName)}`; - const mapTextSql = ` + const buildMapTextSql = (metadata: LinkMappingMetadata): string | null => { + if (!metadata.lookupColumnName || !metadata.foreignTable) { + return null; + } + + return ` DO $v2_link_to_text$ DECLARE - lookup_col text; - foreign_tbl text; - foreign_schema text; - foreign_name text; + lookup_col text := ${quoteLiteral(metadata.lookupColumnName)}; +${buildTableIdentifierDeclarations(metadata.foreignTable)} BEGIN - SELECT db_field_name INTO lookup_col - FROM field - WHERE id = ${quoteLiteral(oldField.lookupFieldId().toString())} AND deleted_time IS NULL - LIMIT 1; - - SELECT db_table_name INTO foreign_tbl - FROM table_meta - WHERE id = ${quoteLiteral(oldField.foreignTableId().toString())} AND deleted_time IS NULL - LIMIT 1; - - IF lookup_col IS NULL OR foreign_tbl IS NULL THEN + IF lookup_col IS NULL OR foreign_schema IS NULL OR foreign_name IS NULL THEN RETURN; END IF; - IF strpos(foreign_tbl, '.') > 0 THEN - foreign_schema := split_part(foreign_tbl, '.', 1); - foreign_name := split_part(foreign_tbl, '.', 2); - ELSE - foreign_schema := 'public'; - foreign_name := foreign_tbl; - END IF; - EXECUTE format( 'WITH parsed AS (\ SELECT t.__id AS source_id, 1::integer AS part_idx, NULLIF(trim((t.%I::jsonb ->> ''id'')::text), '''') AS link_id\ @@ -970,13 +1322,19 @@ BEGIN ); END $v2_link_to_text$;`; + }; + const previewMapTextSql = + buildMapTextSql(getPreviewLinkMappingMetadata(params, oldField)) ?? + `DO $v2_link_to_text$ BEGIN RETURN; END $v2_link_to_text$;`; const dropTmpSql = `ALTER TABLE ${quoteIdent(sourceSchema)}.${quoteIdent(sourceTableName)} DROP COLUMN IF EXISTS ${quoteIdent(tmpColumnName)}`; return ok([ createCompiledStatementBuilder(params.db, renameSql), ...dropStatements, ...createStatements, - createCompiledStatementBuilder(params.db, mapTextSql), + createCustomDataStatementBuilder(params.db, previewMapTextSql, async (ctx) => + buildMapTextSql(await fetchLinkMappingMetadata(ctx, oldField)) + ), createCompiledStatementBuilder(params.db, dropTmpSql), ]); }); @@ -1012,36 +1370,21 @@ const buildLinkToSelectMigrationStatements = ( const createStatements = yield* newField.accept(createVisitor); const renameSql = `ALTER TABLE ${quoteIdent(sourceSchema)}.${quoteIdent(sourceTableName)} RENAME COLUMN ${quoteIdent(dbFieldName)} TO ${quoteIdent(tmpColumnName)}`; - const mapSelectSql = ` + const buildMapSelectSql = (metadata: LinkMappingMetadata): string | null => { + if (!metadata.lookupColumnName || !metadata.foreignTable) { + return null; + } + + return ` DO $v2_link_to_select$ DECLARE - lookup_col text; - foreign_tbl text; - foreign_schema text; - foreign_name text; + lookup_col text := ${quoteLiteral(metadata.lookupColumnName)}; +${buildTableIdentifierDeclarations(metadata.foreignTable)} BEGIN - SELECT db_field_name INTO lookup_col - FROM field - WHERE id = ${quoteLiteral(oldField.lookupFieldId().toString())} AND deleted_time IS NULL - LIMIT 1; - - SELECT db_table_name INTO foreign_tbl - FROM table_meta - WHERE id = ${quoteLiteral(oldField.foreignTableId().toString())} AND deleted_time IS NULL - LIMIT 1; - - IF lookup_col IS NULL OR foreign_tbl IS NULL THEN + IF lookup_col IS NULL OR foreign_schema IS NULL OR foreign_name IS NULL THEN RETURN; END IF; - IF strpos(foreign_tbl, '.') > 0 THEN - foreign_schema := split_part(foreign_tbl, '.', 1); - foreign_name := split_part(foreign_tbl, '.', 2); - ELSE - foreign_schema := 'public'; - foreign_name := foreign_tbl; - END IF; - EXECUTE format( 'WITH parsed AS (\ SELECT t.__id AS source_id, 1::integer AS part_idx, NULLIF(trim((t.%I::jsonb ->> ''id'')::text), '''') AS link_id\ @@ -1085,6 +1428,10 @@ BEGIN ); END $v2_link_to_select$;`; + }; + const previewMapSelectSql = + buildMapSelectSql(getPreviewLinkMappingMetadata(params, oldField)) ?? + `DO $v2_link_to_select$ BEGIN RETURN; END $v2_link_to_select$;`; const singleSelectOptions = buildSelectOptionsFromValuesStatement( params, @@ -1110,7 +1457,9 @@ $v2_link_to_select$;`; createCompiledStatementBuilder(params.db, renameSql), ...dropStatements, ...createStatements, - createCompiledStatementBuilder(params.db, mapSelectSql), + createCustomDataStatementBuilder(params.db, previewMapSelectSql, async (ctx) => + buildMapSelectSql(await fetchLinkMappingMetadata(ctx, oldField)) + ), ...(isMultipleSelect ? multipleSelectOptions ? [multipleSelectOptions] @@ -1253,36 +1602,21 @@ const buildLinkToLinkForeignTableMigrationStatements = ( } } - const mapSql = ` + const buildMapSql = (metadata: LinkMappingMetadata): string | null => { + if (!metadata.lookupColumnName || !metadata.foreignTable) { + return null; + } + + return ` DO $v2_link_remap$ DECLARE - lookup_col text; - foreign_tbl text; - foreign_schema text; - foreign_name text; + lookup_col text := ${quoteLiteral(metadata.lookupColumnName)}; +${buildTableIdentifierDeclarations(metadata.foreignTable)} BEGIN - SELECT db_field_name INTO lookup_col - FROM field - WHERE id = ${quoteLiteral(newLinkField.lookupFieldId().toString())} AND deleted_time IS NULL - LIMIT 1; - - SELECT db_table_name INTO foreign_tbl - FROM table_meta - WHERE id = ${quoteLiteral(newLinkField.foreignTableId().toString())} AND deleted_time IS NULL - LIMIT 1; - - IF lookup_col IS NULL OR foreign_tbl IS NULL THEN + IF lookup_col IS NULL OR foreign_schema IS NULL OR foreign_name IS NULL THEN RETURN; END IF; - IF strpos(foreign_tbl, '.') > 0 THEN - foreign_schema := split_part(foreign_tbl, '.', 1); - foreign_name := split_part(foreign_tbl, '.', 2); - ELSE - foreign_schema := 'public'; - foreign_name := foreign_tbl; - END IF; - ${ensureForeignFkSql} ${ensureForeignOrderSql} @@ -1319,6 +1653,10 @@ BEGIN ); END $v2_link_remap$;`; + }; + const previewMapSql = + buildMapSql(getPreviewLinkMappingMetadata(params, newLinkField)) ?? + `DO $v2_link_remap$ BEGIN RETURN; END $v2_link_remap$;`; const dropTmpSql = `ALTER TABLE ${quoteIdent(sourceSchema)}.${quoteIdent(sourceTableName)} DROP COLUMN IF EXISTS ${quoteIdent(tmpColumnName)}`; @@ -1326,7 +1664,9 @@ $v2_link_remap$;`; createCompiledStatementBuilder(params.db, renameSql), ...dropStatements, ...createStatements, - createCompiledStatementBuilder(params.db, mapSql), + createCustomDataStatementBuilder(params.db, previewMapSql, async (ctx) => + buildMapSql(await fetchLinkMappingMetadata(ctx, newLinkField)) + ), createCompiledStatementBuilder(params.db, dropTmpSql), ]); }); @@ -1445,92 +1785,19 @@ export abstract class BaseFieldConversionVisitor extends AbstractFieldVisitor< * 4. Update the field table's options column */ protected generateSelectOptionsFromValues(): TableSchemaStatementBuilder | null { - const { db, fieldId, dbFieldName } = this.params; + const { fieldId, dbFieldName } = this.params; if (!fieldId) { return null; } const fullTableName = this.fullTableName; - const colors = SELECT_OPTION_COLORS; - - // This SQL: - // 1. Gets distinct non-null, non-empty values from the column (cast to text) - // 2. Filters out values that already exist in options.choices - // 3. Generates new choice objects with random IDs and colors - // 4. Merges new choices with existing choices - // 5. Updates the field table - return { - scope: 'data', - compile: () => - sql` - WITH distinct_values AS ( - SELECT DISTINCT ${sql.raw(`"${dbFieldName}"::text`)} AS name - FROM ${sql.raw(fullTableName)} - WHERE ${sql.raw(`"${dbFieldName}"`)} IS NOT NULL - AND ${sql.raw(`"${dbFieldName}"::text`)} <> '' - ), - existing_choices AS ( - SELECT jsonb_array_elements(COALESCE(options::jsonb->'choices', '[]'::jsonb)) AS choice - FROM field - WHERE id = ${sql.val(fieldId)} - ), - existing_names AS ( - SELECT choice->>'name' AS name FROM existing_choices - ), - oversized_values AS ( - SELECT name FROM distinct_values - WHERE char_length(name) > ${sql.val(SELECT_CHOICE_NAME_MAX_LENGTH)} - ), - oversized_guard AS ( - SELECT CASE - WHEN EXISTS (SELECT 1 FROM oversized_values) - -- Keep the invalid cast runtime-bound; Postgres folds constant casts before CASE short-circuits. - THEN CAST( - ${sql.val(selectChoiceNameLengthError)} - || COALESCE((SELECT '' FROM oversized_values LIMIT 1), '') - AS integer - ) - ELSE 1 - END AS ok - ), - new_values AS ( - SELECT name, ROW_NUMBER() OVER () AS rn - FROM distinct_values - CROSS JOIN oversized_guard - WHERE name NOT IN (SELECT name FROM existing_names WHERE name IS NOT NULL) - AND oversized_guard.ok = 1 - ), - new_choices AS ( - SELECT jsonb_agg( - jsonb_build_object( - 'id', 'cho' || substr(md5(random()::text || clock_timestamp()::text), 1, 16), - 'name', name, - 'color', (ARRAY[${sql.join( - colors.map((c) => sql.val(c)), - sql`, ` - )}])[(rn % ${sql.val(colors.length)}) + 1] - ) - ) AS choices - FROM new_values - ), - merged_choices AS ( - SELECT COALESCE( - (SELECT jsonb_agg(choice) FROM existing_choices), - '[]'::jsonb - ) || COALESCE( - (SELECT choices FROM new_choices), - '[]'::jsonb - ) AS choices - ) - UPDATE field - SET options = jsonb_set( - COALESCE(options::jsonb, '{}'::jsonb), - '{choices}', - (SELECT choices FROM merged_choices) - ) - WHERE id = ${sql.val(fieldId)} - `.compile(db), - }; + return buildSelectOptionsFromValuesStatement( + this.params, + `SELECT DISTINCT ${quoteIdent(dbFieldName)}::text AS name + FROM ${fullTableName} + WHERE ${quoteIdent(dbFieldName)} IS NOT NULL + AND ${quoteIdent(dbFieldName)}::text <> ''` + ); } /** @@ -1828,7 +2095,7 @@ class TextFieldConversionVisitor extends BaseFieldConversionVisitor { // Text → MultipleSelect: CSV-aware split into array (V1 parity) // Respects double-quoted fields so commas inside "..." are preserved. // Auto-generate options from distinct split values - const { db, dbFieldName, fieldId } = this.params; + const { db, dbFieldName } = this.params; const col = `"${dbFieldName}"`; const fullTableName = this.fullTableName; const statements: TableSchemaStatementBuilder[] = []; @@ -1836,63 +2103,15 @@ class TextFieldConversionVisitor extends BaseFieldConversionVisitor { // E-string \\n/\\r → PG \n/\r → literal newline/CR in regex const csvFieldRegex = `E' *(?:"([^"]*)"|([^,\\n\\r]+))'`; - // Generate options from distinct split values (not whole cell values) - if (fieldId) { - const colors = SELECT_OPTION_COLORS; - statements.push({ - scope: 'data', - compile: () => - sql` - WITH distinct_values AS ( - SELECT DISTINCT COALESCE(trim(m[1]), trim(m[2])) AS name - FROM ${sql.raw(fullTableName)}, regexp_matches(${sql.raw(col)}, ${sql.raw(csvFieldRegex)}, 'g') AS m - WHERE ${sql.raw(col)} IS NOT NULL - AND COALESCE(trim(m[1]), trim(m[2])) <> '' - ), - existing_choices AS ( - SELECT jsonb_array_elements(COALESCE(options::jsonb->'choices', '[]'::jsonb)) AS choice - FROM field - WHERE id = ${sql.val(fieldId)} - ), - existing_names AS ( - SELECT choice->>'name' AS name FROM existing_choices - ), - new_values AS ( - SELECT name, ROW_NUMBER() OVER () AS rn - FROM distinct_values - WHERE name NOT IN (SELECT name FROM existing_names WHERE name IS NOT NULL) - ), - new_choices AS ( - SELECT jsonb_agg( - jsonb_build_object( - 'id', 'cho' || substr(md5(random()::text || clock_timestamp()::text), 1, 16), - 'name', name, - 'color', (ARRAY[${sql.join( - colors.map((c) => sql.val(c)), - sql`, ` - )}])[(rn % ${sql.val(colors.length)}) + 1] - ) - ) AS choices - FROM new_values - ), - merged_choices AS ( - SELECT COALESCE( - (SELECT jsonb_agg(choice) FROM existing_choices), - '[]'::jsonb - ) || COALESCE( - (SELECT choices FROM new_choices), - '[]'::jsonb - ) AS choices - ) - UPDATE field - SET options = jsonb_set( - COALESCE(options::jsonb, '{}'::jsonb), - '{choices}', - (SELECT choices FROM merged_choices) - ) - WHERE id = ${sql.val(fieldId)} - `.compile(db), - }); + const optionsStatement = buildSelectOptionsFromValuesStatement( + this.params, + `SELECT DISTINCT COALESCE(trim(m[1]), trim(m[2])) AS name + FROM ${fullTableName}, regexp_matches(${col}, ${csvFieldRegex}, 'g') AS m + WHERE ${col} IS NOT NULL + AND COALESCE(trim(m[1]), trim(m[2])) <> ''` + ); + if (optionsStatement) { + statements.push(optionsStatement); } // CSV-aware split: aggregate quoted/unquoted fields into jsonb array @@ -1921,89 +2140,41 @@ class TextFieldConversionVisitor extends BaseFieldConversionVisitor { visitUserField( field: UserField ): Result, DomainError> { - // Text → User: split comma-separated text, match each part against base collaborators - // by id, name, or email (matching V1 convert2User behavior). - // PostgreSQL doesn't allow subqueries in ALTER USING, so we use two steps: - // 1. UPDATE to transform text values to JSONB user objects (stored as text temporarily) - // 2. ALTER to change column type from text to jsonb - const { db, dbFieldName, tableId } = this.params; + // Text -> User: resolve text parts through the meta DB at execution time, then write only + // the resolved snapshots back to the data DB. This keeps data SQL free of meta-table joins + // for BYODB while preserving the old "unmatched text becomes NULL" conversion behavior. + const { db, dbFieldName } = this.params; const fullTableName = this.fullTableName; const col = `"${dbFieldName}"`; const isMultiple = field.multiplicity().toBoolean(); - - const avatarPrefix = resolveUserAvatarUrlPrefix(); - const userJsonBuild = `jsonb_build_object('id', cu.uid, 'title', cu.uname, 'email', cu.uemail, 'avatarUrl', ${quoteLiteral( - avatarPrefix - )} || cu.uid)`; - - // Single UPDATE that resolves all text values to user JSON (or NULL for no match). - // Uses a CTE to find base collaborators and match comma-separated parts. - // Wrapped in a DO block that checks for collaborator table existence, - // since in pglite/v2-only environments these v1 schema tables don't exist. - const matchCte = `WITH base_info AS ( - SELECT base_id FROM table_meta WHERE id = ${quoteLiteral(tableId)} AND deleted_time IS NULL LIMIT 1 - ), - space_info AS ( - SELECT space_id FROM base WHERE id = (SELECT base_id FROM base_info) AND deleted_time IS NULL LIMIT 1 - ), - collab_users AS ( - SELECT DISTINCT u.id AS uid, u.name AS uname, u.email AS uemail - FROM collaborator c - JOIN users u ON c.principal_id = u.id - WHERE c.resource_id IN ( - (SELECT base_id FROM base_info), - (SELECT space_id FROM space_info) - ) - ), - matched_users AS ( - SELECT DISTINCT ON (t.__id, cu.uid) - t.__id AS rid, cu.uid, cu.uname, cu.uemail, parts.part_idx - FROM ${fullTableName} t - CROSS JOIN LATERAL ( - SELECT trim(part) AS val, row_number() OVER () AS part_idx - FROM regexp_split_to_table(t.${col}, ${quoteLiteral(',')}) AS part - ) parts - JOIN collab_users cu ON ( - parts.val = cu.uid OR parts.val = cu.uname OR parts.val = cu.uemail - ) - WHERE t.${col} IS NOT NULL - ORDER BY t.__id, cu.uid, parts.part_idx - ), - aggregated AS ( - SELECT m.rid, ${ - isMultiple - ? `jsonb_agg(${userJsonBuild.replace(/cu\./g, 'm.')} ORDER BY m.part_idx)::text AS user_json` - : `(SELECT jsonb_build_object(${quoteLiteral('id')}, m2.uid, ${quoteLiteral('title')}, m2.uname, ${quoteLiteral('email')}, m2.uemail, ${quoteLiteral('avatarUrl')}, ${quoteLiteral(avatarPrefix)} || m2.uid)::text FROM matched_users m2 WHERE m2.rid = m.rid ORDER BY m2.part_idx LIMIT 1) AS user_json` - } - FROM matched_users m - GROUP BY m.rid - ) - UPDATE ${fullTableName} AS t - SET ${col} = agg.user_json - FROM aggregated agg - WHERE t.__id = agg.rid`; - - const updateSql = ` -DO $v2_user_conv$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'collaborator') THEN - RETURN; - END IF; - EXECUTE $v2_inner$${matchCte}$v2_inner$; -END -$v2_user_conv$;`; - - // Null out any remaining text values that weren't matched (no collaborator found) const cleanupSql = `UPDATE ${fullTableName} SET ${col} = NULL WHERE ${col} IS NOT NULL AND left(${col}, 1) NOT IN ('{', '[')`; + const previewSql = + buildTextToUserTransformSql(fullTableName, col, isMultiple, [ + { + lookupValue: '__preview_user__', + id: '__preview_user__', + title: '__preview_user__', + email: null, + }, + ]) ?? cleanupSql; return ok([ { scope: 'data', - compile: () => sql.raw(updateSql).compile(db), - }, - { - scope: 'data', - compile: () => sql.raw(cleanupSql).compile(db), + compile: () => sql.raw(previewSql).compile(db), + execute: async (ctx) => { + const mappings = await fetchTextUserMappings(ctx, fullTableName, col); + const transformSql = buildTextToUserTransformSql( + fullTableName, + col, + isMultiple, + mappings + ); + if (transformSql) { + await sql.raw(transformSql).execute(ctx.dataDb); + } + await sql.raw(cleanupSql).execute(ctx.dataDb); + }, }, { scope: 'data', @@ -2558,66 +2729,18 @@ class MultipleSelectFieldConversionVisitor extends BaseFieldConversionVisitor { _field: SingleSelectField ): Result, DomainError> { // MultipleSelect → SingleSelect: take first item - const { db, dbFieldName, fieldId } = this.params; + const { dbFieldName } = this.params; const col = `"${dbFieldName}"`; const statements: TableSchemaStatementBuilder[] = []; - if (fieldId) { - const colors = SELECT_OPTION_COLORS; - statements.push({ - scope: 'data', - compile: () => - sql` - WITH distinct_values AS ( - SELECT DISTINCT value::text AS name - FROM ${sql.raw(this.fullTableName)}, jsonb_array_elements_text(${sql.raw(col)}) - WHERE ${sql.raw(col)} IS NOT NULL - ), - existing_choices AS ( - SELECT jsonb_array_elements(COALESCE(options::jsonb->'choices', '[]'::jsonb)) AS choice - FROM field - WHERE id = ${sql.val(fieldId)} - ), - existing_names AS ( - SELECT choice->>'name' AS name FROM existing_choices - ), - new_values AS ( - SELECT name, ROW_NUMBER() OVER () AS rn - FROM distinct_values - WHERE name NOT IN (SELECT name FROM existing_names WHERE name IS NOT NULL) - AND name IS NOT NULL AND name <> '' - ), - new_choices AS ( - SELECT jsonb_agg( - jsonb_build_object( - 'id', 'cho' || substr(md5(random()::text || clock_timestamp()::text), 1, 16), - 'name', name, - 'color', (ARRAY[${sql.join( - colors.map((c) => sql.val(c)), - sql`, ` - )}])[(rn % ${sql.val(colors.length)}) + 1] - ) - ) AS choices - FROM new_values - ), - merged_choices AS ( - SELECT COALESCE( - (SELECT jsonb_agg(choice) FROM existing_choices), - '[]'::jsonb - ) || COALESCE( - (SELECT choices FROM new_choices), - '[]'::jsonb - ) AS choices - ) - UPDATE field - SET options = jsonb_set( - COALESCE(options::jsonb, '{}'::jsonb), - '{choices}', - (SELECT choices FROM merged_choices) - ) - WHERE id = ${sql.val(fieldId)} - `.compile(db), - }); + const optionsStatement = buildSelectOptionsFromValuesStatement( + this.params, + `SELECT DISTINCT value::text AS name + FROM ${this.fullTableName}, jsonb_array_elements_text(${col}) AS value + WHERE ${col} IS NOT NULL` + ); + if (optionsStatement) { + statements.push(optionsStatement); } statements.push( @@ -2903,63 +3026,14 @@ class UserFieldConversionVisitor extends BaseFieldConversionVisitor { // Generate options from distinct values in the array // Note: For multipleSelect, we need to unnest the arrays to get distinct values - const { fieldId } = this.params; - if (fieldId) { - const colors = SELECT_OPTION_COLORS; - statements.push({ - scope: 'data', - compile: () => - sql` - WITH distinct_values AS ( - SELECT DISTINCT value::text AS name - FROM ${sql.raw(fullTableName)}, jsonb_array_elements_text("${sql.raw(dbFieldName)}") - WHERE "${sql.raw(dbFieldName)}" IS NOT NULL - ), - existing_choices AS ( - SELECT jsonb_array_elements(COALESCE(options::jsonb->'choices', '[]'::jsonb)) AS choice - FROM field - WHERE id = ${sql.val(fieldId)} - ), - existing_names AS ( - SELECT choice->>'name' AS name FROM existing_choices - ), - new_values AS ( - SELECT name, ROW_NUMBER() OVER () AS rn - FROM distinct_values - WHERE name NOT IN (SELECT name FROM existing_names WHERE name IS NOT NULL) - AND name IS NOT NULL AND name <> '' - ), - new_choices AS ( - SELECT jsonb_agg( - jsonb_build_object( - 'id', 'cho' || substr(md5(random()::text || clock_timestamp()::text), 1, 16), - 'name', name, - 'color', (ARRAY[${sql.join( - colors.map((c) => sql.val(c)), - sql`, ` - )}])[(rn % ${sql.val(colors.length)}) + 1] - ) - ) AS choices - FROM new_values - ), - merged_choices AS ( - SELECT COALESCE( - (SELECT jsonb_agg(choice) FROM existing_choices), - '[]'::jsonb - ) || COALESCE( - (SELECT choices FROM new_choices), - '[]'::jsonb - ) AS choices - ) - UPDATE field - SET options = jsonb_set( - COALESCE(options::jsonb, '{}'::jsonb), - '{choices}', - (SELECT choices FROM merged_choices) - ) - WHERE id = ${sql.val(fieldId)} - `.compile(db), - }); + const optionsStatement = buildSelectOptionsFromValuesStatement( + this.params, + `SELECT DISTINCT value::text AS name + FROM ${fullTableName}, jsonb_array_elements_text("${dbFieldName}") AS value + WHERE "${dbFieldName}" IS NOT NULL` + ); + if (optionsStatement) { + statements.push(optionsStatement); } return ok(statements); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldValueChangeCollectorVisitor.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldValueChangeCollectorVisitor.ts index ba961afe74..68be7af00e 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldValueChangeCollectorVisitor.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/FieldValueChangeCollectorVisitor.ts @@ -82,6 +82,7 @@ export class FieldValueChangeCollectorVisitor implements ITableSpecVisitor private readonly selfBackfillFieldIdSet = new Map(); private readonly valueChangedFieldIdSet = new Map(); private readonly deferredBackfillFieldIdSet = new Map(); + private readonly recordActionFieldIdSet = new Map(); private dbStorageTypeChanged = false; selfBackfillFields(): ReadonlyArray { @@ -92,6 +93,10 @@ export class FieldValueChangeCollectorVisitor implements ITableSpecVisitor return [...this.valueChangedFieldIdSet.values()]; } + recordActionFields(): ReadonlyArray { + return [...this.recordActionFieldIdSet.values()]; + } + deferredBackfillFields(): ReadonlyArray { return [...this.deferredBackfillFieldIdSet.values()]; } @@ -114,6 +119,10 @@ export class FieldValueChangeCollectorVisitor implements ITableSpecVisitor this.valueChangedFieldIdSet.set(fieldId.toString(), fieldId); } + private addRecordAction(fieldId: FieldId): void { + this.recordActionFieldIdSet.set(fieldId.toString(), fieldId); + } + private addDeferredBackfill(fieldId: FieldId): void { this.deferredBackfillFieldIdSet.set(fieldId.toString(), fieldId); } @@ -392,6 +401,9 @@ export class FieldValueChangeCollectorVisitor implements ITableSpecVisitor if (spec.renamedOptions().length > 0 || spec.removedOptions().length > 0) { this.addValueChanged(spec.fieldId()); } + if (spec.removedOptions().length > 0) { + this.addRecordAction(spec.fieldId()); + } return ok(undefined); } @@ -415,6 +427,9 @@ export class FieldValueChangeCollectorVisitor implements ITableSpecVisitor if (spec.renamedOptions().length > 0 || spec.removedOptions().length > 0) { this.addValueChanged(spec.fieldId()); } + if (spec.removedOptions().length > 0) { + this.addRecordAction(spec.fieldId()); + } return ok(undefined); } diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.spec.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.spec.ts index 49b7a34a30..3fe0360e55 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.spec.ts @@ -8,6 +8,55 @@ const mocks = vi.hoisted(() => ({ createSchemaRuleContext: vi.fn(), upAll: vi.fn(), PostgresSchemaIntrospector: vi.fn(), + GeneratedColumnRule: class MockGeneratedColumnRule { + readonly id: string; + + constructor( + private readonly field: { id: () => { toString: () => string } }, + private readonly columnType = 'text generated always as ("__created_by") stored' + ) { + this.id = `generated_column:${field.id().toString()}`; + } + + createTableColumnType() { + return this.columnType; + } + }, + FkColumnRule: class MockFkColumnRule { + readonly id: string; + + constructor( + private readonly field: { id: () => { toString: () => string } }, + private readonly columnName: string, + private readonly targetTable?: { schema: string | null; tableName: string } + ) { + this.id = `fk_column:${field.id().toString()}`; + } + + createTableColumnForTarget(targetTable: { schema: string | null; tableName: string }) { + const target = this.targetTable ?? targetTable; + if ((target.schema ?? null) !== (targetTable.schema ?? null)) return undefined; + if (target.tableName !== targetTable.tableName) return undefined; + return { columnName: this.columnName, dataType: 'text' }; + } + }, + OrderColumnRule: class MockOrderColumnRule { + readonly id: string; + + constructor( + private readonly field: { id: () => { toString: () => string } }, + private readonly columnName: string, + private readonly targetTable: { schema: string | null; tableName: string } + ) { + this.id = `order_column:${field.id().toString()}`; + } + + createTableColumnForTarget(targetTable: { schema: string | null; tableName: string }) { + if ((this.targetTable.schema ?? null) !== (targetTable.schema ?? null)) return undefined; + if (this.targetTable.tableName !== targetTable.tableName) return undefined; + return { columnName: this.columnName, dataType: 'double precision' }; + } + }, })); const methodNames = [ @@ -56,6 +105,9 @@ const loadPostgresTableSchemaFieldCreateVisitor = async () => { createFieldSchemaRules: mocks.createFieldSchemaRules, createSchemaRuleContext: mocks.createSchemaRuleContext, PostgresSchemaIntrospector: mocks.PostgresSchemaIntrospector, + GeneratedColumnRule: mocks.GeneratedColumnRule, + FkColumnRule: mocks.FkColumnRule, + OrderColumnRule: mocks.OrderColumnRule, schemaRuleResolver: { upAll: mocks.upAll, }, @@ -96,6 +148,7 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { schema: 'public', tableName: 'tasks', tableId: 'tbl_tasks', + optimizeForEmptyTables: true, }); const fields = methodNames.map((methodName, index) => createField(methodName, index)); @@ -112,6 +165,7 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { tableId: 'tbl_tasks', introspector: expect.any(Object), field: fields[0], + optimizeForEmptyTables: true, }) ); expect(mocks.upAll).toHaveBeenCalledWith( @@ -125,6 +179,9 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { it('supports table inputs and table-creation mode metadata', async () => { const db = createTestDb(); + const builder = { + addColumn: vi.fn().mockReturnThis(), + }; mocks.PostgresSchemaIntrospector.mockImplementation(function ( this: { currentDb: unknown }, @@ -133,7 +190,14 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { this.currentDb = currentDb; }); mocks.createFieldSchemaRules.mockImplementation((field) => - ok([`create:${field.id().toString()}`]) + ok([ + { + id: `column:${field.id().toString()}`, + }, + { + id: `reference:${field.id().toString()}`, + }, + ]) ); mocks.createSchemaRuleContext.mockImplementation((context) => context); mocks.upAll.mockImplementation((rules) => ok([{ compile: vi.fn(), rules }])); @@ -142,8 +206,12 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { await loadPostgresTableSchemaFieldCreateVisitor(); const field = createField('visitSingleLineTextField', 99); + field.dbFieldName = () => + ok({ + value: () => ok('Name'), + }); const visitor = PostgresTableSchemaFieldCreateVisitor.forTableCreation({ - builderRef: { builder: {} as never }, + builderRef: { builder: builder as never }, db: db as never, schema: null, tableName: 'draft_tasks', @@ -161,6 +229,11 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { const result = visitor.apply(table as never); expect(result._unsafeUnwrap()).toHaveLength(1); + expect(builder.addColumn).toHaveBeenCalledWith('Name', 'text'); + expect(mocks.upAll).toHaveBeenCalledWith( + [expect.objectContaining({ id: `reference:${field.id().toString()}` })], + expect.any(Object) + ); expect(mocks.createSchemaRuleContext).toHaveBeenLastCalledWith( expect.objectContaining({ schema: null, @@ -170,4 +243,209 @@ describe('PostgresTableSchemaFieldCreateVisitor', () => { }) ); }); + + it('folds generated and same-table helper columns into CREATE TABLE statements', async () => { + const db = createTestDb(); + const builder = { + addColumn: vi.fn().mockReturnThis(), + }; + const generatedColumnType = 'text generated always as ("__created_by") stored'; + + mocks.PostgresSchemaIntrospector.mockImplementation(function ( + this: { currentDb: unknown }, + currentDb + ) { + this.currentDb = currentDb; + }); + mocks.createFieldSchemaRules.mockImplementation((field) => { + const fieldId = field.id().toString(); + if (fieldId.endsWith('100')) { + return ok([ + new mocks.GeneratedColumnRule(field, generatedColumnType), + { id: `reference:${fieldId}` }, + ]); + } + return ok([ + { id: `link_value_column:${fieldId}` }, + new mocks.FkColumnRule(field, '__fk_assignee', { + schema: 'public', + tableName: 'draft_tasks', + }), + new mocks.OrderColumnRule(field, '__fk_assignee_order', { + schema: 'public', + tableName: 'draft_tasks', + }), + { id: `index:${fieldId}:fk_column` }, + { id: `field_meta:${fieldId}:order_column` }, + ]); + }); + mocks.createSchemaRuleContext.mockImplementation((context) => context); + mocks.upAll.mockImplementation((rules) => ok([{ compile: vi.fn(), rules }])); + + const { PostgresTableSchemaFieldCreateVisitor } = + await loadPostgresTableSchemaFieldCreateVisitor(); + + const generatedField = createField('visitCreatedByField', 100); + generatedField.dbFieldName = () => + ok({ + value: () => ok('created_by_display'), + }); + const linkField = createField('visitLinkField', 101); + linkField.dbFieldName = () => + ok({ + value: () => ok('assignee_display'), + }); + + const visitor = PostgresTableSchemaFieldCreateVisitor.forTableCreation({ + builderRef: { builder: builder as never }, + db: db as never, + schema: 'public', + tableName: 'draft_tasks', + tableId: 'tbl_draft', + }); + + const result = visitor.apply([generatedField, linkField] as never); + + expect(result.isOk()).toBe(true); + expect(builder.addColumn).toHaveBeenCalledWith('created_by_display', generatedColumnType); + expect(builder.addColumn).toHaveBeenCalledWith('assignee_display', 'jsonb'); + expect(builder.addColumn).toHaveBeenCalledWith('__fk_assignee', 'text'); + expect(builder.addColumn).toHaveBeenCalledWith('__fk_assignee_order', 'double precision'); + expect(mocks.upAll).toHaveBeenCalledWith( + [expect.objectContaining({ id: `reference:${generatedField.id().toString()}` })], + expect.any(Object) + ); + expect(mocks.upAll).toHaveBeenCalledWith( + [ + expect.objectContaining({ id: `index:${linkField.id().toString()}:fk_column` }), + expect.objectContaining({ id: `field_meta:${linkField.id().toString()}:order_column` }), + ], + expect.any(Object) + ); + }); + + it('keeps cross-table helper columns as schema statements', async () => { + const db = createTestDb(); + const builder = { + addColumn: vi.fn().mockReturnThis(), + }; + + mocks.PostgresSchemaIntrospector.mockImplementation(function ( + this: { currentDb: unknown }, + currentDb + ) { + this.currentDb = currentDb; + }); + mocks.createFieldSchemaRules.mockImplementation((field) => { + const fieldId = field.id().toString(); + return ok([ + new mocks.FkColumnRule(field, '__fk_foreign', { + schema: 'public', + tableName: 'foreign_tasks', + }), + new mocks.OrderColumnRule(field, '__fk_foreign_order', { + schema: 'public', + tableName: 'foreign_tasks', + }), + ]); + }); + mocks.createSchemaRuleContext.mockImplementation((context) => context); + mocks.upAll.mockImplementation((rules) => ok([{ compile: vi.fn(), rules }])); + + const { PostgresTableSchemaFieldCreateVisitor } = + await loadPostgresTableSchemaFieldCreateVisitor(); + + const linkField = createField('visitLinkField', 102); + linkField.dbFieldName = () => + ok({ + value: () => ok('foreign_display'), + }); + + const visitor = PostgresTableSchemaFieldCreateVisitor.forTableCreation({ + builderRef: { builder: builder as never }, + db: db as never, + schema: 'public', + tableName: 'draft_tasks', + tableId: 'tbl_draft', + }); + + const result = visitor.apply([linkField] as never); + + expect(result.isOk()).toBe(true); + expect(builder.addColumn).not.toHaveBeenCalledWith('__fk_foreign', 'text'); + expect(builder.addColumn).not.toHaveBeenCalledWith('__fk_foreign_order', 'double precision'); + expect(mocks.upAll).toHaveBeenCalledWith( + [ + expect.objectContaining({ id: `fk_column:${linkField.id().toString()}` }), + expect.objectContaining({ id: `order_column:${linkField.id().toString()}` }), + ], + expect.any(Object) + ); + }); + + it('deduplicates shared same-table helper columns during table creation', async () => { + const db = createTestDb(); + const builder = { + addColumn: vi.fn().mockReturnThis(), + }; + + mocks.PostgresSchemaIntrospector.mockImplementation(function ( + this: { currentDb: unknown }, + currentDb + ) { + this.currentDb = currentDb; + }); + mocks.createFieldSchemaRules.mockImplementation((field) => { + const fieldId = field.id().toString(); + return ok([ + new mocks.FkColumnRule(field, '__fk_shared', { + schema: 'public', + tableName: 'draft_tasks', + }), + new mocks.OrderColumnRule(field, '__fk_shared_order', { + schema: 'public', + tableName: 'draft_tasks', + }), + { id: `index:${fieldId}:fk_column` }, + ]); + }); + mocks.createSchemaRuleContext.mockImplementation((context) => context); + mocks.upAll.mockImplementation((rules) => ok([{ compile: vi.fn(), rules }])); + + const { PostgresTableSchemaFieldCreateVisitor } = + await loadPostgresTableSchemaFieldCreateVisitor(); + + const firstLinkField = createField('visitLinkField', 103); + const secondLinkField = createField('visitLinkField', 104); + const visitor = PostgresTableSchemaFieldCreateVisitor.forTableCreation({ + builderRef: { builder: builder as never }, + db: db as never, + schema: 'public', + tableName: 'draft_tasks', + tableId: 'tbl_draft', + }); + + const result = visitor.apply([firstLinkField, secondLinkField] as never); + + expect(result.isOk()).toBe(true); + expect( + builder.addColumn.mock.calls.filter( + ([columnName, dataType]) => columnName === '__fk_shared' && dataType === 'text' + ) + ).toHaveLength(1); + expect( + builder.addColumn.mock.calls.filter( + ([columnName, dataType]) => + columnName === '__fk_shared_order' && dataType === 'double precision' + ) + ).toHaveLength(1); + expect(mocks.upAll).toHaveBeenCalledWith( + [expect.objectContaining({ id: `index:${firstLinkField.id().toString()}:fk_column` })], + expect.any(Object) + ); + expect(mocks.upAll).toHaveBeenCalledWith( + [expect.objectContaining({ id: `index:${secondLinkField.id().toString()}:fk_column` })], + expect.any(Object) + ); + }); }); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.ts index 50ddf02cc3..3956201515 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/PostgresTableSchemaFieldCreateVisitor.ts @@ -34,13 +34,18 @@ import type { Result } from 'neverthrow'; import { createFieldSchemaRules, createSchemaRuleContext, + FkColumnRule, + GeneratedColumnRule, + OrderColumnRule, PostgresSchemaIntrospector, schemaRuleResolver, type FieldSchemaRulesContext, type SchemaRuleContext, } from '../rules'; +import type { ISchemaRule } from '../rules/core'; import type { TableSchemaStatementBuilder } from '../rules/core'; import type { TableIdentifier } from '../rules/helpers'; +import { resolveColumnName, resolveColumnType } from './PostgresTableSchemaFieldColumn'; type ICreateTableBuilder = CreateTableBuilder; @@ -76,6 +81,8 @@ export const buildTableLocationsById = ( export class PostgresTableSchemaFieldCreateVisitor extends AbstractFieldVisitor< ReadonlyArray > { + private readonly createTableHelperColumnNames = new Set(); + private constructor( private readonly db: Kysely, private rulesContext: FieldSchemaRulesContext, @@ -95,6 +102,7 @@ export class PostgresTableSchemaFieldCreateVisitor extends AbstractFieldVisitor< tableName: string; tableId: string; tableLocationsById?: ReadonlyMap; + optimizeForEmptyTables?: boolean; }): PostgresTableSchemaFieldCreateVisitor { return new PostgresTableSchemaFieldCreateVisitor( params.db, @@ -103,6 +111,7 @@ export class PostgresTableSchemaFieldCreateVisitor extends AbstractFieldVisitor< tableName: params.tableName, tableId: params.tableId, tableLocationsById: params.tableLocationsById, + optimizeForEmptyTables: params.optimizeForEmptyTables, }, params.builderRef ); @@ -118,12 +127,14 @@ export class PostgresTableSchemaFieldCreateVisitor extends AbstractFieldVisitor< tableName: string; tableId: string; tableLocationsById?: ReadonlyMap; + optimizeForEmptyTables?: boolean; }): PostgresTableSchemaFieldCreateVisitor { return new PostgresTableSchemaFieldCreateVisitor(params.db, { schema: params.schema, tableName: params.tableName, tableId: params.tableId, tableLocationsById: params.tableLocationsById, + optimizeForEmptyTables: params.optimizeForEmptyTables, }); } @@ -178,9 +189,96 @@ export class PostgresTableSchemaFieldCreateVisitor extends AbstractFieldVisitor< tableName: this.rulesContext.tableName, tableId: this.rulesContext.tableId, field, + optimizeForEmptyTables: this.rulesContext.optimizeForEmptyTables, }); } + private addCreateTableColumnFromRules( + field: Field, + rules: ReadonlyArray + ): Result, DomainError> { + return safeTry, DomainError>( + function* (this: PostgresTableSchemaFieldCreateVisitor) { + if (!this.builderRef) { + return ok(rules); + } + + const fieldId = field.id().toString(); + const columnRuleId = `column:${fieldId}`; + const linkValueColumnRuleId = `link_value_column:${fieldId}`; + const generatedColumnRuleId = `generated_column:${fieldId}`; + const fkColumnRuleId = `fk_column:${fieldId}`; + const orderColumnRuleId = `order_column:${fieldId}`; + const skipRuleIds = new Set(); + const currentTable: TableIdentifier = { + schema: this.rulesContext.schema, + tableName: this.rulesContext.tableName, + }; + + const generatedColumnRule = rules.find( + (rule): rule is GeneratedColumnRule => + rule.id === generatedColumnRuleId && rule instanceof GeneratedColumnRule + ); + + if (generatedColumnRule) { + const columnName = yield* resolveColumnName(field); + this.builderRef.builder = this.builderRef.builder.addColumn( + columnName, + generatedColumnRule.createTableColumnType() + ); + skipRuleIds.add(generatedColumnRuleId); + } else if (rules.some((rule) => rule.id === columnRuleId)) { + const columnName = yield* resolveColumnName(field); + const dataType = yield* resolveColumnType(field); + this.builderRef.builder = this.builderRef.builder.addColumn(columnName, dataType); + skipRuleIds.add(columnRuleId); + skipRuleIds.add(`generated_meta:${fieldId}`); + } else if (rules.some((rule) => rule.id === linkValueColumnRuleId)) { + const columnName = yield* resolveColumnName(field); + this.builderRef.builder = this.builderRef.builder.addColumn(columnName, 'jsonb'); + skipRuleIds.add(linkValueColumnRuleId); + } + + const fkColumnRule = rules.find( + (rule): rule is FkColumnRule => rule.id === fkColumnRuleId && rule instanceof FkColumnRule + ); + const fkColumn = fkColumnRule?.createTableColumnForTarget(currentTable); + if (fkColumn) { + if (!this.createTableHelperColumnNames.has(fkColumn.columnName)) { + this.builderRef.builder = this.builderRef.builder.addColumn( + fkColumn.columnName, + fkColumn.dataType + ); + this.createTableHelperColumnNames.add(fkColumn.columnName); + } + skipRuleIds.add(fkColumnRuleId); + } + + const orderColumnRule = rules.find( + (rule): rule is OrderColumnRule => + rule.id === orderColumnRuleId && rule instanceof OrderColumnRule + ); + const orderColumn = orderColumnRule?.createTableColumnForTarget(currentTable); + if (orderColumn) { + if (!this.createTableHelperColumnNames.has(orderColumn.columnName)) { + this.builderRef.builder = this.builderRef.builder.addColumn( + orderColumn.columnName, + orderColumn.dataType + ); + this.createTableHelperColumnNames.add(orderColumn.columnName); + } + skipRuleIds.add(orderColumnRuleId); + } + + if (skipRuleIds.size === 0) { + return ok(rules); + } + + return ok(rules.filter((rule) => !skipRuleIds.has(rule.id))); + }.bind(this) + ); + } + /** * Generates statements using the rules system. * For table creation mode, it delegates to a specialized handler that @@ -192,7 +290,9 @@ export class PostgresTableSchemaFieldCreateVisitor extends AbstractFieldVisitor< return safeTry, DomainError>( function* (this: PostgresTableSchemaFieldCreateVisitor) { const rulesResult = createFieldSchemaRules(field, this.rulesContext); - const rules = yield* rulesResult; + const rules = yield* rulesResult.andThen((rules) => + this.addCreateTableColumnFromRules(field, rules) + ); const ctx = this.createRuleContext(field); const statementsResult = schemaRuleResolver.upAll(rules, ctx); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/TableSchemaUpdateVisitor.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/TableSchemaUpdateVisitor.ts index 560f9fd2eb..2a04938e5c 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/TableSchemaUpdateVisitor.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/TableSchemaUpdateVisitor.ts @@ -258,6 +258,24 @@ export class TableSchemaUpdateVisitor return this.resolveDbFieldNameText(tableFieldResult.value); } + private buildCurrentTableFieldMetadataById(): Result< + ReadonlyMap, + DomainError + > { + return safeTry, DomainError>( + function* (this: TableSchemaUpdateVisitor) { + const fieldsById = new Map(); + for (const field of this.params.table.getFields()) { + fieldsById.set(field.id().toString(), { + dbFieldName: yield* this.resolveDbFieldNameText(field), + tableId: this.params.tableId, + }); + } + return ok(fieldsById); + }.bind(this) + ); + } + /** * Build a conditional ALTER INDEX RENAME statement for a field's search index. * Only executes if the old index exists. @@ -674,6 +692,8 @@ export class TableSchemaUpdateVisitor tableId: visitor.params.tableId, dbFieldName, fieldId: newField.id().toString(), + tableLocationsById: visitor.params.tableLocationsById, + fieldsById: yield* visitor.buildCurrentTableFieldMetadataById(), }; const conversionStatements = yield* generateFieldConversionStatements( @@ -1356,6 +1376,9 @@ export class TableSchemaUpdateVisitor // Helper to build fully-qualified table name from fkHostTableName (which is "baseId.tableName") const quoteIdentifier = (name: string): string => `"${name.replaceAll('"', '""')}"`; + const quoteLiteral = (value: string): string => `'${value.replaceAll("'", "''")}'`; + const sqlNullableLiteral = (value: string | null | undefined): string => + value == null ? 'NULL' : quoteLiteral(value); const quoteTableName = (name: string): string => { if (!name.includes('.')) return quoteIdentifier(name); const [s, t] = name.split('.'); @@ -1432,6 +1455,76 @@ export class TableSchemaUpdateVisitor }, ]; }; + const parseTableIdentifier = (dbTableName: string): TableIdentifier => { + const [schemaName, ...rest] = dbTableName.split('.'); + if (rest.length === 0) { + return { schema: null, tableName: schemaName ?? dbTableName }; + } + return { schema: schemaName ?? null, tableName: rest.join('.') }; + }; + const fallbackForeignTableIdentifier = (tableId: string): TableIdentifier => + this.params.tableLocationsById?.get(tableId) ?? { + schema: this.params.schema, + tableName: tableId, + }; + const buildForeignTableDeclarations = (table: TableIdentifier | null): string => { + const schemaName = table?.schema ?? (table ? 'public' : null); + return [ + ` foreign_schema text := ${sqlNullableLiteral(schemaName)};`, + ` foreign_name text := ${sqlNullableLiteral(table?.tableName)};`, + ].join('\n'); + }; + const resolveCurrentTableFieldDbName = (fieldId: string): string | null => { + const field = this.params.table + ?.getFields() + .find((candidate) => candidate.id().toString() === fieldId); + if (!field) { + return null; + } + const dbFieldNameResult = this.resolveDbFieldNameText(field); + return dbFieldNameResult.isOk() ? dbFieldNameResult.value : null; + }; + const fetchFieldDbName = async ( + ctx: Parameters>[0], + fieldId: string + ): Promise => { + const rows = await sql<{ db_field_name: string | null }>` + SELECT db_field_name + FROM field + WHERE id = ${fieldId} AND deleted_time IS NULL + LIMIT 1 + `.execute(ctx.metaDb); + return rows.rows[0]?.db_field_name ?? null; + }; + const fetchTableIdentifier = async ( + ctx: Parameters>[0], + tableId: string + ): Promise => { + const rows = await sql<{ db_table_name: string | null }>` + SELECT db_table_name + FROM table_meta + WHERE id = ${tableId} AND deleted_time IS NULL + LIMIT 1 + `.execute(ctx.metaDb); + const dbTableName = rows.rows[0]?.db_table_name; + return dbTableName ? parseTableIdentifier(dbTableName) : null; + }; + const customDataStatement = ( + previewSql: string, + resolveSql: ( + ctx: Parameters>[0] + ) => Promise + ): TableSchemaStatementBuilder => ({ + scope: 'data', + compile: () => sql.raw(previewSql).compile(db), + execute: async (ctx) => { + const sqlText = await resolveSql(ctx); + if (!sqlText) { + return; + } + await sql.raw(sqlText).execute(ctx.dataDb); + }, + }); // For oneWay conversions that don't change storage type // (both manyMany and oneMany oneWay use junction table, or both manyOne and oneOne use FK): @@ -1691,44 +1784,28 @@ export class TableSchemaUpdateVisitor if (symmetricFieldId) { const nextRelationship = spec.nextRelationship().toString(); const symmetricIsMultiple = nextRelationship === 'manyOne'; - const escapedSymmetricFieldId = symmetricFieldId.toString().replace(/'/g, "''"); - const escapedForeignTableId = nextConfig.foreignTableId().toString().replace(/'/g, "''"); - const escapedFkColumnName = newFkColumnName.replace(/'/g, "''"); + const symmetricFieldIdString = symmetricFieldId.toString(); + const foreignTableIdString = nextConfig.foreignTableId().toString(); + const previewSymmetricColumn = resolveCurrentTableFieldDbName(symmetricFieldIdString); + const previewForeignTable = fallbackForeignTableIdentifier(foreignTableIdString); + const buildTrimSymmetricSql = ( + symmetricColumnName: string | null, + foreignTable: TableIdentifier | null + ): string | null => { + if (!symmetricColumnName || !foreignTable) { + return null; + } - const trimSymmetricSql = ` + return ` DO $v2_link_trim$ DECLARE - sym_col text; - foreign_tbl text; - foreign_schema text; - foreign_name text; + sym_col text := ${quoteLiteral(symmetricColumnName)}; +${buildForeignTableDeclarations(foreignTable)} BEGIN - IF to_regclass('public.field') IS NULL OR to_regclass('public.table_meta') IS NULL THEN - RETURN; - END IF; - - SELECT db_field_name INTO sym_col - FROM field - WHERE id = '${escapedSymmetricFieldId}' AND deleted_time IS NULL - LIMIT 1; - - SELECT db_table_name INTO foreign_tbl - FROM table_meta - WHERE id = '${escapedForeignTableId}' AND deleted_time IS NULL - LIMIT 1; - - IF sym_col IS NULL OR foreign_tbl IS NULL THEN + IF sym_col IS NULL OR foreign_schema IS NULL OR foreign_name IS NULL THEN RETURN; END IF; - IF strpos(foreign_tbl, '.') > 0 THEN - foreign_schema := split_part(foreign_tbl, '.', 1); - foreign_name := split_part(foreign_tbl, '.', 2); - ELSE - foreign_schema := 'public'; - foreign_name := foreign_tbl; - END IF; - IF ${symmetricIsMultiple ? 'TRUE' : 'FALSE'} THEN EXECUTE format( 'UPDATE %I.%I AS f @@ -1761,7 +1838,7 @@ BEGIN sym_col, sym_col, sym_col, - '${escapedFkColumnName}' + ${quoteLiteral(newFkColumnName)} ); ELSE EXECUTE format( @@ -1793,16 +1870,24 @@ BEGIN sym_col, sym_col, sym_col, - '${escapedFkColumnName}' + ${quoteLiteral(newFkColumnName)} ); END IF; END $v2_link_trim$;`; - - statements.push({ - scope: 'data', - compile: () => sql.raw(trimSymmetricSql).compile(db), - }); + }; + const previewTrimSql = + buildTrimSymmetricSql(previewSymmetricColumn, previewForeignTable) ?? + `DO $v2_link_trim$ BEGIN RETURN; END $v2_link_trim$;`; + + statements.push( + customDataStatement(previewTrimSql, async (ctx) => + buildTrimSymmetricSql( + await fetchFieldDbName(ctx, symmetricFieldIdString), + (await fetchTableIdentifier(ctx, foreignTableIdString)) ?? previewForeignTable + ) + ) + ); } statements.push(...buildLinkValueShapeRewriteStatements()); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.pglite.spec.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.pglite.spec.ts index 5c4f59526a..73473211f1 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.pglite.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.pglite.spec.ts @@ -4,9 +4,43 @@ * These tests validate that the generated SQL statements actually work * against a real PostgreSQL-compatible database using PGlite. */ -import { describe, it } from 'vitest'; +import type { Field } from '@teable/v2-core'; +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; +import { Kysely, sql } from 'kysely'; +import { describe, expect, it } from 'vitest'; + +import { executeTableSchemaStatements } from '../../../shared/db'; +import { FieldTypeConversionVisitorFactory } from '../FieldTypeConversionVisitor'; +import { createPGliteDb } from './helpers/createPGliteDb'; +import { createTextField, createUsrField } from './helpers/fieldFactories'; describe('FieldTypeConversionVisitor (PGlite)', () => { + const DB_FIELD_NAME = 'user_text'; + + const executeConversion = async ( + dataDb: Kysely, + metaDb: Kysely, + sourceField: Field, + targetField: Field + ) => { + const factory = new FieldTypeConversionVisitorFactory({ + db: dataDb, + schema: null, + tableName: 'records', + tableId: 'tblTestTable0000001', + dbFieldName: DB_FIELD_NAME, + }); + const visitorResult = sourceField.accept(factory); + expect(visitorResult.isOk()).toBe(true); + const statementsResult = targetField.accept(visitorResult._unsafeUnwrap()); + expect(statementsResult.isOk()).toBe(true); + await executeTableSchemaStatements(dataDb, statementsResult._unsafeUnwrap(), { + dataDb, + metaDb, + enforceRelationAccess: true, + }); + }; + describe('Text to Number', () => { it.todo( 'should convert valid numeric strings to numbers' @@ -91,6 +125,119 @@ describe('FieldTypeConversionVisitor (PGlite)', () => { }); describe('Text to User', () => { + it('should resolve matching text via meta DB and clear non-matching text', async () => { + const { db: dataDb } = await createPGliteDb(); + const { db: metaDb } = await createPGliteDb(); + try { + await sql`CREATE TABLE records (__id text PRIMARY KEY, user_text text)`.execute(dataDb); + await sql` + INSERT INTO records (__id, user_text) + VALUES + ('rec_email', 'alice@example.com'), + ('rec_name', 'Bob'), + ('rec_missing', 'random text value') + `.execute(dataDb); + await sql`CREATE TABLE users (id text PRIMARY KEY, name text NOT NULL, email text)`.execute( + metaDb + ); + await sql` + INSERT INTO users (id, name, email) + VALUES + ('usr_alice', 'Alice', 'alice@example.com'), + ('usr_bob', 'Bob', NULL) + `.execute(metaDb); + + await executeConversion( + dataDb, + metaDb, + createTextField('srcText', 'Source Text', DB_FIELD_NAME)._unsafeUnwrap(), + createUsrField('tgtUsr', 'Target User', DB_FIELD_NAME, false)._unsafeUnwrap() + ); + + const rows = await sql<{ __id: string; user_text: unknown | null }>` + SELECT __id, user_text + FROM records + ORDER BY __id + `.execute(dataDb); + + expect(rows.rows).toEqual([ + { + __id: 'rec_email', + user_text: { + id: 'usr_alice', + title: 'Alice', + email: 'alice@example.com', + avatarUrl: '/api/attachments/read/public/avatar/usr_alice', + }, + }, + { __id: 'rec_missing', user_text: null }, + { + __id: 'rec_name', + user_text: { + id: 'usr_bob', + title: 'Bob', + email: null, + avatarUrl: '/api/attachments/read/public/avatar/usr_bob', + }, + }, + ]); + } finally { + await dataDb.destroy(); + await metaDb.destroy(); + } + }); + + it('should create array format for multiple user field', async () => { + const { db: dataDb } = await createPGliteDb(); + const { db: metaDb } = await createPGliteDb(); + try { + await sql`CREATE TABLE records (__id text PRIMARY KEY, user_text text)`.execute(dataDb); + await sql`INSERT INTO records (__id, user_text) VALUES ('rec_multi', 'Alice, Bob')`.execute( + dataDb + ); + await sql`CREATE TABLE users (id text PRIMARY KEY, name text NOT NULL, email text)`.execute( + metaDb + ); + await sql` + INSERT INTO users (id, name, email) + VALUES + ('usr_alice', 'Alice', 'alice@example.com'), + ('usr_bob', 'Bob', NULL) + `.execute(metaDb); + + await executeConversion( + dataDb, + metaDb, + createTextField('srcText', 'Source Text', DB_FIELD_NAME)._unsafeUnwrap(), + createUsrField('tgtUsr', 'Target User', DB_FIELD_NAME, true)._unsafeUnwrap() + ); + + const rows = await sql<{ user_text: unknown }>` + SELECT user_text + FROM records + WHERE __id = 'rec_multi' + `.execute(dataDb); + + expect(rows.rows[0]?.user_text).toEqual([ + { + id: 'usr_alice', + title: 'Alice', + email: 'alice@example.com', + avatarUrl: '/api/attachments/read/public/avatar/usr_alice', + }, + { + id: 'usr_bob', + title: 'Bob', + email: null, + avatarUrl: '/api/attachments/read/public/avatar/usr_bob', + }, + ]); + } finally { + await dataDb.destroy(); + await metaDb.destroy(); + } + }); + it.todo( 'should match text against user email' // Setup: Create users table with test user, insert user email as text diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.spec.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.spec.ts index 1bcf3c41de..4ec0959ca8 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldTypeConversionVisitor.spec.ts @@ -100,6 +100,17 @@ const getVisitorSqls = (sourceField: Field, targetField: Field): string[] => { return statements.map((s) => s.compile(db as never).sql); }; +const getVisitorStatements = (sourceField: Field, targetField: Field) => { + const params = createParams(); + const factory = new FieldTypeConversionVisitorFactory(params); + const visitorResult = sourceField.accept(factory); + expect(visitorResult.isOk()).toBe(true); + const visitor = visitorResult._unsafeUnwrap(); + const statementsResult = targetField.accept(visitor); + expect(statementsResult.isOk()).toBe(true); + return statementsResult._unsafeUnwrap(); +}; + // ----- Field factory helpers ----- const mkTextField = () => createTextField('srcText', 'Source Text', DB_FIELD_NAME)._unsafeUnwrap(); @@ -327,7 +338,7 @@ describe('FieldTypeConversionVisitor', () => { const optionsSql = sqls[0]; expect(optionsSql).toContain('distinct_values'); expect(optionsSql).toContain('DISTINCT'); - expect(optionsSql).toContain('UPDATE field'); + expect(optionsSql).not.toContain('UPDATE field'); }); it('should not generate type conversion statement (same DB type)', () => { @@ -342,7 +353,7 @@ describe('FieldTypeConversionVisitor', () => { it('should generate CSV-aware SQL to split text into jsonb array (V1 parity)', () => { const sqls = getVisitorSqls(mkTextField(), mkMultiSelField()); // Should use regexp_matches with CSV-aware pattern instead of regexp_split_to_table - const updateSql = sqls.find((s) => s.includes('regexp_matches')); + const updateSql = sqls.find((s) => s.includes('jsonb_agg')); expect(updateSql).toBeDefined(); expect(updateSql).toContain('jsonb_agg'); expect(updateSql).toContain('COALESCE(trim(m[1]), trim(m[2]))'); @@ -361,19 +372,26 @@ describe('FieldTypeConversionVisitor', () => { }); describe('text -> user', () => { - it('should generate SQL to match text against collaborators by id, name, or email', () => { - const sqls = getVisitorSqls(mkTextField(), mkUsrField(false)); + it('should execute as a custom data statement without joining meta-plane users or collaborators in data SQL', () => { + const statements = getVisitorStatements(mkTextField(), mkUsrField(false)); + const sqls = statements.map((s) => s.compile(db as never).sql); + expect(statements[0]?.scope).toBe('data'); + expect(statements[0]?.execute).toBeDefined(); expect(sqls.length).toBeGreaterThanOrEqual(2); - // First statement is the DO block for user matching const matchSql = sqls[0]; - expect(matchSql).toContain('collab_users'); - expect(matchSql).toContain('matched_users'); + expect(matchSql).toContain('text_parts'); + expect(matchSql).toContain('jsonb_build_object'); + expect(matchSql).not.toContain('collaborator'); + expect(matchSql).not.toMatch(/\b(from|join)\s+(public\.)?users\b/i); + expect(matchSql).not.toContain('table_meta'); + expect(matchSql).not.toContain('base'); }); it('should generate SQL for single user format (object)', () => { const sqls = getVisitorSqls(mkTextField(), mkUsrField(false)); const matchSql = sqls[0]; expect(matchSql).toContain('jsonb_build_object'); + expect(matchSql).toContain('SELECT DISTINCT ON (rid)'); }); it('should generate SQL for multiple user format (array of objects)', () => { @@ -1032,49 +1050,59 @@ describe('FieldTypeConversionVisitor', () => { it('should generate SQL to filter out existing options', () => { const sqls = getVisitorSqls(mkTextField(), mkSingleSelField()); - const optionsSql = sqls.find((s) => s.includes('existing_choices')); + const optionsSql = sqls.find((s) => s.includes('distinct_values')); expect(optionsSql).toBeDefined(); - expect(optionsSql).toContain('NOT IN'); - expect(optionsSql).toContain('existing_names'); + expect(optionsSql).not.toContain('existing_choices'); + expect(optionsSql).not.toContain('UPDATE field'); }); it('should guard generated select option names against the configured max length', () => { const sqls = getVisitorSqls(mkTextField(), mkSingleSelField()); - const optionsSql = sqls.find((s) => s.includes('oversized_guard')); + const optionsSql = sqls.find((s) => s.includes('oversized_values')); expect(optionsSql).toBeDefined(); expect(optionsSql).toContain('char_length(name)'); expect(optionsSql).toContain('oversized_values'); - expect(optionsSql).toContain("COALESCE((SELECT '' FROM oversized_values"); - expect(optionsSql).toContain('THEN CAST('); - expect(optionsSql).toMatch(/\$\d+\s+\|\| COALESCE/); - expect(optionsSql).toContain('AS integer'); + expect(optionsSql).toContain('NOT EXISTS'); }); - it('should generate SQL with random ID generation', () => { + it('should generate options through a custom executor instead of data-scope metadata SQL', () => { + const params = createParams(); + const factory = new FieldTypeConversionVisitorFactory(params); + const visitorResult = mkTextField().accept(factory); + expect(visitorResult.isOk()).toBe(true); + const statementsResult = mkSingleSelField().accept(visitorResult._unsafeUnwrap()); + expect(statementsResult.isOk()).toBe(true); + const optionStatement = statementsResult + ._unsafeUnwrap() + .find((statement) => statement.compile(db as never).sql.includes('distinct_values')); + expect(optionStatement).toBeDefined(); + expect(optionStatement?.scope).toBe('meta'); + expect(optionStatement?.execute).toBeTypeOf('function'); + }); + + it('should not generate random IDs in data-scope SQL', () => { const sqls = getVisitorSqls(mkTextField(), mkSingleSelField()); - const optionsSql = sqls.find((s) => s.includes('cho')); + const optionsSql = sqls.find((s) => s.includes('distinct_values')); expect(optionsSql).toBeDefined(); - expect(optionsSql).toContain("'cho'"); - expect(optionsSql).toContain('md5(random()'); - expect(optionsSql).toContain('substr'); + expect(optionsSql).not.toContain("'cho'"); + expect(optionsSql).not.toContain('md5(random()'); + expect(optionsSql).not.toContain('substr'); }); - it('should generate SQL with color assignment from palette', () => { + it('should not generate color assignment in data-scope SQL', () => { const sqls = getVisitorSqls(mkTextField(), mkSingleSelField()); - const optionsSql = sqls.find((s) => s.includes('color')); + const optionsSql = sqls.find((s) => s.includes('distinct_values')); expect(optionsSql).toBeDefined(); - expect(optionsSql).toContain('ARRAY['); - // Colors are parameterized by Kysely (e.g., $2, $3, ...) rather than literal strings - expect(optionsSql).toContain("'color'"); - expect(optionsSql).toMatch(/\(rn % \$\d+\) \+ 1/); + expect(optionsSql).not.toContain('ARRAY['); + expect(optionsSql).not.toContain("'color'"); }); - it('should generate SQL to merge with existing choices', () => { + it('should not merge metadata choices in data-scope SQL', () => { const sqls = getVisitorSqls(mkTextField(), mkSingleSelField()); - const optionsSql = sqls.find((s) => s.includes('merged_choices')); + const optionsSql = sqls.find((s) => s.includes('distinct_values')); expect(optionsSql).toBeDefined(); - expect(optionsSql).toContain('COALESCE'); - expect(optionsSql).toContain("'[]'::jsonb"); + expect(optionsSql).not.toContain('merged_choices'); + expect(optionsSql).not.toContain('UPDATE field'); }); it('should return null if fieldId is not provided', () => { diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldValueChangeCollectorVisitor.spec.ts b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldValueChangeCollectorVisitor.spec.ts index 44ade35fd2..578859486b 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldValueChangeCollectorVisitor.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/FieldValueChangeCollectorVisitor.spec.ts @@ -208,6 +208,7 @@ describe('FieldValueChangeCollectorVisitor', () => { spec.accept(visitor); expect(visitor.valueChangedFields().map((id) => id.toString())).toEqual([fieldId.toString()]); + expect(visitor.recordActionFields()).toEqual([]); }); it('should collect valueChangedFieldIds when options are removed', () => { @@ -225,6 +226,7 @@ describe('FieldValueChangeCollectorVisitor', () => { spec.accept(visitor); expect(visitor.valueChangedFields().map((id) => id.toString())).toEqual([fieldId.toString()]); + expect(visitor.recordActionFields().map((id) => id.toString())).toEqual([fieldId.toString()]); }); it('should not collect anything when only options are added', () => { @@ -243,6 +245,7 @@ describe('FieldValueChangeCollectorVisitor', () => { spec.accept(visitor); expect(visitor.valueChangedFields()).toEqual([]); + expect(visitor.recordActionFields()).toEqual([]); expect(visitor.selfBackfillFields()).toEqual([]); }); }); @@ -275,6 +278,7 @@ describe('FieldValueChangeCollectorVisitor', () => { spec.accept(visitor); expect(visitor.valueChangedFields().map((id) => id.toString())).toEqual([fieldId.toString()]); + expect(visitor.recordActionFields()).toEqual([]); }); it('should collect valueChangedFieldIds when options are removed', () => { @@ -292,6 +296,7 @@ describe('FieldValueChangeCollectorVisitor', () => { spec.accept(visitor); expect(visitor.valueChangedFields().map((id) => id.toString())).toEqual([fieldId.toString()]); + expect(visitor.recordActionFields().map((id) => id.toString())).toEqual([fieldId.toString()]); }); }); diff --git a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/__snapshots__/TableSchemaUpdateVisitor.spec.ts.snap b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/__snapshots__/TableSchemaUpdateVisitor.spec.ts.snap index 2a1b9f40db..a4514f64aa 100644 --- a/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/__snapshots__/TableSchemaUpdateVisitor.spec.ts.snap +++ b/packages/v2/adapter-table-repository-postgres/src/schema/visitors/__tests__/__snapshots__/TableSchemaUpdateVisitor.spec.ts.snap @@ -27,108 +27,6 @@ exports[`TableSchemaUpdateVisitor > visitUpdateLinkRelationship > should generat WHERE j."__fk_fldSrcField00000001" = h."__id" )", "DROP TABLE IF EXISTS "bseTestBase000000001"."junction_fldSrcField00000001_fldSymField00000001" CASCADE", - " -DO $v2_link_trim$ -DECLARE - sym_col text; - foreign_tbl text; - foreign_schema text; - foreign_name text; -BEGIN - IF to_regclass('public.field') IS NULL OR to_regclass('public.table_meta') IS NULL THEN - RETURN; - END IF; - - SELECT db_field_name INTO sym_col - FROM field - WHERE id = 'fldSymField00000001' AND deleted_time IS NULL - LIMIT 1; - - SELECT db_table_name INTO foreign_tbl - FROM table_meta - WHERE id = 'tblForeignTbl000001' AND deleted_time IS NULL - LIMIT 1; - - IF sym_col IS NULL OR foreign_tbl IS NULL THEN - RETURN; - END IF; - - IF strpos(foreign_tbl, '.') > 0 THEN - foreign_schema := split_part(foreign_tbl, '.', 1); - foreign_name := split_part(foreign_tbl, '.', 2); - ELSE - foreign_schema := 'public'; - foreign_name := foreign_tbl; - END IF; - - IF FALSE THEN - EXECUTE format( - 'UPDATE %I.%I AS f - SET %I = ( - SELECT CASE - WHEN filtered.value IS NULL OR jsonb_array_length(filtered.value) = 0 THEN NULL - ELSE filtered.value - END - FROM ( - SELECT jsonb_agg(elem) AS value - FROM jsonb_array_elements( - CASE - WHEN f.%I IS NULL THEN ''[]''::jsonb - WHEN jsonb_typeof(f.%I::jsonb) = ''array'' THEN f.%I::jsonb - ELSE jsonb_build_array(f.%I::jsonb) - END - ) AS elem - WHERE EXISTS ( - SELECT 1 - FROM "bseTestBase000000001"."tblForeignTbl000001" AS h - WHERE h.%I = f.__id - AND h.__id = elem->>''id'' - ) - ) AS filtered - )', - foreign_schema, - foreign_name, - sym_col, - sym_col, - sym_col, - sym_col, - sym_col, - '__fk_fldSymField00000001' - ); - ELSE - EXECUTE format( - 'UPDATE %I.%I AS f - SET %I = ( - SELECT filtered.elem - FROM ( - SELECT elem - FROM jsonb_array_elements( - CASE - WHEN f.%I IS NULL THEN ''[]''::jsonb - WHEN jsonb_typeof(f.%I::jsonb) = ''array'' THEN f.%I::jsonb - ELSE jsonb_build_array(f.%I::jsonb) - END - ) AS elem - WHERE EXISTS ( - SELECT 1 - FROM "bseTestBase000000001"."tblForeignTbl000001" AS h - WHERE h.%I = f.__id - AND h.__id = elem->>''id'' - ) - LIMIT 1 - ) AS filtered - )', - foreign_schema, - foreign_name, - sym_col, - sym_col, - sym_col, - sym_col, - sym_col, - '__fk_fldSymField00000001' - ); - END IF; -END -$v2_link_trim$;", + "DO $v2_link_trim$ BEGIN RETURN; END $v2_link_trim$;", ] `; diff --git a/packages/v2/adapter-table-repository-postgres/src/shared/db.spec.ts b/packages/v2/adapter-table-repository-postgres/src/shared/db.spec.ts index 2e9bc62775..8c624c099c 100644 --- a/packages/v2/adapter-table-repository-postgres/src/shared/db.spec.ts +++ b/packages/v2/adapter-table-repository-postgres/src/shared/db.spec.ts @@ -1,5 +1,6 @@ import type { ISpan, ITracer, SpanAttributes, SpanAttributeValue } from '@teable/v2-core'; import type { CompiledQuery, Kysely } from 'kysely'; +import { sql } from 'kysely'; import { describe, expect, it, vi } from 'vitest'; import type { TableSchemaStatementBuilder } from '../schema/rules/core'; @@ -63,6 +64,30 @@ const statement = ( compile: vi.fn(() => compiled), }); +const executorDb = (compiledSql: string | ReadonlyArray) => { + const compiledSqls = Array.isArray(compiledSql) ? [...compiledSql] : [compiledSql]; + let compileIndex = 0; + const executeQuery = vi.fn(async () => ({ rows: [] })); + const executor = { + transformQuery: vi.fn((node) => node), + compileQuery: vi.fn(() => { + const sql = compiledSqls[compileIndex] ?? compiledSqls.at(-1) ?? 'select 1'; + compileIndex += 1; + return compiledQuery(sql); + }), + executeQuery, + withPlugins: vi.fn(() => executor), + }; + + return { + executeQuery: vi.fn(async () => ({ rows: [] })), + getExecutor: vi.fn(() => executor), + executorExecuteQuery: executeQuery, + } as unknown as Kysely & { + executorExecuteQuery: ReturnType; + }; +}; + describe('executeTableSchemaStatements', () => { it('creates timing spans for each schema statement when a tracer is provided', async () => { const db = { @@ -134,6 +159,27 @@ describe('executeTableSchemaStatements', () => { expect(db.executeQuery).not.toHaveBeenCalled(); }); + it('rejects data-scoped statements that access user metadata relations', async () => { + const db = { + executeQuery: vi.fn(async () => ({ rows: [] })), + } as unknown as Kysely; + + await expect( + executeTableSchemaStatements( + db, + [ + statement( + compiledQuery( + 'select u.id from public.users u join collaborator c on c.principal_id = u.id' + ) + ), + ], + { enforceRelationAccess: true } + ) + ).rejects.toThrow('cannot access relations owned by another storage plane'); + expect(db.executeQuery).not.toHaveBeenCalled(); + }); + it('allows metadata-scoped statements to access metadata relations', async () => { const db = { executeQuery: vi.fn(async () => ({ rows: [] })), @@ -145,4 +191,48 @@ describe('executeTableSchemaStatements', () => { expect(db.executeQuery).toHaveBeenCalledTimes(1); }); + + it('rejects custom data executor statements that access metadata relations', async () => { + const dataDb = executorDb('select * from field'); + const metaDb = executorDb('select * from field'); + const customStatement: TableSchemaStatementBuilder = { + scope: 'data', + compile: vi.fn(() => compiledQuery('select 1')), + execute: async ({ dataDb }) => { + await sql.raw('select * from field').execute(dataDb); + }, + }; + + await expect( + executeTableSchemaStatements(dataDb, [customStatement], { + dataDb, + metaDb, + enforceRelationAccess: true, + }) + ).rejects.toThrow('cannot access relations owned by another storage plane'); + expect(dataDb.executorExecuteQuery).not.toHaveBeenCalled(); + expect(metaDb.executorExecuteQuery).not.toHaveBeenCalled(); + }); + + it('allows custom statements to use each storage plane through the matching executor', async () => { + const dataDb = executorDb('select * from "tblData"'); + const metaDb = executorDb('select * from field'); + const customStatement: TableSchemaStatementBuilder = { + scope: 'meta', + compile: vi.fn(() => compiledQuery('select * from field')), + execute: async ({ dataDb, metaDb }) => { + await sql.raw('select * from "tblData"').execute(dataDb); + await sql.raw('select * from field').execute(metaDb); + }, + }; + + await executeTableSchemaStatements(metaDb, [customStatement], { + dataDb, + metaDb, + enforceRelationAccess: true, + }); + + expect(dataDb.executorExecuteQuery).toHaveBeenCalledTimes(1); + expect(metaDb.executorExecuteQuery).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/v2/adapter-table-repository-postgres/src/shared/db.ts b/packages/v2/adapter-table-repository-postgres/src/shared/db.ts index 1d9da97f3c..29d2219ec6 100644 --- a/packages/v2/adapter-table-repository-postgres/src/shared/db.ts +++ b/packages/v2/adapter-table-repository-postgres/src/shared/db.ts @@ -1,8 +1,11 @@ import type { ISpan, ITracer, SpanAttributes } from '@teable/v2-core'; -import type { Kysely, Transaction, CompiledQuery } from 'kysely'; +import type { Kysely, Transaction, CompiledQuery, QueryResult } from 'kysely'; import type { TableSchemaStatementBuilder } from '../schema/rules/core'; -import { assertSchemaStatementRelationAccess } from '../schema/rules/core'; +import { + assertSchemaStatementRelationAccess, + findSchemaStatementRelationAccessViolations, +} from '../schema/rules/core'; export { getPostgresTransaction, resolvePostgresDbOrTx, @@ -12,8 +15,12 @@ type ExecuteQueryTraceOptions = { readonly tracer?: ITracer; readonly attributes?: SpanAttributes; readonly enforceRelationAccess?: boolean; + readonly dataDb?: Kysely | Transaction; + readonly metaDb?: Kysely | Transaction; }; +type KyselyQueryExecutor = ReturnType['getExecutor']>; + const schemaStatementSpanName = 'teable.postgres.schema_statement.execute'; const describeError = (error: unknown): string => { @@ -88,6 +95,130 @@ const executeCompiledQuery = async ( } }; +const executeCustomTableSchemaStatement = async ( + db: Kysely | Transaction, + statement: TableSchemaStatementBuilder, + statementIndex: number, + statementCount: number, + trace?: ExecuteQueryTraceOptions +): Promise => { + const statementForScope = ( + scope: TableSchemaStatementBuilder['scope'] + ): TableSchemaStatementBuilder => + scope === statement.scope + ? statement + : { + ...statement, + scope, + }; + const scopedExecutor = ( + executor: Kysely | Transaction, + scope: TableSchemaStatementBuilder['scope'] + ): Kysely | Transaction => { + if (!trace?.enforceRelationAccess) { + return executor; + } + + const scopedStatement = statementForScope(scope); + const assertAccess = (compiledQuery: CompiledQuery) => { + const violations = findSchemaStatementRelationAccessViolations( + scopedStatement, + compiledQuery + ); + if (violations.length > 0) { + assertSchemaStatementRelationAccess(scopedStatement, compiledQuery); + } + }; + const wrapExecutor = (executor: KyselyQueryExecutor): KyselyQueryExecutor => + new Proxy(executor, { + get(target, prop, receiver) { + if (prop === 'executeQuery') { + return async (compiledQuery: CompiledQuery): Promise> => { + assertAccess(compiledQuery); + return target.executeQuery(compiledQuery); + }; + } + + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + + return new Proxy(executor, { + get(target, prop, receiver) { + if (prop === 'executeQuery') { + return async (compiledQuery: CompiledQuery): Promise> => { + assertAccess(compiledQuery); + return target.executeQuery(compiledQuery); + }; + } + if (prop === 'getExecutor') { + return () => wrapExecutor(target.getExecutor()); + } + + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }; + + const execute = async () => { + if (trace?.enforceRelationAccess) { + assertSchemaStatementRelationAccess(statement, statement.compile(db)); + } + + await statement.execute?.({ + scopedDb: scopedExecutor(db as Kysely | Transaction, statement.scope), + dataDb: scopedExecutor( + (trace?.dataDb ?? db) as Kysely | Transaction, + 'data' + ), + metaDb: scopedExecutor( + (trace?.metaDb ?? db) as Kysely | Transaction, + 'meta' + ), + }); + }; + + const tracer = trace?.tracer; + if (!tracer) { + await execute(); + return; + } + + let span: ISpan; + try { + span = tracer.startSpan(schemaStatementSpanName, { + ...trace.attributes, + 'db.system': 'postgresql', + 'db.operation': 'CUSTOM', + 'teable.db.statement.index': statementIndex, + 'teable.db.statement.count': statementCount, + 'teable.db.statement.sql_bytes': 0, + 'teable.db.statement.parameter_count': 0, + }); + } catch { + await execute(); + return; + } + + const startedAt = Date.now(); + try { + await tracer.withSpan(span, async () => { + try { + await execute(); + } catch (error) { + span.recordError(describeError(error)); + throw error; + } finally { + span.setAttribute('teable.db.statement.duration_ms', Date.now() - startedAt); + } + }); + } finally { + span.end(); + } +}; + export const executeCompiledQueries = async ( db: Kysely | Transaction, compiled: ReadonlyArray, @@ -103,13 +234,16 @@ export const executeTableSchemaStatements = async ( statements: ReadonlyArray, trace?: ExecuteQueryTraceOptions ): Promise => { - const compiled = statements.map((statement) => { + for (const [index, statement] of statements.entries()) { + if (statement.execute) { + await executeCustomTableSchemaStatement(db, statement, index, statements.length, trace); + continue; + } + const compiledQuery = statement.compile(db); if (trace?.enforceRelationAccess) { assertSchemaStatementRelationAccess(statement, compiledQuery); } - return compiledQuery; - }); - - await executeCompiledQueries(db, compiled, trace); + await executeCompiledQuery(db, compiledQuery, index, statements.length, trace); + } }; diff --git a/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.spec.ts b/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.spec.ts new file mode 100644 index 0000000000..09ab12e27e --- /dev/null +++ b/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.spec.ts @@ -0,0 +1,153 @@ +import { + Kysely, + PostgresAdapter, + PostgresIntrospector, + PostgresQueryCompiler, + type CompiledQuery, + type DatabaseConnection, + type Driver, + type QueryResult, +} from 'kysely'; +import { describe, expect, it } from 'vitest'; + +import { toQualifiedIdentifierLiteral } from './sqlIdentifiers'; +import { ensureUndoCaptureInfrastructure } from './undoCapture'; + +type RowProvider = (compiledQuery: CompiledQuery) => unknown[]; + +class RecordingConnection implements DatabaseConnection { + constructor( + private readonly queries: CompiledQuery[], + private readonly rowProvider: RowProvider + ) {} + + async executeQuery(compiledQuery: CompiledQuery): Promise> { + this.queries.push(compiledQuery); + return { rows: this.rowProvider(compiledQuery) as R[] }; + } + + async *streamQuery(): AsyncIterableIterator> { + yield { rows: [] }; + } +} + +class RecordingDriver implements Driver { + readonly queries: CompiledQuery[] = []; + + constructor(private readonly rowProvider: RowProvider) {} + + async init(): Promise { + return undefined; + } + + async acquireConnection(): Promise { + return new RecordingConnection(this.queries, this.rowProvider); + } + + async beginTransaction(): Promise { + return undefined; + } + + async commitTransaction(): Promise { + return undefined; + } + + async rollbackTransaction(): Promise { + return undefined; + } + + async releaseConnection(): Promise { + return undefined; + } + + async destroy(): Promise { + return undefined; + } +} + +const defaultRowsForUndoProbe: RowProvider = (compiledQuery) => { + const text = compiledQuery.sql; + if (text.includes('FROM information_schema.tables')) { + return [{ exists: true }]; + } + if (text.includes('FROM information_schema.columns')) { + return [{ exists: true }]; + } + if (text.includes('FROM pg_proc')) { + return [{ exists: true }]; + } + if (text.includes('FROM pg_trigger AS t')) { + return [{ exists: true }]; + } + return []; +}; + +const createRecordingDb = (rowProvider?: RowProvider) => { + const driver = new RecordingDriver((compiledQuery) => { + const providedRows = rowProvider?.(compiledQuery) ?? []; + return providedRows.length > 0 ? providedRows : defaultRowsForUndoProbe(compiledQuery); + }); + const db = new Kysely({ + dialect: { + createAdapter: () => new PostgresAdapter(), + createDriver: () => driver, + createIntrospector: (kysely) => new PostgresIntrospector(kysely), + createQueryCompiler: () => new PostgresQueryCompiler(), + }, + }); + return { db, driver }; +}; + +describe('ensureUndoCaptureInfrastructure', () => { + it('accepts an existing undo trigger only when it targets the current schema function', async () => { + const { db, driver } = createRecordingDb(); + try { + const result = await ensureUndoCaptureInfrastructure( + db, + db, + toQualifiedIdentifierLiteral('bseMigrated', 'tblRecords'), + 'bseMigrated.tblRecords' + ); + + expect(result).toBe('ready'); + const triggerProbe = driver.queries.find((query) => + query.sql.includes('FROM pg_trigger AS t') + ); + expect(triggerProbe?.sql).toContain('JOIN pg_proc AS p ON p.oid = t.tgfoid'); + expect(triggerProbe?.sql).toContain('p.pronamespace = current_schema()::regnamespace'); + expect( + driver.queries.some((query) => + query.sql.includes('CREATE OR REPLACE TRIGGER "__teable_undo_capture"') + ) + ).toBe(false); + } finally { + await db.destroy(); + } + }); + + it('reinstalls stale restored triggers that point outside the current schema', async () => { + const { db, driver } = createRecordingDb((compiledQuery) => { + if (compiledQuery.sql.includes('FROM pg_trigger AS t')) { + return [{ exists: false }]; + } + return []; + }); + try { + const result = await ensureUndoCaptureInfrastructure( + db, + db, + toQualifiedIdentifierLiteral('bseMigrated', 'tblRecords'), + 'bseMigrated.tblRecords' + ); + + expect(result).toBe('ready'); + expect( + driver.queries.some((query) => + query.sql.includes('CREATE OR REPLACE TRIGGER "__teable_undo_capture"') + ) + ).toBe(true); + } finally { + await db.destroy(); + } + }); +}); diff --git a/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.ts b/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.ts index 9d6941c59a..ce86a7d9d4 100644 --- a/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.ts +++ b/packages/v2/adapter-table-repository-postgres/src/shared/undoCapture.ts @@ -161,10 +161,13 @@ const hasUndoCaptureTrigger = async (db: Kysely, tableKey: string): Prom FROM pg_trigger AS t JOIN pg_class AS c ON c.oid = t.tgrelid JOIN pg_namespace AS n ON n.oid = c.relnamespace + JOIN pg_proc AS p ON p.oid = t.tgfoid WHERE NOT t.tgisinternal AND t.tgname = '__teable_undo_capture' AND n.nspname = ${schema} AND c.relname = ${table} + AND p.proname = '__teable_capture_undo_row' + AND p.pronamespace = current_schema()::regnamespace ) AS "exists" `.execute(db); diff --git a/packages/v2/contract-http/src/shared/neverthrow.ts b/packages/v2/contract-http/src/shared/neverthrow.ts index 50f6d2d2b2..0507c70488 100644 --- a/packages/v2/contract-http/src/shared/neverthrow.ts +++ b/packages/v2/contract-http/src/shared/neverthrow.ts @@ -1,11 +1,14 @@ import type { DomainError } from '@teable/v2-core'; -import { ok } from 'neverthrow'; +import { err, ok } from 'neverthrow'; import type { Result } from 'neverthrow'; export const sequenceResults = ( values: ReadonlyArray> -): Result, DomainError> => - values.reduce, DomainError>>( - (acc, next) => acc.andThen((arr) => next.map((v) => [...arr, v])), - ok([]) - ); +): Result, DomainError> => { + const result: T[] = []; + for (const value of values) { + if (value.isErr()) return err, DomainError>(value.error); + result.push(value.value); + } + return ok(result); +}; diff --git a/packages/v2/core/src/application/projections/FieldCreatedRealtimeProjection.ts b/packages/v2/core/src/application/projections/FieldCreatedRealtimeProjection.ts index 0a90dac404..adac127892 100644 --- a/packages/v2/core/src/application/projections/FieldCreatedRealtimeProjection.ts +++ b/packages/v2/core/src/application/projections/FieldCreatedRealtimeProjection.ts @@ -5,7 +5,6 @@ import type { Result } from 'neverthrow'; import type { DomainError } from '../../domain/shared/DomainError'; import { domainError } from '../../domain/shared/DomainError'; import { FieldCreated } from '../../domain/table/events/FieldCreated'; -import { Table } from '../../domain/table/Table'; import type { IEventHandler } from '../../ports/EventHandler'; import type * as ExecutionContextPort from '../../ports/ExecutionContext'; import * as TableMapperPort from '../../ports/mappers/TableMapper'; @@ -14,6 +13,8 @@ import * as RealtimeEnginePort from '../../ports/RealtimeEngine'; import * as TableRepositoryPort from '../../ports/TableRepository'; import { v2CoreTokens } from '../../ports/tokens'; import { ProjectionHandler } from './Projection'; +import { loadRealtimeTableSnapshot } from './RealtimeTableSnapshotCache'; +import { scheduleRealtimeProjection } from './scheduleRealtimeProjection'; const tableCollectionPrefix = 'tbl'; const fieldCollectionPrefix = 'fld'; @@ -36,37 +37,45 @@ export class FieldCreatedRealtimeProjection implements IEventHandler> { const { realtimeEngine, tableRepository, tableMapper } = this; - return safeTry(async function* () { - // Fetch table data from repository - const spec = yield* Table.specs(event.baseId).byId(event.tableId).build().safeUnwrap(); - const table = yield* (await tableRepository.findOne(context, spec)).safeUnwrap(); - const snapshot = yield* tableMapper.toDTO(table).safeUnwrap(); + return scheduleRealtimeProjection(context, FieldCreatedRealtimeProjection.name, (context) => + safeTry(async function* () { + const snapshot = yield* ( + await loadRealtimeTableSnapshot(context, { + baseId: event.baseId, + tableId: event.tableId, + tableRepository, + tableMapper, + isSnapshotUsable: (candidate) => + candidate.fields.some((field) => field.id === event.fieldId.toString()), + }) + ).safeUnwrap(); - // Ensure table document exists (for tables created before realtime was enabled) - const tableCollection = `${tableCollectionPrefix}_${event.baseId.toString()}`; - const tableDocId = yield* RealtimeDocId.fromParts( - tableCollection, - event.tableId.toString() - ).safeUnwrap(); - yield* (await realtimeEngine.ensure(context, tableDocId, snapshot)).safeUnwrap(); + // Ensure table document exists (for tables created before realtime was enabled) + const tableCollection = `${tableCollectionPrefix}_${event.baseId.toString()}`; + const tableDocId = yield* RealtimeDocId.fromParts( + tableCollection, + event.tableId.toString() + ).safeUnwrap(); + yield* (await realtimeEngine.ensure(context, tableDocId, snapshot)).safeUnwrap(); - // Create field document - const fieldDto = snapshot.fields.find((field) => field.id === event.fieldId.toString()); - if (!fieldDto) { - return err( - domainError.validation({ - message: `Missing field snapshot for ${event.fieldId.toString()}`, - }) - ); - } + // Create field document + const fieldDto = snapshot.fields.find((field) => field.id === event.fieldId.toString()); + if (!fieldDto) { + return err( + domainError.validation({ + message: `Missing field snapshot for ${event.fieldId.toString()}`, + }) + ); + } - const fieldCollection = `${fieldCollectionPrefix}_${event.tableId.toString()}`; - const fieldDocId = yield* RealtimeDocId.fromParts( - fieldCollection, - event.fieldId.toString() - ).safeUnwrap(); + const fieldCollection = `${fieldCollectionPrefix}_${event.tableId.toString()}`; + const fieldDocId = yield* RealtimeDocId.fromParts( + fieldCollection, + event.fieldId.toString() + ).safeUnwrap(); - return realtimeEngine.ensure(context, fieldDocId, fieldDto); - }); + return realtimeEngine.ensure(context, fieldDocId, fieldDto); + }) + ); } } diff --git a/packages/v2/core/src/application/projections/FieldRealtimeShapeRefresh.ts b/packages/v2/core/src/application/projections/FieldRealtimeShapeRefresh.ts index e85d9b9f39..aaefb8c921 100644 --- a/packages/v2/core/src/application/projections/FieldRealtimeShapeRefresh.ts +++ b/packages/v2/core/src/application/projections/FieldRealtimeShapeRefresh.ts @@ -31,7 +31,17 @@ const dynamicShapeKeys = [ ] as const; type DynamicShapeKey = (typeof dynamicShapeKeys)[number]; -type DynamicShapeSnapshot = Record; + +// Lookup fields persist their resolved inner field's metadata (`type`/`options`/`dbFieldType`) at +// the document root (v1 format). A lookupOptions change can re-resolve the inner field, so the +// realtime op must refresh these too — but ONLY for lookup fields. They are intentionally kept OUT +// of the fixed `dynamicShapeKeys` list, because that list emits a `set` for every key (including +// null), which would wipe these on every other field type's shape refresh. +const lookupRootShapeKeys = ['type', 'options', 'dbFieldType'] as const; +type LookupRootShapeKey = (typeof lookupRootShapeKeys)[number]; + +type DynamicShapeSnapshot = Record & + Partial>; const hasOwn = (value: object, key: string): boolean => Object.prototype.hasOwnProperty.call(value, key); @@ -48,7 +58,10 @@ const emptyDynamicShapeSnapshot = (): DynamicShapeSnapshot => ({ innerOptions: null, }); -const readShapeValue = (fieldDto: ITableFieldPersistenceDTO, key: DynamicShapeKey): unknown => { +const readShapeValue = ( + fieldDto: ITableFieldPersistenceDTO, + key: DynamicShapeKey | LookupRootShapeKey +): unknown => { if (!hasOwn(fieldDto, key)) { return null; } @@ -131,6 +144,11 @@ class FieldRealtimeShapeSnapshotVisitor { isLookup: readShapeValue(fieldDto, 'isLookup'), isConditionalLookup: readShapeValue(fieldDto, 'isConditionalLookup'), lookupOptions: readShapeValue(fieldDto, 'lookupOptions'), + // A regular lookup stores its resolved inner field metadata at the root; changing the + // looked-up field can change these, so refresh them alongside lookupOptions. + type: readShapeValue(fieldDto, 'type'), + options: readShapeValue(fieldDto, 'options'), + dbFieldType: readShapeValue(fieldDto, 'dbFieldType'), }; } @@ -177,7 +195,7 @@ export const buildFieldShapeRefreshChanges = ( return []; } - return dynamicShapeKeys.flatMap((key) => { + const changes: RealtimeChange[] = dynamicShapeKeys.flatMap((key) => { const path = [key]; const pathKey = JSON.stringify(path); if (seenPaths.has(pathKey)) { @@ -193,4 +211,32 @@ export const buildFieldShapeRefreshChanges = ( }, ]; }); + + // Lookup fields additionally refresh their root inner-field metadata. These keys are present only + // on the lookup snapshot branch, so other field types are unaffected, and they are deduped against + // the main projection loop via `seenPaths`. + // + // - `options`: refreshed even when absent — emit {} to CLEAR any stale inner options (e.g. number + // formatting) left from the previous target, including when the field went pending. + // - `type` / `dbFieldType`: required scalars — only refreshed when the snapshot carries a value, + // so a transiently-pending lookup never publishes a null that would corrupt these fields. + for (const key of lookupRootShapeKeys) { + if (!(key in snapshot)) { + continue; + } + const path = [key]; + const pathKey = JSON.stringify(path); + if (seenPaths.has(pathKey)) { + continue; + } + const rawValue = snapshot[key]; + const value = key === 'options' ? rawValue ?? {} : rawValue; + if (value == null) { + continue; + } + seenPaths.add(pathKey); + changes.push({ type: 'set', path, value }); + } + + return changes; }; diff --git a/packages/v2/core/src/application/projections/RealtimeProjections.spec.ts b/packages/v2/core/src/application/projections/RealtimeProjections.spec.ts index 2c5783c29d..1cf237e165 100644 --- a/packages/v2/core/src/application/projections/RealtimeProjections.spec.ts +++ b/packages/v2/core/src/application/projections/RealtimeProjections.spec.ts @@ -1,5 +1,5 @@ import { err, ok } from 'neverthrow'; -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it } from 'vitest'; import { BaseId } from '../../domain/base/BaseId'; import { ActorId } from '../../domain/shared/ActorId'; @@ -19,6 +19,10 @@ import { ViewColumnMetaUpdated } from '../../domain/table/events/ViewColumnMetaU import { FieldId } from '../../domain/table/fields/FieldId'; import { FieldName } from '../../domain/table/fields/FieldName'; import { LinkFieldConfig } from '../../domain/table/fields/types/LinkFieldConfig'; +import { LinkRelationship } from '../../domain/table/fields/types/LinkRelationship'; +import { LookupOptions } from '../../domain/table/fields/types/LookupOptions'; +import { NumberField } from '../../domain/table/fields/types/NumberField'; +import { NumberFormatting } from '../../domain/table/fields/types/NumberFormatting'; import { SelectOption } from '../../domain/table/fields/types/SelectOption'; import { RecordId } from '../../domain/table/records/RecordId'; import { TableAddSelectOptionsSpec } from '../../domain/table/specs/TableAddSelectOptionsSpec'; @@ -48,6 +52,7 @@ import { RecordsBatchUpdatedRealtimeProjection } from './RecordsBatchUpdatedReal import { RecordsDeletedRealtimeProjection } from './RecordsDeletedRealtimeProjection'; import { RecordUpdatedRealtimeProjection } from './RecordUpdatedRealtimeProjection'; import { REALTIME_TASK_CONCURRENCY_LIMIT } from './runRealtimeTasks'; +import { setRealtimeProjectionSchedulerForTest } from './scheduleRealtimeProjection'; import { TableCreatedRealtimeProjection } from './TableCreatedRealtimeProjection'; import { buildRecordCollection } from './TableRecordRealtimeDTO'; import { ViewColumnMetaUpdatedRealtimeProjection } from './ViewColumnMetaUpdatedRealtimeProjection'; @@ -68,6 +73,11 @@ const fieldUpdateSemantics = { presencePath: ['options', 'formatting'], mayRequirePresence: true, }, + lookupOptions: { + realtimePath: ['lookupOptions'], + presencePath: ['lookupOptions'], + mayRequirePresence: true, + }, } as const; const createContext = (): IExecutionContext => { @@ -146,6 +156,12 @@ class FakeRealtimeEngine implements IRealtimeEngine { } class FakeTableRepository implements ITableRepository { + findOneCount = 0; + findOneDeferred?: { + promise: Promise; + resolve: () => void; + }; + constructor(private readonly table: Table) {} async insert() { @@ -157,6 +173,8 @@ class FakeTableRepository implements ITableRepository { } async findOne() { + this.findOneCount += 1; + await this.findOneDeferred?.promise; return ok(this.table); } @@ -212,7 +230,19 @@ class FakeAttachmentUrlSignerService implements IAttachmentUrlSignerService { const buildAttachmentValueDecorator = () => new AttachmentValueDecoratorService(new FakeAttachmentUrlSignerService()); +const captureRealtimeTasks = () => { + const tasks: Array<() => Promise> = []; + setRealtimeProjectionSchedulerForTest((task) => { + tasks.push(task); + }); + return tasks; +}; + describe('Realtime projections', () => { + afterEach(() => { + setRealtimeProjectionSchedulerForTest(); + }); + it('builds record collection names', () => { expect(buildRecordCollection('tbl123')).toBe('rec_tbl123'); }); @@ -917,6 +947,7 @@ describe('Realtime projections', () => { const repository = new FakeTableRepository(table); const mapper = new FakeTableMapper(buildTableDto); const projection = new FieldCreatedRealtimeProjection(engine, repository, mapper); + const realtimeTasks = captureRealtimeTasks(); const event = FieldCreated.create({ baseId: table.baseId(), @@ -927,6 +958,10 @@ describe('Realtime projections', () => { const result = await projection.handle(createContext(), event); result._unsafeUnwrap(); + expect(engine.ensures.length).toBe(0); + expect(realtimeTasks).toHaveLength(1); + await realtimeTasks[0]!(); + expect(engine.ensures.length).toBe(2); }); @@ -939,6 +974,7 @@ describe('Realtime projections', () => { fields: [], })); const projection = new FieldCreatedRealtimeProjection(engine, repository, mapper); + const realtimeTasks = captureRealtimeTasks(); const event = FieldCreated.create({ baseId: table.baseId(), @@ -947,8 +983,12 @@ describe('Realtime projections', () => { }); const result = await projection.handle(createContext(), event); + result._unsafeUnwrap(); + + expect(realtimeTasks).toHaveLength(1); + await realtimeTasks[0]!(); - expect(result._unsafeUnwrapErr().message).toContain('Missing field snapshot'); + expect(engine.ensures).toHaveLength(1); }); it('projects field deletion', async () => { @@ -975,6 +1015,7 @@ describe('Realtime projections', () => { const repository = new FakeTableRepository(table); const mapper = new FakeTableMapper(buildTableDto); const projection = new ViewColumnMetaUpdatedRealtimeProjection(engine, repository, mapper); + const realtimeTasks = captureRealtimeTasks(); const event = ViewColumnMetaUpdated.create({ baseId: table.baseId(), @@ -988,6 +1029,11 @@ describe('Realtime projections', () => { const result = await projection.handle(createContext(), event); result._unsafeUnwrap(); + expect(engine.ensures).toHaveLength(0); + expect(engine.changes).toHaveLength(0); + expect(realtimeTasks).toHaveLength(1); + await realtimeTasks[0]!(); + expect(engine.ensures).toHaveLength(2); expect(engine.ensures[0]?.docId.toString()).toBe( `tbl_${table.baseId().toString()}/${table.id().toString()}` @@ -1010,12 +1056,167 @@ describe('Realtime projections', () => { expect(engine.changes[1]?.options).toEqual({ version: 7 }); }); + it('reuses a table snapshot across field create realtime projections in one context', async () => { + const table = buildTable('c', 'd', 'f'); + const viewId = table.views()[0]?.id() ?? ViewId.create(`viw${'b'.repeat(16)}`)._unsafeUnwrap(); + const engine = new FakeRealtimeEngine(); + const repository = new FakeTableRepository(table); + const mapper = new FakeTableMapper(buildTableDto); + const fieldProjection = new FieldCreatedRealtimeProjection(engine, repository, mapper); + const viewProjection = new ViewColumnMetaUpdatedRealtimeProjection(engine, repository, mapper); + const context = createContext(); + const realtimeTasks = captureRealtimeTasks(); + + const fieldResult = await fieldProjection.handle( + context, + FieldCreated.create({ + baseId: table.baseId(), + tableId: table.id(), + fieldId: table.primaryFieldId(), + }) + ); + fieldResult._unsafeUnwrap(); + + expect(repository.findOneCount).toBe(0); + await realtimeTasks.shift()!(); + + const viewResult = await viewProjection.handle( + context, + ViewColumnMetaUpdated.create({ + baseId: table.baseId(), + tableId: table.id(), + viewId, + fieldId: table.primaryFieldId(), + }) + ); + viewResult._unsafeUnwrap(); + + await realtimeTasks.shift()!(); + + expect(repository.findOneCount).toBe(1); + }); + + it('deduplicates concurrent table snapshot loads across field create realtime projections', async () => { + const table = buildTable('c', 'd', 'i'); + const viewId = table.views()[0]?.id() ?? ViewId.create(`viw${'c'.repeat(16)}`)._unsafeUnwrap(); + const engine = new FakeRealtimeEngine(); + const repository = new FakeTableRepository(table); + let releaseFindOne!: () => void; + repository.findOneDeferred = { + promise: new Promise((resolve) => { + releaseFindOne = resolve; + }), + resolve: () => releaseFindOne(), + }; + const mapper = new FakeTableMapper(buildTableDto); + const fieldProjection = new FieldCreatedRealtimeProjection(engine, repository, mapper); + const viewProjection = new ViewColumnMetaUpdatedRealtimeProjection(engine, repository, mapper); + const context = createContext(); + const realtimeTasks = captureRealtimeTasks(); + + const fieldResult = await fieldProjection.handle( + context, + FieldCreated.create({ + baseId: table.baseId(), + tableId: table.id(), + fieldId: table.primaryFieldId(), + }) + ); + fieldResult._unsafeUnwrap(); + const viewResult = await viewProjection.handle( + context, + ViewColumnMetaUpdated.create({ + baseId: table.baseId(), + tableId: table.id(), + viewId, + fieldId: table.primaryFieldId(), + }) + ); + viewResult._unsafeUnwrap(); + + const runningTasks = realtimeTasks.map((task) => task()); + await Promise.resolve(); + + expect(repository.findOneCount).toBe(1); + + repository.findOneDeferred.resolve(); + await Promise.all(runningTasks); + + expect(repository.findOneCount).toBe(1); + expect(engine.ensures.length).toBeGreaterThanOrEqual(3); + }); + + it('refreshes a cached table snapshot when a later field create needs a newer field', async () => { + const table = buildTable('c', 'd', 'g'); + const firstFieldId = table.primaryFieldId(); + const secondFieldId = FieldId.create(`fld${'n'.repeat(16)}`)._unsafeUnwrap(); + const engine = new FakeRealtimeEngine(); + const repository = new FakeTableRepository(table); + const mapper = new FakeTableMapper(() => { + const base = buildTableDto(table); + if (repository.findOneCount < 2) { + return base; + } + + return { + ...base, + fields: [ + ...base.fields, + { + id: secondFieldId.toString(), + name: 'Notes', + type: 'longText', + }, + ], + views: base.views.map((view) => ({ + ...view, + columnMeta: { + ...view.columnMeta, + [secondFieldId.toString()]: { order: 1 }, + }, + })), + }; + }); + const projection = new FieldCreatedRealtimeProjection(engine, repository, mapper); + const context = createContext(); + const realtimeTasks = captureRealtimeTasks(); + + const firstResult = await projection.handle( + context, + FieldCreated.create({ + baseId: table.baseId(), + tableId: table.id(), + fieldId: firstFieldId, + }) + ); + firstResult._unsafeUnwrap(); + await realtimeTasks.shift()!(); + + const secondResult = await projection.handle( + context, + FieldCreated.create({ + baseId: table.baseId(), + tableId: table.id(), + fieldId: secondFieldId, + }) + ); + secondResult._unsafeUnwrap(); + await realtimeTasks.shift()!(); + + expect(repository.findOneCount).toBe(2); + expect(engine.ensures.at(-1)?.initial).toMatchObject({ + id: secondFieldId.toString(), + name: 'Notes', + }); + }); + it('ignores missing views', async () => { const table = buildTable('f', 'g', 'h'); const engine = new FakeRealtimeEngine(); const repository = new FakeTableRepository(table); const mapper = new FakeTableMapper(buildTableDto); const projection = new ViewColumnMetaUpdatedRealtimeProjection(engine, repository, mapper); + const realtimeTasks = captureRealtimeTasks(); const event = ViewColumnMetaUpdated.create({ baseId: table.baseId(), @@ -1027,6 +1228,9 @@ describe('Realtime projections', () => { const result = await projection.handle(createContext(), event); result._unsafeUnwrap(); + expect(realtimeTasks).toHaveLength(1); + await realtimeTasks[0]!(); + expect(engine.ensures).toHaveLength(0); expect(engine.changes).toHaveLength(0); }); @@ -1460,6 +1664,156 @@ describe('Realtime projections', () => { ); }); + it('refreshes enriched lookupOptions, cellValueType and inner type/options on a lookup change', async () => { + const baseId = BaseId.create(`bse${'s'.repeat(16)}`)._unsafeUnwrap(); + const tableId = TableId.create(`tbl${'t'.repeat(16)}`)._unsafeUnwrap(); + const foreignTableId = TableId.create(`tbl${'u'.repeat(16)}`)._unsafeUnwrap(); + const primaryFieldId = FieldId.create(`fld${'v'.repeat(16)}`)._unsafeUnwrap(); + const linkFieldId = FieldId.create(`fld${'w'.repeat(16)}`)._unsafeUnwrap(); + const lookupFieldId = FieldId.create(`fld${'x'.repeat(16)}`)._unsafeUnwrap(); + const foreignTargetFieldId = FieldId.create(`fld${'y'.repeat(16)}`)._unsafeUnwrap(); + + const linkConfig = LinkFieldConfig.create({ + relationship: LinkRelationship.manyOne().toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + })._unsafeUnwrap(); + + const builder = Table.builder() + .withId(tableId) + .withBaseId(baseId) + .withName(TableName.create('Lookup Table')._unsafeUnwrap()); + builder + .field() + .singleLineText() + .withId(primaryFieldId) + .withName(FieldName.create('Title')._unsafeUnwrap()) + .primary() + .done(); + builder + .field() + .link() + .withId(linkFieldId) + .withName(FieldName.create('Link')._unsafeUnwrap()) + .withConfig(linkConfig) + .done(); + builder + .field() + .lookup() + .withId(lookupFieldId) + .withName(FieldName.create('Lookup Amount')._unsafeUnwrap()) + .withInnerField( + NumberField.create({ + id: FieldId.create(`fld${'z'.repeat(16)}`)._unsafeUnwrap(), + name: FieldName.create('Amount')._unsafeUnwrap(), + formatting: NumberFormatting.create({ type: 'decimal', precision: 2 })._unsafeUnwrap(), + })._unsafeUnwrap() + ) + .withLookupOptions( + LookupOptions.create({ + linkFieldId: linkFieldId.toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + })._unsafeUnwrap() + ) + .withIsMultipleCellValue(false) + .done(); + builder.view().defaultGrid().done(); + const table = builder.build()._unsafeUnwrap(); + + const engine = new FakeRealtimeEngine(); + const repository = new FakeTableRepository(table); + const mapper = new DefaultTableMapper(); + const projection = new FieldUpdatedRealtimeProjection(engine, repository, mapper); + + const event = FieldUpdated.create({ + baseId: table.baseId(), + tableId: table.id(), + fieldId: lookupFieldId, + updatedProperties: ['lookupOptions'], + propertySemantics: { + lookupOptions: fieldUpdateSemantics.lookupOptions, + }, + }); + + const result = await projection.handle(createContext(), event); + result._unsafeUnwrap(); + + expect(engine.changes).toHaveLength(1); + const changes = engine.changes[0]?.change as RealtimeChange[]; + + // lookupOptions must be published WITH the link's physical metadata (regression: was dropped). + const lookupOptionsChange = changes.find( + (change) => JSON.stringify(change.path) === JSON.stringify(['lookupOptions']) + ); + expect(lookupOptionsChange?.value).toEqual( + expect.objectContaining({ + linkFieldId: linkFieldId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + foreignTableId: foreignTableId.toString(), + fkHostTableName: expect.any(String), + selfKeyName: expect.any(String), + foreignKeyName: expect.any(String), + }) + ); + + // cellValueType must be the resolved inner type, never null (regression: was null). + expect(changes).toContainEqual({ type: 'set', path: ['cellValueType'], value: 'number' }); + // The resolved inner type/options must refresh so changing the looked-up field is reflected + // (dbFieldType, when carried by the persisted snapshot, goes through the same scalar path). + expect(changes).toContainEqual({ type: 'set', path: ['type'], value: 'number' }); + expect( + changes.some((change) => JSON.stringify(change.path) === JSON.stringify(['options'])) + ).toBe(true); + }); + + it('clears stale inner options and skips absent storage metadata when a lookup is pending', async () => { + const table = buildTable('p', 'q', 'r'); + const fieldId = table.primaryFieldId(); + const engine = new FakeRealtimeEngine(); + const repository = new FakeTableRepository(table); + // A pending lookup (inner field unresolved) falls back to singleLineText and omits both + // `options` and `dbFieldType` from its DTO. + const mapper = new FakeTableMapper((candidate) => ({ + ...buildTableDto(candidate), + fields: [ + { + id: fieldId.toString(), + name: 'Pending Lookup', + type: 'singleLineText', + isLookup: true, + isComputed: true, + lookupOptions: { + linkFieldId: `fld${'k'.repeat(16)}`, + lookupFieldId: `fld${'l'.repeat(16)}`, + foreignTableId: `tbl${'m'.repeat(16)}`, + }, + }, + ], + })); + const projection = new FieldUpdatedRealtimeProjection(engine, repository, mapper); + + const event = FieldUpdated.create({ + baseId: table.baseId(), + tableId: table.id(), + fieldId, + updatedProperties: ['lookupOptions'], + propertySemantics: { lookupOptions: fieldUpdateSemantics.lookupOptions }, + }); + + const result = await projection.handle(createContext(), event); + result._unsafeUnwrap(); + + const changes = engine.changes[0]?.change as RealtimeChange[]; + // Stale inner options (e.g. previous number formatting) are cleared, not left untouched. + expect(changes).toContainEqual({ type: 'set', path: ['options'], value: {} }); + expect(changes).toContainEqual({ type: 'set', path: ['type'], value: 'singleLineText' }); + // dbFieldType is absent from the pending snapshot → must NOT publish a null that corrupts it. + expect( + changes.some((change) => JSON.stringify(change.path) === JSON.stringify(['dbFieldType'])) + ).toBe(false); + }); + it('projects formatting-only field updates through the field options snapshot', async () => { const table = buildTable('1', '2', '3'); const fieldId = table.primaryFieldId(); diff --git a/packages/v2/core/src/application/projections/RealtimeTableSnapshotCache.ts b/packages/v2/core/src/application/projections/RealtimeTableSnapshotCache.ts new file mode 100644 index 0000000000..74ea39b614 --- /dev/null +++ b/packages/v2/core/src/application/projections/RealtimeTableSnapshotCache.ts @@ -0,0 +1,97 @@ +import { err, ok } from 'neverthrow'; +import type { Result } from 'neverthrow'; + +import type { BaseId } from '../../domain/base/BaseId'; +import type { DomainError } from '../../domain/shared/DomainError'; +import { Table } from '../../domain/table/Table'; +import type { TableId } from '../../domain/table/TableId'; +import type { IExecutionContext } from '../../ports/ExecutionContext'; +import type { ITableMapper, ITablePersistenceDTO } from '../../ports/mappers/TableMapper'; +import type { ITableRepository } from '../../ports/TableRepository'; + +type RealtimeSnapshotCacheContext = IExecutionContext & { + __realtimeTableSnapshotCache?: Map; +}; + +type RealtimeSnapshotCacheEntry = { + snapshot?: ITablePersistenceDTO; + loading?: Promise>; +}; + +const cacheKey = (baseId: BaseId, tableId: TableId): string => + `${baseId.toString()}:${tableId.toString()}`; + +const getCache = (context: IExecutionContext): Map => { + const cacheContext = context as RealtimeSnapshotCacheContext; + cacheContext.__realtimeTableSnapshotCache ??= new Map(); + return cacheContext.__realtimeTableSnapshotCache; +}; + +export const bindRealtimeTableSnapshotCache = ( + source: IExecutionContext, + target: IExecutionContext +): void => { + const cache = getCache(source); + (target as RealtimeSnapshotCacheContext).__realtimeTableSnapshotCache = cache; +}; + +export const loadRealtimeTableSnapshot = async ( + context: IExecutionContext, + params: { + baseId: BaseId; + tableId: TableId; + tableRepository: ITableRepository; + tableMapper: ITableMapper; + isSnapshotUsable?: (snapshot: ITablePersistenceDTO) => boolean; + } +): Promise> => { + const key = cacheKey(params.baseId, params.tableId); + const cache = getCache(context); + const cached = cache.get(key); + if (cached?.snapshot && (params.isSnapshotUsable?.(cached.snapshot) ?? true)) { + return ok(cached.snapshot); + } + + if (cached?.loading) { + const loadingResult = await cached.loading; + if (loadingResult.isErr()) { + return err(loadingResult.error); + } + if (params.isSnapshotUsable?.(loadingResult.value) ?? true) { + return ok(loadingResult.value); + } + } + + const specResult = Table.specs(params.baseId).byId(params.tableId).build(); + if (specResult.isErr()) { + return err(specResult.error); + } + + const loading = (async (): Promise> => { + const tableResult = await params.tableRepository.findOne(context, specResult.value); + if (tableResult.isErr()) { + return err(tableResult.error); + } + + const snapshotResult = params.tableMapper.toDTO(tableResult.value); + if (snapshotResult.isErr()) { + return err(snapshotResult.error); + } + + return ok(snapshotResult.value); + })(); + cache.set(key, cached?.snapshot ? { snapshot: cached.snapshot, loading } : { loading }); + + const snapshotResult = await loading; + if (snapshotResult.isErr()) { + if (cached?.snapshot) { + cache.set(key, { snapshot: cached.snapshot }); + } else { + cache.delete(key); + } + return err(snapshotResult.error); + } + + cache.set(key, { snapshot: snapshotResult.value }); + return ok(snapshotResult.value); +}; diff --git a/packages/v2/core/src/application/projections/ViewColumnMetaUpdatedRealtimeProjection.ts b/packages/v2/core/src/application/projections/ViewColumnMetaUpdatedRealtimeProjection.ts index eed628c132..69aa11a246 100644 --- a/packages/v2/core/src/application/projections/ViewColumnMetaUpdatedRealtimeProjection.ts +++ b/packages/v2/core/src/application/projections/ViewColumnMetaUpdatedRealtimeProjection.ts @@ -4,7 +4,6 @@ import type { Result } from 'neverthrow'; import type { DomainError } from '../../domain/shared/DomainError'; import { ViewColumnMetaUpdated } from '../../domain/table/events/ViewColumnMetaUpdated'; -import { Table } from '../../domain/table/Table'; import type { IEventHandler } from '../../ports/EventHandler'; import type * as ExecutionContextPort from '../../ports/ExecutionContext'; import * as TableMapperPort from '../../ports/mappers/TableMapper'; @@ -13,6 +12,8 @@ import * as RealtimeEnginePort from '../../ports/RealtimeEngine'; import * as TableRepositoryPort from '../../ports/TableRepository'; import { v2CoreTokens } from '../../ports/tokens'; import { ProjectionHandler } from './Projection'; +import { loadRealtimeTableSnapshot } from './RealtimeTableSnapshotCache'; +import { scheduleRealtimeProjection } from './scheduleRealtimeProjection'; const tableCollectionPrefix = 'tbl'; const viewCollectionPrefix = 'viw'; @@ -37,55 +38,68 @@ export class ViewColumnMetaUpdatedRealtimeProjection ): Promise> { const { realtimeEngine, tableRepository, tableMapper } = this; - return safeTry(async function* () { - // Fetch table data from repository - const spec = yield* Table.specs(event.baseId).byId(event.tableId).build().safeUnwrap(); - const table = yield* (await tableRepository.findOne(context, spec)).safeUnwrap(); - const snapshot = yield* tableMapper.toDTO(table).safeUnwrap(); + return scheduleRealtimeProjection( + context, + ViewColumnMetaUpdatedRealtimeProjection.name, + (context) => + safeTry(async function* () { + const snapshot = yield* ( + await loadRealtimeTableSnapshot(context, { + baseId: event.baseId, + tableId: event.tableId, + tableRepository, + tableMapper, + isSnapshotUsable: (candidate) => { + const view = candidate.views.find((view) => view.id === event.viewId.toString()); + return Boolean(view?.columnMeta[event.fieldId.toString()]); + }, + }) + ).safeUnwrap(); - const viewIndex = snapshot.views.findIndex((view) => view.id === event.viewId.toString()); - if (viewIndex === -1) return ok(undefined); + const viewIndex = snapshot.views.findIndex((view) => view.id === event.viewId.toString()); + if (viewIndex === -1) return ok(undefined); - const viewDto = snapshot.views[viewIndex]; + const viewDto = snapshot.views[viewIndex]; - const collection = `${tableCollectionPrefix}_${event.baseId.toString()}`; - const docId = yield* RealtimeDocId.fromParts( - collection, - event.tableId.toString() - ).safeUnwrap(); + const collection = `${tableCollectionPrefix}_${event.baseId.toString()}`; + const docId = yield* RealtimeDocId.fromParts( + collection, + event.tableId.toString() + ).safeUnwrap(); - // Ensure table document exists first (for tables created before realtime was enabled) - yield* (await realtimeEngine.ensure(context, docId, snapshot)).safeUnwrap(); + // Ensure table document exists first (for tables created before realtime was enabled) + yield* (await realtimeEngine.ensure(context, docId, snapshot)).safeUnwrap(); - // Keep the table snapshot in sync for table-level consumers. - yield* ( - await realtimeEngine.applyChange(context, docId, { - type: 'set', - path: ['views', viewIndex, 'columnMeta'], - value: viewDto.columnMeta, - }) - ).safeUnwrap(); + // Keep the table snapshot in sync for table-level consumers. + yield* ( + await realtimeEngine.applyChange(context, docId, { + type: 'set', + path: ['views', viewIndex, 'columnMeta'], + value: viewDto.columnMeta, + }) + ).safeUnwrap(); - // Keep the standalone view document in sync for ShareDB/SDK view subscriptions. - const viewCollection = `${viewCollectionPrefix}_${event.tableId.toString()}`; - const viewDocId = yield* RealtimeDocId.fromParts( - viewCollection, - event.viewId.toString() - ).safeUnwrap(); - yield* (await realtimeEngine.ensure(context, viewDocId, viewDto)).safeUnwrap(); + // Keep the standalone view document in sync for ShareDB/SDK view subscriptions. + const viewCollection = `${viewCollectionPrefix}_${event.tableId.toString()}`; + const viewDocId = yield* RealtimeDocId.fromParts( + viewCollection, + event.viewId.toString() + ).safeUnwrap(); + yield* (await realtimeEngine.ensure(context, viewDocId, viewDto)).safeUnwrap(); - return realtimeEngine.applyChange( - context, - viewDocId, - { - type: 'set', - path: ['columnMeta'], - value: viewDto.columnMeta, - }, - { - version: event.oldVersion, - } - ); - }); + return realtimeEngine.applyChange( + context, + viewDocId, + { + type: 'set', + path: ['columnMeta'], + value: viewDto.columnMeta, + }, + { + version: event.oldVersion, + } + ); + }) + ); } } diff --git a/packages/v2/core/src/application/projections/scheduleRealtimeProjection.ts b/packages/v2/core/src/application/projections/scheduleRealtimeProjection.ts new file mode 100644 index 0000000000..70abfde031 --- /dev/null +++ b/packages/v2/core/src/application/projections/scheduleRealtimeProjection.ts @@ -0,0 +1,109 @@ +import { ok } from 'neverthrow'; +import type { Result, ResultAsync } from 'neverthrow'; + +import type { DomainError } from '../../domain/shared/DomainError'; +import type { IExecutionContext } from '../../ports/ExecutionContext'; +import { getUnitOfWorkTransaction, withoutTransaction } from '../../ports/ExecutionContext'; +import { createTeableSpanAttributes, TeableSpanAttributes } from '../../ports/Tracer'; +import { bindRealtimeTableSnapshotCache } from './RealtimeTableSnapshotCache'; + +export type RealtimeProjectionScheduler = (task: () => Promise) => void; + +const defaultRealtimeProjectionScheduler: RealtimeProjectionScheduler = (task) => { + const immediate = (globalThis as { setImmediate?: (task: () => void) => void }).setImmediate; + if (typeof immediate === 'function') { + immediate(() => void task()); + return; + } + + const timeout = (globalThis as { setTimeout?: (handler: () => void, timeout: number) => void }) + .setTimeout; + if (typeof timeout === 'function') { + timeout(() => void task(), 0); + return; + } + + const microtask = (globalThis as { queueMicrotask?: (task: () => void) => void }).queueMicrotask; + if (typeof microtask === 'function') { + microtask(() => void task()); + return; + } + + void task(); +}; + +let realtimeProjectionScheduler: RealtimeProjectionScheduler = defaultRealtimeProjectionScheduler; + +export const setRealtimeProjectionSchedulerForTest = ( + scheduler?: RealtimeProjectionScheduler +): void => { + realtimeProjectionScheduler = scheduler ?? defaultRealtimeProjectionScheduler; +}; + +const registerAfterCommit = (context: IExecutionContext, task: () => Promise): boolean => { + const metaTransaction = getUnitOfWorkTransaction(context, 'meta'); + if (metaTransaction?.afterCommit) { + metaTransaction.afterCommit(task); + return true; + } + + const dataTransaction = getUnitOfWorkTransaction(context, 'data'); + if (dataTransaction?.afterCommit) { + dataTransaction.afterCommit(task); + return true; + } + + return false; +}; + +export const scheduleRealtimeProjection = ( + context: IExecutionContext, + projectionName: string, + task: ( + context: IExecutionContext + ) => Promise> | ResultAsync +): Result => { + const backgroundContext = withoutTransaction(context); + bindRealtimeTableSnapshotCache(context, backgroundContext); + const run = async () => { + const tracer = backgroundContext.tracer; + const span = tracer?.startSpan( + `teable.${projectionName}.background`, + createTeableSpanAttributes('projection', `${projectionName}.background`, { + [TeableSpanAttributes.HANDLER]: projectionName, + [TeableSpanAttributes.EVENT_ASYNC]: true, + }) + ); + + const execute = async () => { + try { + const result = await task(backgroundContext); + if (result.isErr()) { + span?.recordError(result.error.message); + } + } catch (error) { + span?.recordError(error instanceof Error ? error.message : String(error)); + } finally { + span?.end(); + } + }; + + if (span && tracer) { + await tracer.withSpan(span, execute); + return; + } + + await execute(); + }; + + const scheduleRun = async () => { + realtimeProjectionScheduler(run); + }; + + if (registerAfterCommit(context, scheduleRun)) { + return ok(undefined); + } + + realtimeProjectionScheduler(run); + return ok(undefined); +}; diff --git a/packages/v2/core/src/application/services/DeleteByRangeApplicationService.ts b/packages/v2/core/src/application/services/DeleteByRangeApplicationService.ts index 86238a0cec..6397396e9a 100644 --- a/packages/v2/core/src/application/services/DeleteByRangeApplicationService.ts +++ b/packages/v2/core/src/application/services/DeleteByRangeApplicationService.ts @@ -10,6 +10,7 @@ import { resolveGroupByToOrderBy, resolveOrderBy, } from '../../commands/shared/orderBy'; +import { ensureRecordIdsWithinScope } from '../../commands/shared/recordWriteScope'; import { resolveSelectionStreamBatchSize } from '../../commands/shared/streamBatchSize'; import { domainError, isNotFoundError, type DomainError } from '../../domain/shared/DomainError'; import type { IDomainEvent } from '../../domain/shared/DomainEvent'; @@ -44,7 +45,10 @@ import type { SpanAttributes } from '../../ports/Tracer'; import * as UnitOfWorkPort from '../../ports/UnitOfWork'; import type { RecordSortValue } from '../../queries/ListTableRecordsQuery'; import type { RecordFilter } from '../../queries/RecordFilterDto'; -import { buildSanitizedRecordConditionSpec } from '../../queries/RecordFilterMapper'; +import { + buildSanitizedRecordConditionSpec, + replaceCurrentUserTagInFilter, +} from '../../queries/RecordFilterMapper'; import { resolveVisibleRowSearch, type RecordQuerySearch, @@ -59,6 +63,9 @@ import { toUndoRedoStackAppendContext, UndoRedoStackService } from './UndoRedoSt const DEFAULT_DELETE_QUERY_PAGE_SIZE = 500; const MAX_DELETE_STREAM_BUFFERED_EVENTS = 64; +const toRangeTypeSpanAttribute = (rangeType: RangeType): 'columns' | 'rows' | 'cells' => + rangeType ?? 'cells'; + type DeleteByRangeCommandLike = { readonly tableId: TableId; readonly viewId: ViewId; @@ -183,10 +190,17 @@ export class DeleteByRangeApplicationService { context: IExecutionContext, command: DeleteByRangeCommandLike ): Promise> { - const planResult = await this.prepareDeletePlan( + const planResult = await this.runInSpan( context, - command, - DEFAULT_DELETE_QUERY_PAGE_SIZE + 'teable.DeleteByRangeApplicationService.prepareDeletePlan', + { + 'teable.table_id': command.tableId.toString(), + 'teable.view_id': command.viewId.toString(), + 'teable.range_type': toRangeTypeSpanAttribute(command.rangeType), + 'teable.range_count': command.rawRanges.length, + 'teable.delete_mode': 'direct', + }, + () => this.prepareDeletePlan(context, command, DEFAULT_DELETE_QUERY_PAGE_SIZE) ); if (planResult.isErr()) { return err(planResult.error); @@ -202,33 +216,78 @@ export class DeleteByRangeApplicationService { } const operationId = context.requestId ?? generateUuid(); - const pluginExecutionResult = await this.prepareDeletePluginExecution( + const pluginExecutionResult = await this.runInSpan( context, - plan.table, - plan.recordIds, - plan.recordSnapshots, - this.createDeletePluginOrchestration({ - mode: 'direct', - scope: 'operation', - operationId, - totalRecordCount: plan.recordIds.length, - totalChunkCount: 1, - }) + 'teable.DeleteByRangeApplicationService.prepareDeletePlugins', + { + 'teable.table_id': plan.table.id().toString(), + 'teable.record_count': plan.recordIds.length, + 'teable.delete_mode': 'direct', + }, + () => + this.prepareDeletePluginExecution( + context, + plan.table, + plan.recordIds, + plan.recordSnapshots, + this.createDeletePluginOrchestration({ + mode: 'direct', + scope: 'operation', + operationId, + totalRecordCount: plan.recordIds.length, + totalChunkCount: 1, + }), + { skipScopeValidation: true } + ) ); if (pluginExecutionResult.isErr()) { return err(pluginExecutionResult.error); } - const persistResult = await this.deleteManyInSingleTransaction( + const scopeValidationResult = await this.runInSpan( context, - plan, - pluginExecutionResult.value + 'teable.DeleteByRangeApplicationService.validateDeletePluginScope', + { + 'teable.table_id': plan.table.id().toString(), + 'teable.record_count': plan.recordIds.length, + 'teable.delete_mode': 'direct', + }, + () => + this.validateDeletePluginScope( + context, + plan.table, + plan.recordIds, + pluginExecutionResult.value + ) + ); + if (scopeValidationResult.isErr()) { + return err(scopeValidationResult.error); + } + + const persistResult = await this.runInSpan( + context, + 'teable.DeleteByRangeApplicationService.deleteRecords', + { + 'teable.table_id': plan.table.id().toString(), + 'teable.record_count': plan.recordIds.length, + 'teable.delete_mode': 'direct', + }, + () => this.deleteManyInSingleTransaction(context, plan, pluginExecutionResult.value) ); if (persistResult.isErr()) { return err(persistResult.error); } - return this.finalizeDeletePlan(context, plan, persistResult.value, pluginExecutionResult.value); + return this.runInSpan( + context, + 'teable.DeleteByRangeApplicationService.finalizeDeletePlan', + { + 'teable.table_id': plan.table.id().toString(), + 'teable.record_count': plan.recordIds.length, + 'teable.delete_mode': 'direct', + }, + () => this.finalizeDeletePlan(context, plan, persistResult.value, pluginExecutionResult.value) + ); } createStream( @@ -252,7 +311,18 @@ export class DeleteByRangeApplicationService { } try { - const planResult = await this.prepareDeleteStreamPlan(context, command); + const planResult = await this.runInSpan( + context, + 'teable.DeleteByRangeApplicationService.prepareDeleteStreamPlan', + { + 'teable.table_id': command.tableId.toString(), + 'teable.view_id': command.viewId.toString(), + 'teable.range_type': toRangeTypeSpanAttribute(command.rangeType), + 'teable.range_count': command.rawRanges.length, + 'teable.delete_mode': 'stream', + }, + () => this.prepareDeleteStreamPlan(context, command) + ); if (planResult.isErr()) { queue.push( this.createErrorEvent(planResult.error, { @@ -374,7 +444,12 @@ export class DeleteByRangeApplicationService { ? command.groupBy ?? undefined : mergedDefaults.group(); - const filterSpecResult = await buildSanitizedRecordConditionSpec(table, effectiveFilter); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + table, + effectiveFilter, + context.actorId.toString() + ); + const filterSpecResult = await buildSanitizedRecordConditionSpec(table, actorResolvedFilter); if (filterSpecResult.isErr()) { return err(filterSpecResult.error); } @@ -402,9 +477,11 @@ export class DeleteByRangeApplicationService { search: visibleRowSearch, totalCols: orderedFieldIdsResult.value.length, pageSize, + projectionFieldIds: this.resolveDeletePreReadProjectionFieldIds(table), // Undo/redo restore snapshots now come from repository capture instead of // the pre-read query, so we do not pay the extra row-order join here. includeOrders: false, + includeTotal: false, }); if (recordsResult.isErr()) { return err(recordsResult.error); @@ -480,7 +557,12 @@ export class DeleteByRangeApplicationService { ? command.groupBy ?? undefined : mergedDefaults.group(); - const filterSpecResult = await buildSanitizedRecordConditionSpec(table, effectiveFilter); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + table, + effectiveFilter, + context.actorId.toString() + ); + const filterSpecResult = await buildSanitizedRecordConditionSpec(table, actorResolvedFilter); if (filterSpecResult.isErr()) { return err(filterSpecResult.error); } @@ -898,19 +980,31 @@ export class DeleteByRangeApplicationService { let deletedCount = 0; let currentRangeKey: string | undefined; let failedRowCountInRange = 0; - const operationPluginExecutionResult = await this.prepareDeletePluginExecution( + const operationPluginExecutionResult = await this.runInSpan( context, - plan.table, - [], - [], - this.createDeletePluginOrchestration({ - mode: 'stream', - scope: 'operation', - operationId: operation.operationId, - totalRecordCount: plan.totalCount, - totalChunkCount: operation.totalChunkCount, - }), - { skipScopeValidation: true, payloadRecordCount: plan.totalCount } + 'teable.DeleteByRangeApplicationService.prepareDeleteStreamPlugins', + { + 'teable.table_id': plan.table.id().toString(), + 'teable.total_record_count': plan.totalCount, + 'teable.total_chunk_count': operation.totalChunkCount, + 'teable.delete_mode': 'stream', + 'teable.plugin_scope': 'operation', + }, + () => + this.prepareDeletePluginExecution( + context, + plan.table, + [], + [], + this.createDeletePluginOrchestration({ + mode: 'stream', + scope: 'operation', + operationId: operation.operationId, + totalRecordCount: plan.totalCount, + totalChunkCount: operation.totalChunkCount, + }), + { skipScopeValidation: true, payloadRecordCount: plan.totalCount } + ) ); if (operationPluginExecutionResult.isErr()) { queue.push( @@ -968,20 +1062,34 @@ export class DeleteByRangeApplicationService { continue; } - const pluginExecutionResult = await this.prepareDeletePluginExecution( + const pluginExecutionResult = await this.runInSpan( context, - plan.table, - chunk.recordIds, - chunk.recordSnapshots, - this.createDeletePluginOrchestration({ - mode: 'stream', - scope: 'chunk', - operationId: operation.operationId, - totalRecordCount: plan.totalCount, - totalChunkCount: operation.totalChunkCount, - chunkIndex: chunk.batchIndex, - }), - { previousExecution: previousPluginExecution } + 'teable.DeleteByRangeApplicationService.prepareDeleteChunkPlugins', + { + 'teable.batch_index': chunk.batchIndex, + 'teable.chunk_record_count': chunk.recordIds.length, + 'teable.total_record_count': plan.totalCount, + 'teable.total_chunk_count': operation.totalChunkCount, + 'teable.table_id': plan.table.id().toString(), + 'teable.delete_mode': 'stream', + 'teable.plugin_scope': 'chunk', + }, + () => + this.prepareDeletePluginExecution( + context, + plan.table, + chunk.recordIds, + chunk.recordSnapshots, + this.createDeletePluginOrchestration({ + mode: 'stream', + scope: 'chunk', + operationId: operation.operationId, + totalRecordCount: plan.totalCount, + totalChunkCount: operation.totalChunkCount, + chunkIndex: chunk.batchIndex, + }), + { previousExecution: previousPluginExecution, skipScopeValidation: true } + ) ); if (pluginExecutionResult.isErr()) { failedRowCountInRange += chunk.recordIds.length; @@ -998,6 +1106,32 @@ export class DeleteByRangeApplicationService { } const pluginExecution = pluginExecutionResult.value; + const scopeValidationResult = await this.runInSpan( + context, + 'teable.DeleteByRangeApplicationService.validateDeleteChunkPluginScope', + { + 'teable.batch_index': chunk.batchIndex, + 'teable.chunk_record_count': chunk.recordIds.length, + 'teable.total_record_count': plan.totalCount, + 'teable.table_id': plan.table.id().toString(), + 'teable.delete_mode': 'stream', + }, + () => this.validateDeletePluginScope(context, plan.table, chunk.recordIds, pluginExecution) + ); + if (scopeValidationResult.isErr()) { + failedRowCountInRange += chunk.recordIds.length; + queue.push( + this.createErrorEvent(scopeValidationResult.error, { + phase: 'guarding', + batchIndex: chunk.batchIndex, + totalCount: plan.totalCount, + deletedCount, + recordIds: [...chunk.deletedRecordIds], + }) + ); + continue; + } + previousPluginExecution = pluginExecution; const deleteResult = await this.runInSpan( context, @@ -1191,6 +1325,7 @@ export class DeleteByRangeApplicationService { queryStartRow, queryStartRow + chunk.rowCount - 1, { + projectionFieldIds: this.resolveDeletePreReadProjectionFieldIds(plan.table), // Repository-backed delete capture supplies the persisted order snapshot. includeOrders: false, includeTotal: false, @@ -1297,6 +1432,40 @@ export class DeleteByRangeApplicationService { ); } + private resolveDeletePreReadProjectionFieldIds(table: Table): ReadonlyArray { + return [table.primaryFieldId()]; + } + + private async validateDeletePluginScope( + context: IExecutionContext, + table: Table, + recordIds: ReadonlyArray, + pluginExecution?: RecordWritePluginExecution + ): Promise> { + if (!pluginExecution || !recordIds.length) { + return ok(undefined); + } + + const pluginRecordSpecResult = pluginExecution.getRecordSpec(); + if (pluginRecordSpecResult.isErr()) { + return err(pluginRecordSpecResult.error); + } + + if (!pluginRecordSpecResult.value) { + return ok(undefined); + } + + return ensureRecordIdsWithinScope( + context, + table, + recordIds, + pluginRecordSpecResult.value, + this.tableRecordQueryRepository, + RecordWriteOperationKind.deleteMany, + { projectionFieldIds: this.resolveDeletePreReadProjectionFieldIds(table) } + ); + } + private async publishDeleteEvents( context: IExecutionContext, input: { diff --git a/packages/v2/core/src/application/services/DuplicateRecordsApplicationService.ts b/packages/v2/core/src/application/services/DuplicateRecordsApplicationService.ts index 7c26419cda..b77f31b966 100644 --- a/packages/v2/core/src/application/services/DuplicateRecordsApplicationService.ts +++ b/packages/v2/core/src/application/services/DuplicateRecordsApplicationService.ts @@ -48,7 +48,10 @@ import type { SpanAttributes } from '../../ports/Tracer'; import * as UnitOfWorkPort from '../../ports/UnitOfWork'; import type { RecordSortValue } from '../../queries/ListTableRecordsQuery'; import type { RecordFilter } from '../../queries/RecordFilterDto'; -import { buildSanitizedRecordConditionSpec } from '../../queries/RecordFilterMapper'; +import { + buildSanitizedRecordConditionSpec, + replaceCurrentUserTagInFilter, +} from '../../queries/RecordFilterMapper'; import { resolveVisibleRowSearch, type RecordQuerySearch, @@ -318,7 +321,12 @@ export class DuplicateRecordsApplicationService { ? command.groupBy ?? undefined : mergedDefaults.group(); - const filterSpecResult = await buildSanitizedRecordConditionSpec(table, effectiveFilter); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + table, + effectiveFilter, + context.actorId.toString() + ); + const filterSpecResult = await buildSanitizedRecordConditionSpec(table, actorResolvedFilter); if (filterSpecResult.isErr()) { return err(filterSpecResult.error); } diff --git a/packages/v2/core/src/application/services/FieldUndoRedoReplayService.spec.ts b/packages/v2/core/src/application/services/FieldUndoRedoReplayService.spec.ts index b82e5ff8c5..7741bf2c81 100644 --- a/packages/v2/core/src/application/services/FieldUndoRedoReplayService.spec.ts +++ b/packages/v2/core/src/application/services/FieldUndoRedoReplayService.spec.ts @@ -334,15 +334,22 @@ class FakeUnitOfWork implements IUnitOfWork { class FakeTableUpdateFlow { calls = 0; + latestTable?: Table; async execute( _: IExecutionContext, - target: { table: Table } + target: { table: Table }, + mutate: (table: Table) => Result<{ table: Table }, DomainError> ): Promise< Result<{ table: Table; events: IDomainEvent[]; postPersistEvents: IDomainEvent[] }, DomainError> > { this.calls += 1; - return ok({ table: target.table, events: [], postPersistEvents: [] }); + const mutateResult = mutate(target.table); + if (mutateResult.isErr()) { + return err(mutateResult.error); + } + this.latestTable = mutateResult.value.table; + return ok({ table: mutateResult.value.table, events: [], postPersistEvents: [] }); } } @@ -379,7 +386,7 @@ describe('FieldUndoRedoReplayService', () => { views: [ { viewId: table.views()[0]!.id().toString(), - columnMeta: { width: 420 }, + columnMeta: { width: 420, order: 99 }, query: { manualSort: true }, orderedFieldIds: [restoredFieldId.toString(), table.primaryFieldId().toString()], }, @@ -394,6 +401,14 @@ describe('FieldUndoRedoReplayService', () => { expect(command.fieldId.toString()).toBe(restoredFieldId.toString()); expect(command.fieldUpdate.name).toBe('Score Restored'); expect(tableUpdateFlow.calls).toBe(2); + const replayedMeta = tableUpdateFlow.latestTable + ?.views()[0] + ?.columnMeta() + ._unsafeUnwrap() + .toDto(); + expect(replayedMeta?.[restoredFieldId.toString()]?.order).toBe(0); + expect(replayedMeta?.[restoredFieldId.toString()]?.width).toBe(420); + expect(replayedMeta?.[table.primaryFieldId().toString()]?.order).toBe(1); }); it('creates a missing field, replays stored values, and defers constraint restoration until after data is restored', async () => { diff --git a/packages/v2/core/src/application/services/FieldUndoRedoReplayService.ts b/packages/v2/core/src/application/services/FieldUndoRedoReplayService.ts index 5a557c4a95..b0f47709f5 100644 --- a/packages/v2/core/src/application/services/FieldUndoRedoReplayService.ts +++ b/packages/v2/core/src/application/services/FieldUndoRedoReplayService.ts @@ -359,6 +359,7 @@ export class FieldUndoRedoReplayService { } const targetFieldId = fieldId.toString(); + const orderedTargetOrder = nextMetaRaw[targetFieldId]?.order; if (snapshot.columnMeta === null) { const currentOrderMeta = nextMetaRaw[targetFieldId]; if ( @@ -373,10 +374,14 @@ export class FieldUndoRedoReplayService { } } else if (snapshot.columnMeta !== undefined) { const currentMetaForTarget = nextMetaRaw[targetFieldId]; - nextMetaRaw[targetFieldId] = + const restoredMeta = currentMetaForTarget && typeof currentMetaForTarget === 'object' ? { ...currentMetaForTarget, ...snapshot.columnMeta } : snapshot.columnMeta; + nextMetaRaw[targetFieldId] = + typeof orderedTargetOrder === 'number' + ? { ...restoredMeta, order: orderedTargetOrder } + : restoredMeta; } const nextMetaResult = ViewColumnMeta.create(nextMetaRaw); diff --git a/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.spec.ts b/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.spec.ts index d576f2bd46..70051d045b 100644 --- a/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.spec.ts +++ b/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.spec.ts @@ -9,6 +9,9 @@ import { FieldId } from '../../domain/table/fields/FieldId'; import { FieldName } from '../../domain/table/fields/FieldName'; import { FormulaExpression } from '../../domain/table/fields/types/FormulaExpression'; import { FormulaField } from '../../domain/table/fields/types/FormulaField'; +import { LinkFieldConfig } from '../../domain/table/fields/types/LinkFieldConfig'; +import { LinkRelationship } from '../../domain/table/fields/types/LinkRelationship'; +import { LookupOptions } from '../../domain/table/fields/types/LookupOptions'; import { NumberField } from '../../domain/table/fields/types/NumberField'; import { SingleLineTextField } from '../../domain/table/fields/types/SingleLineTextField'; import { Table } from '../../domain/table/Table'; @@ -45,7 +48,9 @@ const createIds = () => ({ viewId: ViewId.create(`viw${'f'.repeat(16)}`)._unsafeUnwrap(), }); -const buildTable = (mode: 'number' | 'formula'): { table: Table; targetFieldId: FieldId } => { +const buildTable = ( + mode: 'number' | 'formula' +): { table: Table; targetFieldId: FieldId; titleFieldId: FieldId } => { const { baseId, tableId, titleFieldId, amountFieldId, formulaFieldId, viewId } = createIds(); const titleField = SingleLineTextField.create({ id: titleFieldId, @@ -96,18 +101,26 @@ const buildTable = (mode: 'number' | 'formula'): { table: Table; targetFieldId: primaryFieldId: titleField.id(), })._unsafeUnwrap(), targetFieldId: targetField.id(), + titleFieldId, }; }; class FakeTableRecordQueryRepository implements ITableRecordQueryRepository { calls = 0; + options: ITableRecordQueryOptions[] = []; rows: TableRecordReadModel[] = []; error: DomainError | undefined; - async find(): Promise< - Result<{ records: ReadonlyArray; total: number }, DomainError> - > { + async find( + _context?: IExecutionContext, + _table?: Table, + _spec?: unknown, + options?: ITableRecordQueryOptions + ): Promise; total: number }, DomainError>> { this.calls += 1; + if (options) { + this.options.push(options); + } if (this.error) { return err(this.error); } @@ -160,6 +173,49 @@ describe('FieldUndoRedoSnapshotService', () => { expect(repository.calls).toBe(1); }); + it('captures multiple stored field snapshots with one projected record query', async () => { + const { table, targetFieldId, titleFieldId } = buildTable('number'); + const repository = new FakeTableRecordQueryRepository(); + repository.rows = [ + { + id: `rec${'1'.repeat(16)}`, + fields: { + [titleFieldId.toString()]: 'A', + [targetFieldId.toString()]: 42, + }, + version: 1, + }, + { + id: `rec${'2'.repeat(16)}`, + fields: { + [titleFieldId.toString()]: 'B', + }, + version: 2, + }, + ]; + const service = new FieldUndoRedoSnapshotService(new DefaultTableMapper(), repository); + + const result = await service.captureMany(buildContext(), table, [titleFieldId, targetFieldId]); + const snapshots = result._unsafeUnwrap(); + + expect(repository.calls).toBe(1); + expect(repository.options[0]?.projectionFieldIds?.map((fieldId) => fieldId.toString())).toEqual( + [titleFieldId.toString(), targetFieldId.toString()] + ); + expect(snapshots.map((snapshot) => snapshot.field.id)).toEqual([ + titleFieldId.toString(), + targetFieldId.toString(), + ]); + expect(snapshots[0]?.records).toEqual([ + { recordId: `rec${'1'.repeat(16)}`, value: 'A' }, + { recordId: `rec${'2'.repeat(16)}`, value: 'B' }, + ]); + expect(snapshots[1]?.records).toEqual([ + { recordId: `rec${'1'.repeat(16)}`, value: 42 }, + { recordId: `rec${'2'.repeat(16)}`, value: null }, + ]); + }); + it('skips record capture for computed fields and when includeRecords is false', async () => { const formulaScenario = buildTable('formula'); const repository = new FakeTableRecordQueryRepository(); @@ -184,6 +240,82 @@ describe('FieldUndoRedoSnapshotService', () => { expect(repository.calls).toBe(0); }); + it('captures a lookup field whose enriched options carry link fk metadata', async () => { + const baseId = BaseId.create(`bse${'g'.repeat(16)}`)._unsafeUnwrap(); + const tableId = TableId.create(`tbl${'h'.repeat(16)}`)._unsafeUnwrap(); + const foreignTableId = TableId.create(`tbl${'i'.repeat(16)}`)._unsafeUnwrap(); + const primaryFieldId = FieldId.create(`fld${'j'.repeat(16)}`)._unsafeUnwrap(); + const linkFieldId = FieldId.create(`fld${'k'.repeat(16)}`)._unsafeUnwrap(); + const lookupFieldId = FieldId.create(`fld${'l'.repeat(16)}`)._unsafeUnwrap(); + const foreignTargetFieldId = FieldId.create(`fld${'m'.repeat(16)}`)._unsafeUnwrap(); + + const linkConfig = LinkFieldConfig.create({ + relationship: LinkRelationship.manyOne().toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + })._unsafeUnwrap(); + + const builder = Table.builder() + .withId(tableId) + .withBaseId(baseId) + .withName(TableName.create('Lookup Undo')._unsafeUnwrap()); + builder + .field() + .singleLineText() + .withId(primaryFieldId) + .withName(FieldName.create('Title')._unsafeUnwrap()) + .primary() + .done(); + builder + .field() + .link() + .withId(linkFieldId) + .withName(FieldName.create('Link')._unsafeUnwrap()) + .withConfig(linkConfig) + .done(); + builder + .field() + .lookup() + .withId(lookupFieldId) + .withName(FieldName.create('Lookup Amount')._unsafeUnwrap()) + .withInnerField( + NumberField.create({ + id: FieldId.create(`fld${'n'.repeat(16)}`)._unsafeUnwrap(), + name: FieldName.create('Amount')._unsafeUnwrap(), + })._unsafeUnwrap() + ) + .withLookupOptions( + LookupOptions.create({ + linkFieldId: linkFieldId.toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + })._unsafeUnwrap() + ) + .withIsMultipleCellValue(false) + .done(); + builder.view().defaultGrid().done(); + const table = builder.build()._unsafeUnwrap(); + + const service = new FieldUndoRedoSnapshotService( + new DefaultTableMapper(), + new FakeTableRecordQueryRepository() + ); + + // Regression: the enriched lookupOptions (fk metadata) must be reduced to the strict + // create-field input schema; otherwise capture fails with "Invalid field undo/redo snapshot input". + const snapshot = (await service.capture(buildContext(), table, lookupFieldId))._unsafeUnwrap(); + + expect(snapshot.field.type).toBe('lookup'); + const options = (snapshot.field as { options?: Record }).options ?? {}; + expect(options).toEqual({ + linkFieldId: linkFieldId.toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + }); + expect(options).not.toHaveProperty('fkHostTableName'); + expect(options).not.toHaveProperty('relationship'); + }); + it('returns empty record snapshots when the stored column is missing', async () => { const { table, targetFieldId } = buildTable('number'); const repository = new FakeTableRecordQueryRepository(); diff --git a/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.ts b/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.ts index 835209e42a..1ed9ec3833 100644 --- a/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.ts +++ b/packages/v2/core/src/application/services/FieldUndoRedoSnapshotService.ts @@ -21,6 +21,15 @@ import { v2CoreTokens } from '../../ports/tokens'; import { tableFieldInputSchema } from '../../schemas/field'; import { TraceSpan } from '../../ports/TraceSpan'; +type SnapshotRecordValues = NonNullable; + +type SnapshotSource = { + readonly field: Field; + readonly fieldId: string; + readonly snapshotField: UndoRedoFieldSnapshot['field']; + readonly views: UndoRedoFieldSnapshot['views']; +}; + const stripUndefinedDeep = (value: unknown): unknown => { if (Array.isArray(value)) { return value.map(stripUndefinedDeep); @@ -40,6 +49,20 @@ const stripUndefinedDeep = (value: unknown): unknown => { return result; }; +// The persisted/realtime lookupOptions is enriched with derived data (relationship, and the +// parent link field's physical metadata: fkHostTableName/selfKeyName/foreignKeyName, baseId, +// filterByViewId, visibleFieldIds, isOneWay). The undo/redo snapshot must reduce it back to the +// canonical create-field input accepted by `lookupOptionsSchema` (.strict()), so we keep only the +// input keys rather than stripping a denylist that can drift as enrichment grows. +const LOOKUP_INPUT_OPTION_KEYS = [ + 'linkFieldId', + 'foreignTableId', + 'lookupFieldId', + 'filter', + 'sort', + 'limit', +] as const; + const normalizeLookupOptions = ( options: Record | undefined ): Record | undefined => { @@ -47,8 +70,13 @@ const normalizeLookupOptions = ( return undefined; } - const { relationship: _relationship, ...rest } = options; - return rest; + const result: Record = {}; + for (const key of LOOKUP_INPUT_OPTION_KEYS) { + if (options[key] !== undefined) { + result[key] = options[key]; + } + } + return result; }; const toFieldSnapshotInput = ( @@ -162,37 +190,91 @@ export class FieldUndoRedoSnapshotService { includeRecords?: boolean; } ): Promise> { + const snapshotsResult = await this.captureMany(context, table, [fieldId], options); + return snapshotsResult.map((snapshots) => snapshots[0]!); + } + + @TraceSpan() + async captureMany( + context: IExecutionContext, + table: Table, + fieldIds: ReadonlyArray, + options?: { + includeRecords?: boolean; + } + ): Promise, DomainError>> { const service = this; - return safeTry(async function* () { - const field = yield* table.getField((candidate) => candidate.id().equals(fieldId)); + return safeTry, DomainError>(async function* () { const tableDto = yield* service.tableMapper.toDTO(table); - const fieldDto = tableDto.fields.find((candidate) => candidate.id === fieldId.toString()); - if (!fieldDto) { - return err(domainError.notFound({ message: 'Field snapshot source not found' })); + const orderedFieldIdsByViewId = yield* service.captureOrderedFieldIdsByView(table); + + const sources: SnapshotSource[] = []; + for (const fieldId of fieldIds) { + sources.push( + yield* service.buildSnapshotSource(table, tableDto, orderedFieldIdsByViewId, fieldId) + ); } - const snapshotField = yield* toFieldSnapshotInput(field, fieldDto); - const orderedFieldIdsByViewId = yield* service.captureOrderedFieldIdsByView(table); - const views = tableDto.views.map((view) => - service.toViewSnapshot( + const recordsByFieldId = + options?.includeRecords === false + ? new Map() + : yield* await service.captureRecordsForFields( + context, + table, + sources.map((source) => source.field) + ); + + return ok( + sources.map((source) => service.toSnapshot(source, recordsByFieldId.get(source.fieldId))) + ); + }); + } + + private buildSnapshotSource( + table: Table, + tableDto: { + fields: ReadonlyArray; + views: ReadonlyArray; + }, + orderedFieldIdsByViewId: ReadonlyMap>, + fieldId: FieldId + ): Result { + const field = table.getField((candidate) => candidate.id().equals(fieldId)); + if (field.isErr()) { + return err(field.error); + } + + const fieldIdText = fieldId.toString(); + const fieldDto = tableDto.fields.find((candidate) => candidate.id === fieldIdText); + if (!fieldDto) { + return err(domainError.notFound({ message: 'Field snapshot source not found' })); + } + + return toFieldSnapshotInput(field.value, fieldDto).map((snapshotField) => ({ + field: field.value, + fieldId: fieldIdText, + snapshotField, + views: tableDto.views.map((view) => + this.toViewSnapshot( view, - fieldId.toString(), + fieldIdText, orderedFieldIdsByViewId.get(view.id) ?? table.getFields().map((field) => field.id().toString()) ) - ); - const records = - options?.includeRecords === false - ? undefined - : yield* await service.captureRecords(context, table, field); - - return ok({ - field: snapshotField, - hasError: field.hasError().toBoolean(), - views, - ...(records ? { records } : {}), - }); - }); + ), + })); + } + + private toSnapshot( + source: SnapshotSource, + records: SnapshotRecordValues | undefined + ): UndoRedoFieldSnapshot { + return { + field: source.snapshotField, + hasError: source.field.hasError().toBoolean(), + views: source.views, + ...(records ? { records } : {}), + }; } private toViewSnapshot( @@ -281,6 +363,64 @@ export class FieldUndoRedoSnapshotService { return ok(queryResult.value.records.map((row) => this.toRecordSnapshot(row, fieldId))); } + private async captureRecordsForFields( + context: IExecutionContext, + table: Table, + fields: ReadonlyArray + ): Promise, DomainError>> { + const recordsByFieldId = new Map(); + const storedFields = fields.filter((field) => !field.computed().toBoolean()); + + for (const field of fields) { + if (field.computed().toBoolean()) { + recordsByFieldId.set(field.id().toString(), undefined); + } + } + + if (!storedFields.length) { + return ok(recordsByFieldId); + } + + for (const field of storedFields) { + this.ensureFieldDbFieldName(field); + } + + const queryResult = await this.tableRecordQueryRepository.find(context, table, undefined, { + mode: 'stored', + includeTotal: false, + projectionFieldIds: storedFields.map((field) => field.id()), + }); + if (queryResult.isErr()) { + if (!this.isMissingColumnError(queryResult.error)) { + return err(queryResult.error); + } + + if (storedFields.length === 1) { + recordsByFieldId.set(storedFields[0]!.id().toString(), []); + return ok(recordsByFieldId); + } + + for (const field of storedFields) { + const fieldRecordsResult = await this.captureRecords(context, table, field); + if (fieldRecordsResult.isErr()) { + return err(fieldRecordsResult.error); + } + recordsByFieldId.set(field.id().toString(), fieldRecordsResult.value); + } + return ok(recordsByFieldId); + } + + for (const field of storedFields) { + const fieldId = field.id().toString(); + recordsByFieldId.set( + fieldId, + queryResult.value.records.map((row) => this.toRecordSnapshot(row, fieldId)) + ); + } + + return ok(recordsByFieldId); + } + private toRecordSnapshot( row: TableRecordReadModel, fieldId: string diff --git a/packages/v2/core/src/application/services/ForeignTableLoaderService.spec.ts b/packages/v2/core/src/application/services/ForeignTableLoaderService.spec.ts index 7877a18fe6..2b77185ad8 100644 --- a/packages/v2/core/src/application/services/ForeignTableLoaderService.spec.ts +++ b/packages/v2/core/src/application/services/ForeignTableLoaderService.spec.ts @@ -125,4 +125,25 @@ describe('ForeignTableLoaderService', () => { table.id().toString() ); }); + + it('loads link-title fill foreign tables for single link object mutate specs', async () => { + const table = buildTable('j', 'k'); + const repo = new MemoryTableRepository(); + const context = createContext(); + await repo.insert(context, table); + + const service = new ForeignTableLoaderService(repo); + const fieldId = FieldId.create(`fld${'l'.repeat(16)}`)._unsafeUnwrap(); + const spec = new SetLinkValueSpec( + fieldId, + CellValue.fromValidated({ id: `rec${'m'.repeat(16)}` }), + table.id() + ); + + const result = await service.loadForLinkTitleFill(context, [spec]); + + expect(result._unsafeUnwrap().get(table.id().toString())?.id().toString()).toBe( + table.id().toString() + ); + }); }); diff --git a/packages/v2/core/src/application/services/PreloadedTableContext.ts b/packages/v2/core/src/application/services/PreloadedTableContext.ts new file mode 100644 index 0000000000..cbd30322f5 --- /dev/null +++ b/packages/v2/core/src/application/services/PreloadedTableContext.ts @@ -0,0 +1,49 @@ +import type { BaseId } from '../../domain/base/BaseId'; +import type { Table } from '../../domain/table/Table'; +import type { TableId } from '../../domain/table/TableId'; +import type { IExecutionContext } from '../../ports/ExecutionContext'; + +type PreloadedTableExecutionContext = IExecutionContext & { + __preloadedTables?: Map; +}; + +const cacheKey = (baseId: BaseId, tableId: TableId): string => + `${baseId.toString()}:${tableId.toString()}`; + +const getCache = (context: IExecutionContext): Map => { + const cacheContext = context as PreloadedTableExecutionContext; + cacheContext.__preloadedTables ??= new Map(); + return cacheContext.__preloadedTables; +}; + +export const bindPreloadedTableToExecutionContext = ( + context: IExecutionContext, + table: Table +): void => { + if ( + typeof (table as { baseId?: unknown }).baseId !== 'function' || + typeof (table as { id?: unknown }).id !== 'function' + ) { + return; + } + + getCache(context).set(cacheKey(table.baseId(), table.id()), table); +}; + +export const consumePreloadedTableFromExecutionContext = ( + context: IExecutionContext, + params: { + baseId: BaseId; + tableId: TableId; + } +): Table | undefined => { + const cache = getCache(context); + const key = cacheKey(params.baseId, params.tableId); + const table = cache.get(key); + if (!table) { + return undefined; + } + + cache.delete(key); + return table; +}; diff --git a/packages/v2/core/src/application/services/RecordBulkUpdateService.ts b/packages/v2/core/src/application/services/RecordBulkUpdateService.ts index 949e0f410a..5f7cf66217 100644 --- a/packages/v2/core/src/application/services/RecordBulkUpdateService.ts +++ b/packages/v2/core/src/application/services/RecordBulkUpdateService.ts @@ -14,6 +14,14 @@ import { RecordsBatchUpdated } from '../../domain/table/events/RecordsBatchUpdat import { FieldId } from '../../domain/table/fields/FieldId'; import type { FieldKeyType } from '../../domain/table/fields/FieldKeyType'; import type { RecordWriteSideEffects } from '../../domain/table/fields/visitors/RecordWriteSideEffectVisitor'; +import type { + UpdateRecordTraceEvent, + UpdateRecordTraceHook, + UpdateRecordTracePhase, + UpdateRecordsStreamTraceEvent, + UpdateRecordsStreamTraceHook, + UpdateRecordsStreamTracePhase, +} from '../../domain/table/methods/records'; import type { FieldKeyMapping } from '../../domain/table/records/RecordCreateResult'; import { RecordId } from '../../domain/table/records/RecordId'; import type { RecordInsertOrder } from '../../domain/table/records/RecordInsertOrder'; @@ -37,7 +45,13 @@ import type { TableRecordReadModel } from '../../ports/TableRecordReadModel'; import { ITableRecordRepository } from '../../ports/TableRecordRepository'; import type { UpdateManyStreamBatchInput } from '../../ports/TableRecordRepository'; import { v2CoreTokens } from '../../ports/tokens'; -import { TeableSpanAttributes } from '../../ports/Tracer'; +import { + createTeableSpanAttributes, + type ISpan, + type ITracer, + type SpanAttributes, + TeableSpanAttributes, +} from '../../ports/Tracer'; import { composeUndoRedoCommands, createUndoRedoCommand, @@ -67,6 +81,7 @@ import { toUndoRedoStackAppendContext, UndoRedoStackService } from './UndoRedoSt // A hard-coded id is safe here because it never leaves the aggregate/spec build path. const BULK_UPDATE_SYNTHETIC_RECORD_ID = RecordId.create(`rec${'0'.repeat(16)}`)._unsafeUnwrap(); +const EXPLICIT_UPDATE_MAX_BATCH_SIZE = 1000; type RecordConditionSpec = ISpecification; @@ -203,6 +218,195 @@ const summarizeExplicitUpdates = ( }; }; +const traceRecordBulkUpdatePhase = async ( + context: IExecutionContext, + phase: string, + attributes: SpanAttributes, + callback: (span?: ISpan) => Promise | T +): Promise => { + const tracer = context.tracer; + if (!tracer) { + return callback(); + } + + let span; + try { + span = tracer.startSpan( + `teable.RecordBulkUpdateService.${phase}`, + createTeableSpanAttributes('service', `RecordBulkUpdateService.${phase}`, { + [TeableSpanAttributes.HANDLER]: 'RecordBulkUpdateService', + ...attributes, + }) + ); + } catch { + return callback(); + } + + return tracer.withSpan(span, async () => { + try { + return await callback(span); + } finally { + span.end(); + } + }); +}; + +const GENERATE_UPDATE_BATCH_RECORD_TRACE_SAMPLE_LIMIT = 3; + +const createHrTimer = (): (() => number) => { + const start = Date.now(); + return () => Date.now() - start; +}; + +class BulkUpdateBatchTraceCollector { + private readonly elapsedMs = createHrTimer(); + private readonly streamPhaseDurations = new Map(); + private readonly recordPhaseDurations = new Map(); + private readonly sampledRecords = new Set(); + private recordCount = 0; + private totalRecordFieldCount = 0; + private maxRecordFieldCount = 0; + + constructor( + private readonly tracer: ITracer | undefined, + private readonly attributes: SpanAttributes + ) {} + + traceStream: UpdateRecordsStreamTraceHook = ( + event: UpdateRecordsStreamTraceEvent, + callback: () => T + ): T => { + return this.traceSync( + `teable.RecordBulkUpdateService.generateUpdateBatch.${event.phase}`, + { + ...this.attributes, + 'record.update.batchTracePhase': event.phase, + 'record.update.batchIndex': event.batchIndex, + 'record.update.batchSize': event.batchSize, + 'record.update.targetBatchSize': event.targetBatchSize, + ...(event.recordIndex != null ? { 'record.update.recordIndex': event.recordIndex } : {}), + ...(event.fieldCount != null ? { 'record.update.fieldCount': event.fieldCount } : {}), + }, + this.shouldSampleStreamEvent(event), + (durationMs) => { + this.addDuration(this.streamPhaseDurations, event.phase, durationMs); + if (event.phase === 'updateRecord') { + this.recordCount += 1; + this.totalRecordFieldCount += event.fieldCount ?? 0; + this.maxRecordFieldCount = Math.max(this.maxRecordFieldCount, event.fieldCount ?? 0); + } + }, + callback + ); + }; + + traceRecord: UpdateRecordTraceHook = ( + event: UpdateRecordTraceEvent, + callback: () => Result + ): Result => { + return this.traceSync( + `teable.RecordBulkUpdateService.generateUpdateBatch.updateRecord.${event.phase}`, + { + ...this.attributes, + 'record.update.recordTracePhase': event.phase, + 'record.update.fieldCount': event.fieldCount, + ...(event.recordIndex != null ? { 'record.update.recordIndex': event.recordIndex } : {}), + ...(event.editableFieldCount != null + ? { 'record.update.editableFieldCount': event.editableFieldCount } + : {}), + }, + this.shouldSampleRecordEvent(event), + (durationMs) => this.addDuration(this.recordPhaseDurations, event.phase, durationMs), + callback + ); + }; + + flushToSpan(span: ISpan): void { + const attrs: Record = { + 'record.update.generateBatchTotalRecords': this.recordCount, + 'record.update.generateBatchTotalFieldAssignments': this.totalRecordFieldCount, + 'record.update.generateBatchMaxFieldsPerRecord': this.maxRecordFieldCount, + }; + + for (const [phase, durationMs] of this.streamPhaseDurations.entries()) { + attrs[`record.update.generateBatch.stream.${phase}.ms`] = Number(durationMs.toFixed(3)); + } + for (const [phase, durationMs] of this.recordPhaseDurations.entries()) { + attrs[`record.update.generateBatch.record.${phase}.ms`] = Number(durationMs.toFixed(3)); + } + span.setAttributes(attrs); + } + + private traceSync( + spanName: string, + attributes: SpanAttributes, + shouldCreateSpan: boolean, + onDone: (durationMs: number) => void, + callback: () => T + ): T { + const startMs = this.elapsedMs(); + const span = + shouldCreateSpan && this.tracer + ? this.tryStartSpan(spanName, { + ...attributes, + 'record.update.generateBatchElapsedMs': Number(startMs.toFixed(3)), + }) + : undefined; + + try { + return callback(); + } catch (error) { + span?.recordError(error instanceof Error ? error.message : String(error)); + throw error; + } finally { + const durationMs = this.elapsedMs() - startMs; + onDone(durationMs); + span?.setAttribute('record.update.phaseDurationMs', Number(durationMs.toFixed(3))); + span?.end(); + } + } + + private shouldSampleStreamEvent(event: UpdateRecordsStreamTraceEvent): boolean { + if (event.phase === 'yieldBatch' || event.phase === 'yieldFinalBatch') { + return true; + } + if (event.recordIndex == null) { + return false; + } + return this.shouldSampleRecordIndex(event.recordIndex); + } + + private shouldSampleRecordEvent(event: UpdateRecordTraceEvent): boolean { + if (event.recordIndex == null) { + return false; + } + return this.shouldSampleRecordIndex(event.recordIndex); + } + + private shouldSampleRecordIndex(recordIndex: number): boolean { + if (this.sampledRecords.has(recordIndex)) { + return true; + } + if (this.sampledRecords.size >= GENERATE_UPDATE_BATCH_RECORD_TRACE_SAMPLE_LIMIT) { + return false; + } + this.sampledRecords.add(recordIndex); + return true; + } + + private tryStartSpan(name: string, attributes: SpanAttributes): ISpan | undefined { + try { + return this.tracer?.startSpan(name, attributes); + } catch { + return undefined; + } + } + + private addDuration(durations: Map, phase: T, durationMs: number) { + durations.set(phase, (durations.get(phase) ?? 0) + durationMs); + } +} + @injectable() export class RecordBulkUpdateService { constructor( @@ -284,10 +488,6 @@ export class RecordBulkUpdateService { ); } - if (events.length > 0) { - yield* await service.eventBus.publishMany(context, events); - } - const tableIdText = input.table.id().toString(); const updateUndoCommands: UndoRedoCommandLeafData[] = executionResult.eventData.length > 0 @@ -335,6 +535,10 @@ export class RecordBulkUpdateService { ); } + if (events.length > 0) { + yield* await service.eventBus.publishMany(context, events); + } + await executionResult.pluginExecution.afterCommit(); activeSpan?.setAttribute('record.update.updatedCount', executionResult.updatedCount); return ok({ @@ -504,51 +708,94 @@ export class RecordBulkUpdateService { records: ReadonlyArray ): Promise> { const service = this; + const explicitTraceAttributes: SpanAttributes = { + [TeableSpanAttributes.TABLE_ID]: input.table.id().toString(), + 'record.update.variant': 'explicit', + 'record.update.inputRecordCount': records.length, + 'record.update.typecast': input.typecast, + 'record.update.hasOrder': Boolean(input.order), + }; return safeTry(async function* () { - const resolvedUpdates = yield* service.resolveExplicitUpdates( - input.table, - records, - input.fieldKeyType + const resolvedUpdates = yield* await traceRecordBulkUpdatePhase( + context, + 'resolveExplicitUpdates', + explicitTraceAttributes, + () => service.resolveExplicitUpdates(input.table, records, input.fieldKeyType) ); - const pluginExecution = yield* await service.recordWritePluginRunner.prepare({ - kind: RecordWriteOperationKind.updateMany, - executionContext: context, - table: input.table, - payload: { - variant: 'explicit', - recordUpdates: resolvedUpdates.map((update) => ({ - recordId: update.recordId, - fieldValues: update.fieldValues, - })), - fieldKeyType: input.fieldKeyType, - typecast: input.typecast, - recordIds: resolvedUpdates.map((update) => update.recordId), - recordCount: resolvedUpdates.length, - }, - isTransactionBound: false, - }); - yield* await pluginExecution.guard(); - const pluginRecordSpec = yield* pluginExecution.getRecordSpec(); - const activeSpan = context.tracer?.getActiveSpan(); const resolvedSummary = summarizeExplicitUpdates(resolvedUpdates); + const resolvedTraceAttributes: SpanAttributes = { + ...explicitTraceAttributes, + 'record.update.resolvedRecordCount': resolvedSummary.recordCount, + 'record.update.resolvedUniqueFieldCount': resolvedSummary.uniqueFieldCount, + 'record.update.resolvedTotalFieldAssignments': resolvedSummary.totalFieldAssignments, + 'record.update.resolvedMaxFieldsPerRecord': resolvedSummary.maxFieldsPerRecord, + }; + const pluginExecution = yield* await traceRecordBulkUpdatePhase( + context, + 'pluginPrepare', + resolvedTraceAttributes, + () => + service.recordWritePluginRunner.prepare({ + kind: RecordWriteOperationKind.updateMany, + executionContext: context, + table: input.table, + payload: { + variant: 'explicit', + recordUpdates: resolvedUpdates.map((update) => ({ + recordId: update.recordId, + fieldValues: update.fieldValues, + })), + fieldKeyType: input.fieldKeyType, + typecast: input.typecast, + recordIds: resolvedUpdates.map((update) => update.recordId), + recordCount: resolvedUpdates.length, + }, + isTransactionBound: false, + }) + ); + yield* await traceRecordBulkUpdatePhase(context, 'pluginGuard', resolvedTraceAttributes, () => + pluginExecution.guard() + ); + const pluginRecordSpec = yield* await traceRecordBulkUpdatePhase( + context, + 'pluginGetRecordSpec', + resolvedTraceAttributes, + () => pluginExecution.getRecordSpec() + ); + const activeSpan = context.tracer?.getActiveSpan(); activeSpan?.setAttributes({ 'record.update.resolvedRecordCount': resolvedSummary.recordCount, 'record.update.resolvedUniqueFieldCount': resolvedSummary.uniqueFieldCount, 'record.update.resolvedTotalFieldAssignments': resolvedSummary.totalFieldAssignments, }); - const currentRecordsResult = yield* await service.loadExplicitCurrentRecords( + const currentRecordsResult = yield* await traceRecordBulkUpdatePhase( context, - input.table, - resolvedUpdates.map((update) => update.recordId), - Boolean(input.order) + 'loadExplicitCurrentRecords', + resolvedTraceAttributes, + () => + service.loadExplicitCurrentRecords( + context, + input.table, + resolvedUpdates.map((update) => update.recordId), + Boolean(input.order) + ) ); - const explicitAuthorization = yield* service.filterAuthorizedExplicitUpdates( - input.table, - resolvedUpdates, - currentRecordsResult, - pluginRecordSpec + const explicitAuthorization = yield* await traceRecordBulkUpdatePhase( + context, + 'filterAuthorizedExplicitUpdates', + { + ...resolvedTraceAttributes, + 'record.update.loadedCurrentRecordCount': currentRecordsResult.records.length, + }, + () => + service.filterAuthorizedExplicitUpdates( + input.table, + resolvedUpdates, + currentRecordsResult, + pluginRecordSpec + ) ); const authorizedUpdates = explicitAuthorization.authorizedUpdates; activeSpan?.setAttributes({ @@ -580,22 +827,48 @@ export class RecordBulkUpdateService { const fieldUpdateTargets = authorizedUpdates.filter( (update) => update.fieldValues.size > 0 ); - const preparedWrite = yield* await service.prepareTableForWrite( + const transactionTraceAttributes: SpanAttributes = { + ...resolvedTraceAttributes, + 'record.update.authorizedRecordCount': authorizedUpdates.length, + 'record.update.fieldUpdateTargetCount': fieldUpdateTargets.length, + 'record.update.missingRecordCount': explicitAuthorization.missingRecordIds.length, + 'record.update.pluginFilteredRecordCount': + explicitAuthorization.pluginFilteredRecordIds.length, + }; + const preparedWrite = yield* await traceRecordBulkUpdatePhase( transactionContext, - input.table, - fieldUpdateTargets.map((update) => update.fieldValues), - input.typecast + 'prepareTableForWrite', + transactionTraceAttributes, + () => + service.prepareTableForWrite( + transactionContext, + input.table, + fieldUpdateTargets.map((update) => update.fieldValues), + input.typecast + ) ); - yield* await pluginExecution.beforePersist(transactionContext); + yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'pluginBeforePersist', + transactionTraceAttributes, + () => pluginExecution.beforePersist(transactionContext) + ); - const reorderResult = input.order - ? yield* await service.recordReorderService.reorder(transactionContext, { - table: preparedWrite.tableForWrite, - recordIds: authorizedUpdates.map((update) => update.recordId), - currentRecords: authorizedUpdates.map((update) => update.currentRecord), - order: input.order, - }) + const order = input.order; + const reorderResult = order + ? yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'reorderRecords', + transactionTraceAttributes, + () => + service.recordReorderService.reorder(transactionContext, { + table: preparedWrite.tableForWrite, + recordIds: authorizedUpdates.map((update) => update.recordId), + currentRecords: authorizedUpdates.map((update) => update.currentRecord), + order, + }) + ) : emptyRecordReorderResult(); let updatedCount = reorderResult.updatedCount; @@ -604,27 +877,85 @@ export class RecordBulkUpdateService { const updatedRecordMap = new Map(); if (fieldUpdateTargets.length > 0) { - const updateBatches = preparedWrite.tableForWrite.updateRecordsStream( - fieldUpdateTargets.map((update) => ({ - recordId: update.recordId, - fieldValues: update.fieldValues, - })), - { typecast: input.typecast } + let currentBatchTraceCollector: BulkUpdateBatchTraceCollector | undefined; + const updateBatches = yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'createUpdateRecordsStream', + transactionTraceAttributes, + () => + ok( + preparedWrite.tableForWrite.updateRecordsStream( + fieldUpdateTargets.map((update) => ({ + recordId: update.recordId, + fieldValues: update.fieldValues, + })), + { + typecast: input.typecast, + maxBatchSize: EXPLICIT_UPDATE_MAX_BATCH_SIZE, + trace: (event, callback) => + currentBatchTraceCollector + ? currentBatchTraceCollector.traceStream(event, callback) + : callback(), + traceRecord: (event, callback) => + currentBatchTraceCollector + ? currentBatchTraceCollector.traceRecord(event, callback) + : callback(), + } + ) + ) ); const persistedBatches: Array> = []; const fillLinkTitleSpecs: ICellValueSpec[] = []; let resolvedIndex = 0; let maxBatchSize = 0; + let batchIndex = 0; - for (const batchResult of updateBatches) { + const updateBatchIterator = updateBatches[Symbol.iterator](); + while (true) { + const nextBatch = yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'generateUpdateBatch', + { + ...transactionTraceAttributes, + 'record.update.batchIndex': batchIndex, + }, + (span) => { + currentBatchTraceCollector = new BulkUpdateBatchTraceCollector( + transactionContext.tracer, + { + ...transactionTraceAttributes, + 'record.update.batchIndex': batchIndex, + } + ); + try { + return ok(updateBatchIterator.next()); + } finally { + if (span) { + currentBatchTraceCollector.flushToSpan(span); + } + currentBatchTraceCollector = undefined; + } + } + ); + if (nextBatch.done) { + break; + } + const batchResult = nextBatch.value; if (batchResult.isErr()) { return err(batchResult.error); } - const resolvedBatch = yield* await service.resolveUpdateBatch( + const batchTraceAttributes: SpanAttributes = { + ...transactionTraceAttributes, + 'record.update.batchIndex': batchIndex, + 'record.update.batchInputSize': batchResult.value.length, + }; + const resolvedBatch = yield* await traceRecordBulkUpdatePhase( transactionContext, - batchResult.value + 'resolveUpdateBatch', + batchTraceAttributes, + () => service.resolveUpdateBatch(transactionContext, batchResult.value) ); for (const updateResult of batchResult.value) { @@ -639,65 +970,80 @@ export class RecordBulkUpdateService { ); maxBatchSize = Math.max(maxBatchSize, resolvedBatch.length); - for (const updateResult of resolvedBatch) { - const pending = fieldUpdateTargets[resolvedIndex]; - if (!pending) { - return err( - domainError.unexpected({ - code: 'record.update_many.event_mismatch', - message: 'Failed to map bulk record updates to resolved v2 batch results', - }) - ); - } - - const changes = yield* service.buildRecordChangesFromUpdateResult( - pending.currentRecord, - updateResult - ); - const changedValues = - changes.length > 0 - ? new Map(changes.map((change) => [change.fieldId, change.newValue])) - : undefined; - const decoratedValues = - yield* await service.recordChangedValueDecoratorService.decorateChangedFields( - preparedWrite.tableForWrite, - changedValues, - pending.currentRecord.fields - ); - const decoratedChanges = changes.map((change) => ({ - ...change, - newValue: decoratedValues?.get(change.fieldId) ?? change.newValue, - })); - pendingEventData.push({ - recordId: pending.recordId.toString(), - oldVersion: pending.currentRecord.version, - changes: decoratedChanges, - }); - const updatedFields = new Map( - updateResult.record - .fields() - .entries() - .map((entry) => [entry.fieldId.toString(), entry.value.toValue()]) - ); - for (const [fieldId, value] of decoratedValues ?? []) { - updatedFields.set(fieldId, value); - } - const mergedFields = { - ...Object.fromEntries( - Object.entries(pending.currentRecord.fields).filter( - ([, value]) => value !== null && value !== undefined - ) - ), - ...Object.fromEntries(updatedFields), - }; - const mergedRecord = yield* TableRecord.fromRawFieldValues({ - id: pending.recordId.toString(), - tableId: input.table.id(), - fields: mergedFields, - }); - updatedRecordMap.set(pending.recordId.toString(), mergedRecord); - resolvedIndex += 1; - } + yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'materializeResolvedBatch', + { + ...batchTraceAttributes, + 'record.update.resolvedBatchSize': resolvedBatch.length, + }, + async () => + safeTry(async function* () { + for (const updateResult of resolvedBatch) { + const pending = fieldUpdateTargets[resolvedIndex]; + if (!pending) { + return err( + domainError.unexpected({ + code: 'record.update_many.event_mismatch', + message: + 'Failed to map bulk record updates to resolved v2 batch results', + }) + ); + } + + const changes = yield* service.buildRecordChangesFromUpdateResult( + pending.currentRecord, + updateResult + ); + const changedValues = + changes.length > 0 + ? new Map(changes.map((change) => [change.fieldId, change.newValue])) + : undefined; + const decoratedValues = + yield* await service.recordChangedValueDecoratorService.decorateChangedFields( + preparedWrite.tableForWrite, + changedValues, + pending.currentRecord.fields + ); + const decoratedChanges = changes.map((change) => ({ + ...change, + newValue: decoratedValues?.get(change.fieldId) ?? change.newValue, + })); + pendingEventData.push({ + recordId: pending.recordId.toString(), + oldVersion: pending.currentRecord.version, + changes: decoratedChanges, + }); + const updatedFields = new Map( + updateResult.record + .fields() + .entries() + .map((entry) => [entry.fieldId.toString(), entry.value.toValue()]) + ); + for (const [fieldId, value] of decoratedValues ?? []) { + updatedFields.set(fieldId, value); + } + const mergedFields = { + ...Object.fromEntries( + Object.entries(pending.currentRecord.fields).filter( + ([, value]) => value !== null && value !== undefined + ) + ), + ...Object.fromEntries(updatedFields), + }; + const mergedRecord = yield* TableRecord.fromRawFieldValues({ + id: pending.recordId.toString(), + tableId: input.table.id(), + fields: mergedFields, + }); + updatedRecordMap.set(pending.recordId.toString(), mergedRecord); + resolvedIndex += 1; + } + + return ok(undefined); + }) + ); + batchIndex += 1; } const authorizedSummary = summarizeExplicitUpdates(authorizedUpdates); @@ -729,75 +1075,132 @@ export class RecordBulkUpdateService { } const fillLinkTitleForeignTables = input.typecast - ? yield* await service.foreignTableLoaderService.loadForLinkTitleFill( + ? yield* await traceRecordBulkUpdatePhase( transactionContext, - fillLinkTitleSpecs + 'loadForeignTablesForLinkTitleFill', + { + ...transactionTraceAttributes, + 'record.update.fillLinkTitleSpecCount': fillLinkTitleSpecs.length, + }, + () => + service.foreignTableLoaderService.loadForLinkTitleFill( + transactionContext, + fillLinkTitleSpecs + ) ) : new Map(); - const persistResult = yield* await service.tableRecordRepository.updateManyStream( + const persistResult = yield* await traceRecordBulkUpdatePhase( transactionContext, - preparedWrite.tableForWrite, - service.createSyncUpdateBatchesGenerator(persistedBatches), + 'updateManyStream', { - deferComputedUpdates: input.deferComputedUpdates, - enqueueDeferredComputedUpdates: input.enqueueDeferredComputedUpdates, - ...(input.typecast ? { fillLinkTitles: true } : {}), - ...(fillLinkTitleForeignTables.size > 0 ? { fillLinkTitleForeignTables } : {}), - } + ...transactionTraceAttributes, + 'record.update.batchCount': persistedBatches.length, + 'record.update.maxBatchSize': maxBatchSize, + 'record.update.fillLinkTitleForeignTableCount': fillLinkTitleForeignTables.size, + }, + () => + service.tableRecordRepository.updateManyStream( + transactionContext, + preparedWrite.tableForWrite, + service.createSyncUpdateBatchesGenerator(persistedBatches), + { + deferComputedUpdates: input.deferComputedUpdates, + enqueueDeferredComputedUpdates: input.enqueueDeferredComputedUpdates, + ...(input.typecast ? { fillLinkTitles: true } : {}), + ...(fillLinkTitleForeignTables.size > 0 + ? { fillLinkTitleForeignTables } + : {}), + } + ) ); - const persistedVersions = new Map( - (persistResult.updatedRecords ?? []).map((record) => [ - record.recordId.toString(), - record.newVersion, - ]) + yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'buildPersistedEventData', + { + ...transactionTraceAttributes, + 'record.update.pendingEventCount': pendingEventData.length, + 'record.update.persistedRecordCount': persistResult.updatedRecords?.length ?? 0, + }, + async () => + safeTry(async function* () { + const persistedVersions = new Map( + (persistResult.updatedRecords ?? []).map((record) => [ + record.recordId.toString(), + record.newVersion, + ]) + ); + for (const pendingEvent of pendingEventData) { + const newVersion = persistedVersions.get(pendingEvent.recordId); + if (newVersion == null) { + continue; + } + if (pendingEvent.changes.length === 0) { + continue; + } + eventData.push({ + recordId: pendingEvent.recordId, + oldVersion: pendingEvent.oldVersion, + newVersion, + changes: pendingEvent.changes, + }); + } + return ok(undefined); + }) ); - for (const pendingEvent of pendingEventData) { - const newVersion = persistedVersions.get(pendingEvent.recordId); - if (newVersion == null) { - continue; - } - if (pendingEvent.changes.length === 0) { - continue; - } - eventData.push({ - recordId: pendingEvent.recordId, - oldVersion: pendingEvent.oldVersion, - newVersion, - changes: pendingEvent.changes, - }); - } // Reorder and field updates can target the same rows; max keeps row count semantics stable. updatedCount = Math.max(updatedCount, persistResult.totalUpdated); } const committedWrite = updatedCount > 0 - ? yield* await service.commitPreparedTableWrite( + ? yield* await traceRecordBulkUpdatePhase( transactionContext, - input.table, - preparedWrite + 'commitPreparedTableWrite', + { + ...transactionTraceAttributes, + 'record.update.updatedCount': updatedCount, + 'record.update.sideEffectCount': preparedWrite.sideEffects.length, + }, + () => + service.commitPreparedTableWrite( + transactionContext, + input.table, + preparedWrite + ) ) : preparedWrite; const materializedRecords: TableRecord[] = []; - for (const update of authorizedUpdates) { - const updatedRecord = updatedRecordMap.get(update.recordId.toString()); - if (updatedRecord) { - materializedRecords.push(updatedRecord); - continue; - } - - const currentRecordEntity = yield* TableRecord.fromRawFieldValues({ - id: update.currentRecord.id, - tableId: input.table.id(), - fields: Object.fromEntries( - Object.entries(update.currentRecord.fields).filter( - ([, value]) => value !== null && value !== undefined - ) - ), - }); - materializedRecords.push(currentRecordEntity); - } + yield* await traceRecordBulkUpdatePhase( + transactionContext, + 'materializeResultRecords', + { + ...transactionTraceAttributes, + 'record.update.updatedRecordMapSize': updatedRecordMap.size, + }, + async () => + safeTry(async function* () { + for (const update of authorizedUpdates) { + const updatedRecord = updatedRecordMap.get(update.recordId.toString()); + if (updatedRecord) { + materializedRecords.push(updatedRecord); + continue; + } + + const currentRecordEntity = yield* TableRecord.fromRawFieldValues({ + id: update.currentRecord.id, + tableId: input.table.id(), + fields: Object.fromEntries( + Object.entries(update.currentRecord.fields).filter( + ([, value]) => value !== null && value !== undefined + ) + ), + }); + materializedRecords.push(currentRecordEntity); + } + return ok(undefined); + }) + ); return ok({ updatedCount, diff --git a/packages/v2/core/src/application/services/TableCreationService.spec.ts b/packages/v2/core/src/application/services/TableCreationService.spec.ts index e724882e7b..0bef282e61 100644 --- a/packages/v2/core/src/application/services/TableCreationService.spec.ts +++ b/packages/v2/core/src/application/services/TableCreationService.spec.ts @@ -12,7 +12,10 @@ import { TableId } from '../../domain/table/TableId'; import { TableName } from '../../domain/table/TableName'; import type { IExecutionContext } from '../../ports/ExecutionContext'; import type { ITableRepository } from '../../ports/TableRepository'; -import type { ITableSchemaRepository } from '../../ports/TableSchemaRepository'; +import type { + ITableSchemaRepository, + TableSchemaInsertManyOptions, +} from '../../ports/TableSchemaRepository'; import { TableCreationService } from './TableCreationService'; const createContext = (): IExecutionContext => { @@ -68,7 +71,7 @@ class FakeTableRepository implements ITableRepository { class FakeTableSchemaRepository implements ITableSchemaRepository { inserted: Table[] = []; - insertManyOptions: Array<{ knownTables?: ReadonlyArray
} | undefined> = []; + insertManyOptions: Array = []; async insert(_context: IExecutionContext, table: Table) { this.inserted.push(table); @@ -78,7 +81,7 @@ class FakeTableSchemaRepository implements ITableSchemaRepository { async insertMany( _context: IExecutionContext, tables: ReadonlyArray
, - options?: { knownTables?: ReadonlyArray
} + options?: TableSchemaInsertManyOptions ) { this.inserted.push(...tables); this.insertManyOptions.push(options); @@ -196,4 +199,60 @@ describe('TableCreationService', () => { schemaRepository.insertManyOptions[0]?.knownTables?.map((t) => t.id().toString()) ).toEqual([externalTable.id().toString(), table.id().toString()]); }); + + it('passes schema optimization options to schema repository', async () => { + const table = buildTable('e', 'a'); + const tableRepository = new FakeTableRepository(); + const schemaRepository = new FakeTableSchemaRepository(); + const sideEffectService = new FakeFieldCreationSideEffectService(); + const service = new TableCreationService( + tableRepository, + schemaRepository, + sideEffectService as never + ); + + const result = await service.execute(createContext(), { + baseId: table.baseId(), + tables: [table], + externalTables: [], + referencesByTable: [[]], + schemaOptions: { + optimizeForEmptyTables: true, + skipUndoCaptureSetup: true, + }, + }); + + expect(result.isOk()).toBe(true); + expect(schemaRepository.insertManyOptions[0]?.optimizeForEmptyTables).toBe(true); + expect(schemaRepository.insertManyOptions[0]?.skipUndoCaptureSetup).toBe(true); + }); + + it('skips field creation side effects when requested', async () => { + const table = buildTable('f', 'a'); + const externalTable = buildTable('f', 'b'); + const tableRepository = new FakeTableRepository(); + const schemaRepository = new FakeTableSchemaRepository(); + const sideEffectService = new FakeFieldCreationSideEffectService(); + const service = new TableCreationService( + tableRepository, + schemaRepository, + sideEffectService as never + ); + + const result = await service.execute(createContext(), { + baseId: table.baseId(), + tables: [table], + externalTables: [externalTable], + referencesByTable: [[]], + sideEffectOptions: { + skipFieldCreationSideEffects: true, + }, + }); + + const payload = result._unsafeUnwrap(); + expect(sideEffectService.calls).toHaveLength(0); + expect(payload.sideEffectEvents).toEqual([]); + expect(payload.tableState.get(externalTable.id().toString())).toBe(externalTable); + expect(payload.tableState.get(table.id().toString())).toBe(table); + }); }); diff --git a/packages/v2/core/src/application/services/TableCreationService.ts b/packages/v2/core/src/application/services/TableCreationService.ts index 5c85dd95c7..bf32bae1f3 100644 --- a/packages/v2/core/src/application/services/TableCreationService.ts +++ b/packages/v2/core/src/application/services/TableCreationService.ts @@ -37,6 +37,24 @@ export type TableCreationServiceInput = { * Used for determining table insertion order. */ referencesByTable: ReadonlyArray>; + + /** + * Adapter-level schema creation options. Keep this narrow: callers should only + * opt in when the newly-created physical tables are known to have no rows. + */ + schemaOptions?: { + optimizeForEmptyTables?: boolean; + skipUndoCaptureSetup?: boolean; + }; + + /** + * Application-level side effect options. Duplicate/import callers that already + * provide complete copied field structures can opt out of redundant field + * creation side effects. + */ + sideEffectOptions?: { + skipFieldCreationSideEffects?: boolean; + }; }; export type TableCreationServiceResult = { @@ -206,6 +224,8 @@ export class TableCreationService { // Create physical table structures yield* await service.tableSchemaRepository.insertMany(context, persistedTables, { knownTables: [...externalTables, ...persistedTables], + optimizeForEmptyTables: input.schemaOptions?.optimizeForEmptyTables, + skipUndoCaptureSetup: input.schemaOptions?.skipUndoCaptureSetup, }); // Build initial table state @@ -231,6 +251,14 @@ export class TableCreationService { tableState.set(table.id().toString(), table); } + if (input.sideEffectOptions?.skipFieldCreationSideEffects) { + return ok({ + persistedTables: orderedPersistedTables, + tableState, + sideEffectEvents: [], + }); + } + // Process field creation side effects const sideEffectEvents: IDomainEvent[] = []; diff --git a/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.spec.ts b/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.spec.ts index 6630f4a293..865642cacd 100644 --- a/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.spec.ts +++ b/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.spec.ts @@ -46,7 +46,8 @@ const values = (entries: ReadonlyArray = []): Record const createContext = ( kind: RecordWriteOperationKind, payload: Record, - tableLimits: TableDataSafetyLimitConfig + tableLimits: TableDataSafetyLimitConfig, + orchestration?: RecordWritePluginContext['orchestration'] ): RecordWritePluginContext => ({ kind, @@ -56,6 +57,7 @@ const createContext = ( } satisfies IExecutionContext, table, payload, + orchestration, isTransactionBound: false, }) as unknown as RecordWritePluginContext; @@ -250,6 +252,59 @@ describe('TableDataSafetyLimitRecordWritePlugin', () => { } ); + it('checks streamed operation hooks by concrete payload size and streamed chunks by chunk size', async () => { + const tableLimits = { recordValues: { maxRecordsPerMutation: 2 } }; + const operationResult = await runPlugin( + createContext( + RecordWriteOperationKind.importAppend, + { + sourceType: 'csv', + sourceColumnMap: {}, + recordsFieldValues: [], + batchSize: 2, + typecast: false, + recordCount: 3, + }, + tableLimits, + { + mode: 'stream', + scope: 'operation', + operationId: 'import-records:test', + totalRecordCount: 3, + totalChunkCount: 2, + } + ) + ); + + expect(operationResult.isOk()).toBe(true); + + const chunkResult = await runPlugin( + createContext( + RecordWriteOperationKind.importAppend, + { + sourceType: 'csv', + sourceColumnMap: {}, + recordsFieldValues: [values(), values(), values()], + batchSize: 3, + typecast: false, + recordCount: 3, + }, + tableLimits, + { + mode: 'stream', + scope: 'chunk', + operationId: 'import-records:test', + totalRecordCount: 3, + totalChunkCount: 1, + chunkIndex: 0, + } + ) + ); + + expect(chunkResult.isErr()).toBe(true); + expect(chunkResult._unsafeUnwrapErr().code).toBe('validation.limit.records_per_mutation_max'); + }); + it('rejects oversized cell values', async () => { const result = await runPlugin( createContext( diff --git a/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.ts b/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.ts index 69216e1b49..79b5015382 100644 --- a/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.ts +++ b/packages/v2/core/src/application/services/TableDataSafetyLimitRecordWritePlugin.ts @@ -51,6 +51,10 @@ const recordsFromContext = ( }; const recordCountFromContext = (context: RecordWritePluginContext): number => { + if (context.orchestration?.mode === 'stream' && context.orchestration.scope === 'operation') { + return recordsFromContext(context).length; + } + switch (context.kind) { case RecordWriteOperationKind.createOne: case RecordWriteOperationKind.submit: diff --git a/packages/v2/core/src/application/services/TableDataSafetyLimitTableOperationPlugin.ts b/packages/v2/core/src/application/services/TableDataSafetyLimitTableOperationPlugin.ts index 8340d8135f..a49468c13c 100644 --- a/packages/v2/core/src/application/services/TableDataSafetyLimitTableOperationPlugin.ts +++ b/packages/v2/core/src/application/services/TableDataSafetyLimitTableOperationPlugin.ts @@ -28,6 +28,8 @@ type PreparedTableDataSafetyOperationLimitState = { const tableNameLength = (context: TableOperationPluginContext): number => { switch (context.kind) { + case TableOperationKind.read: + return 0; case TableOperationKind.create: case TableOperationKind.duplicate: case TableOperationKind.importCsv: @@ -52,8 +54,8 @@ export class TableDataSafetyLimitTableOperationPlugin private readonly limitComposer: TableDataSafetyLimitComposer ) {} - supports(_operation: TableOperationKind): boolean { - return true; + supports(operation: TableOperationKind): boolean { + return operation !== TableOperationKind.read; } async prepare( @@ -82,6 +84,8 @@ export class TableDataSafetyLimitTableOperationPlugin if (nameResult.isErr()) return nameResult; switch (context.kind) { + case TableOperationKind.read: + return ok(undefined); case TableOperationKind.create: { const tableCountResult = await this.ensureTablesPerBaseLimit( context.executionContext, diff --git a/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.spec.ts b/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.spec.ts index bec6ca6382..597ba47d39 100644 --- a/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.spec.ts +++ b/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.spec.ts @@ -211,6 +211,45 @@ describe('TableSchemaOperationRepairHandler', () => { ); }); + it('repairs a schema-only duplicate operation and completes the same operation key', async () => { + const table = createTable('h', 'Repair Duplicate'); + const { handler, tableRepository, tableSchemaRepository, fieldCreationSideEffectService } = + createHandler([table]); + + const result = await handler.run( + context(), + operation(table, { + type: 'table.duplicate', + status: 'pending', + phase: 'metadata_pending', + payload: { + tableId: table.id().toString(), + }, + lastError: null, + }) + ); + + expect(result._unsafeUnwrap()).toEqual({ + result: { + repaired: 'table_schema', + tableIds: [table.id().toString()], + }, + }); + expect(tableSchemaRepository.ensureInsertedMany).toHaveBeenCalledWith(expect.any(Object), [ + table, + ]); + expect(fieldCreationSideEffectService.execute).toHaveBeenCalledOnce(); + expect(tableRepository.setProvisionState).toHaveBeenCalledWith( + expect.any(Object), + table, + 'ready', + expect.objectContaining({ + idempotencyKey: `repair-op:table:${table.id().toString()}`, + operationType: 'table.duplicate', + }) + ); + }); + it('keeps arbitrary table update failures visible without replaying unsafe schema repair', async () => { const table = createTable('g', 'Unsafe Update'); const { handler, tableRepository, tableSchemaRepository } = createHandler([table]); diff --git a/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.ts b/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.ts index feeacd0e9a..11f4f8f44c 100644 --- a/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.ts +++ b/packages/v2/core/src/application/services/TableSchemaOperationRepairHandler.ts @@ -25,7 +25,13 @@ import { failRecoverableTableSchemaOperation, } from './TableSchemaOperationLifecycleService'; -const repairTypes = ['table.create', 'table.create_many', 'table.import', 'table.update'] as const; +const repairTypes = [ + 'table.create', + 'table.create_many', + 'table.duplicate', + 'table.import', + 'table.update', +] as const; type PayloadRecord = Record; diff --git a/packages/v2/core/src/application/services/TableUpdateFlow.spec.ts b/packages/v2/core/src/application/services/TableUpdateFlow.spec.ts index 3378d34114..61c74f915d 100644 --- a/packages/v2/core/src/application/services/TableUpdateFlow.spec.ts +++ b/packages/v2/core/src/application/services/TableUpdateFlow.spec.ts @@ -412,6 +412,69 @@ describe('TableUpdateFlow', () => { expect(observedNames).toEqual(['Flow Table Final']); }); + it('marks physical schema updates pending before applying data schema changes', async () => { + const table = buildTable(); + const repository = new FakeTableRepository(); + const order: string[] = []; + const flow = new TableUpdateFlow( + repository, + { + insert: async () => ok(undefined), + insertMany: async () => ok(undefined), + update: async (_context, nextTable) => { + order.push(`schema-update-after-${repository.provisionStateChanges.join(',')}`); + return ok(nextTable); + }, + delete: async () => ok(undefined), + }, + new FakeEventBus(), + new FakeUnitOfWork() + ); + + const addedField = buildTextField('c', 'Added Field'); + const result = await flow.execute(createContext(), { table }, (tableToUpdate) => + tableToUpdate.update((mutator) => mutator.addField(addedField)) + ); + + expect(result.isOk()).toBe(true); + expect(order).toEqual(['schema-update-after-pending']); + expect(repository.provisionStateChanges).toEqual(['pending', 'ready']); + expect(repository.provisionOperations.map((operation) => operation?.status)).toEqual([ + 'pending', + undefined, + ]); + }); + + it('resets early pending state when physical schema prepare validation fails', async () => { + const table = buildTable(); + const repository = new FakeTableRepository(); + const flow = new TableUpdateFlow( + repository, + new FakeTableSchemaRepository(), + new FakeEventBus(), + new FakeUnitOfWork() + ); + + const addedField = buildTextField('d', 'Prepare Failed Field'); + const result = await flow.execute( + createContext(), + { table }, + (tableToUpdate) => tableToUpdate.update((mutator) => mutator.addField(addedField)), + { + hooks: { + prepare: async () => err(domainError.validation({ message: 'prepare failed' })), + }, + } + ); + + expect(result._unsafeUnwrapErr().message).toBe('prepare failed'); + expect(repository.provisionStateChanges).toEqual(['pending', 'ready']); + expect(repository.provisionOperations.map((operation) => operation?.status)).toEqual([ + 'pending', + undefined, + ]); + }); + it('keeps tables available when an outer transaction rolls back after deferring ready', async () => { const table = buildTable(); const repository = new FakeTableRepository(); @@ -464,7 +527,7 @@ describe('TableUpdateFlow', () => { ); expect(result._unsafeUnwrapErr().message).toBe('meta failed'); - expect(repository.provisionStateChanges).toEqual(['ready', 'ready']); + expect(repository.provisionStateChanges).toEqual(['pending', 'ready']); expect(repository.provisionOperations.map((operation) => operation?.status)).toEqual([ 'pending', undefined, @@ -474,6 +537,34 @@ describe('TableUpdateFlow', () => { }); }); + it('records schema failures when metadata persists but the data phase fails', async () => { + const table = buildTable(); + const repository = new FakeTableRepository(); + const flow = new TableUpdateFlow( + repository, + { + insert: async () => ok(undefined), + insertMany: async () => ok(undefined), + update: async () => err(domainError.infrastructure({ message: 'data failed' })), + delete: async () => ok(undefined), + }, + new FakeEventBus(), + new FakeUnitOfWork() + ); + + const nextName = TableName.create('Flow Table Data Failed')._unsafeUnwrap(); + const result = await flow.execute(createContext(), { table }, (tableToUpdate) => + tableToUpdate.update((mutator) => mutator.rename(nextName)) + ); + + expect(result._unsafeUnwrapErr().message).toBe('data failed'); + expect(repository.provisionStateChanges).toEqual(['ready', 'ready']); + expect(repository.provisionOperations.map((operation) => operation?.status)).toEqual([ + 'pending', + 'error', + ]); + }); + it('records repairable schema failures when an outer rollback can leave physical schema missing', async () => { const table = buildTable(); const repository = new FakeTableRepository(); @@ -499,7 +590,7 @@ describe('TableUpdateFlow', () => { ); expect(outerResult._unsafeUnwrapErr().message).toBe('outer rollback'); - expect(repository.provisionStateChanges).toEqual(['ready', 'ready']); + expect(repository.provisionStateChanges).toEqual(['pending', 'ready']); expect(repository.provisionOperations.map((operation) => operation?.status)).toEqual([ 'pending', 'error', diff --git a/packages/v2/core/src/application/services/TableUpdateFlow.ts b/packages/v2/core/src/application/services/TableUpdateFlow.ts index 13b43be5e0..2231f8a07f 100644 --- a/packages/v2/core/src/application/services/TableUpdateFlow.ts +++ b/packages/v2/core/src/application/services/TableUpdateFlow.ts @@ -173,10 +173,27 @@ export class TableUpdateFlow { events.push(...hostEvents); const mutateSpec = updated.mutateSpec; + const requiresPhysicalSchemaRepair = mayRequirePhysicalSchemaRepair(mutateSpec); let transactionContextRef: IExecutionContext | undefined; let schemaOperationStarted = false; let tableMetadataPersisted = false; + if (requiresPhysicalSchemaRepair) { + // Split meta/data databases can expose committed DDL before the meta transaction commits. + yield* await beginTableSchemaOperation( + handler.unitOfWork, + handler.tableRepository, + context, + latestTable, + { + type: 'table.update', + state: 'pending', + status: 'pending', + } + ); + schemaOperationStarted = true; + } + const transactionResult = await handler.unitOfWork.withTransaction( context, async (metaTransactionContext) => { @@ -195,18 +212,20 @@ export class TableUpdateFlow { latestTable = normalizedResult.table ?? latestTable; } - yield* await beginTableSchemaOperation( - handler.unitOfWork, - handler.tableRepository, - metaTransactionContext, - latestTable, - { - type: 'table.update', - state: 'ready', - status: 'pending', - } - ); - schemaOperationStarted = true; + if (!requiresPhysicalSchemaRepair) { + yield* await beginTableSchemaOperation( + handler.unitOfWork, + handler.tableRepository, + metaTransactionContext, + latestTable, + { + type: 'table.update', + state: 'ready', + status: 'pending', + } + ); + schemaOperationStarted = true; + } tableUpdatePersistResult = yield* await handler.tableRepository.updateOne( metaTransactionContext, @@ -255,7 +274,7 @@ export class TableUpdateFlow { abortTableUpdateTransactionScope(transactionContextRef); } if (schemaOperationStarted && shouldFailSchemaOperation(transactionResult.error)) { - await (tableMetadataPersisted && mayRequirePhysicalSchemaRepair(mutateSpec) + await (tableMetadataPersisted ? failRecoverableTableSchemaOperation( handler.unitOfWork, handler.tableRepository, @@ -278,6 +297,19 @@ export class TableUpdateFlow { }, } )); + } else if (schemaOperationStarted && requiresPhysicalSchemaRepair) { + await completeTableSchemaOperation( + handler.unitOfWork, + handler.tableRepository, + context, + latestTable, + { + type: 'table.update', + result: { + nonRepairableFailure: transactionResult.error.message, + }, + } + ); } // Preserve the original failure; the operation-state write can fail when // a reused transaction has already been aborted by the data phase. @@ -298,7 +330,7 @@ export class TableUpdateFlow { }; if (registerAfterCommit(context, finalizeReady)) { registerAfterRollback(context, async () => { - const operationResult = mayRequirePhysicalSchemaRepair(mutateSpec) + const operationResult = requiresPhysicalSchemaRepair ? await failRecoverableTableSchemaOperation( handler.unitOfWork, handler.tableRepository, diff --git a/packages/v2/core/src/commands/ClearHandler.ts b/packages/v2/core/src/commands/ClearHandler.ts index 56f6227077..e4d6501f82 100644 --- a/packages/v2/core/src/commands/ClearHandler.ts +++ b/packages/v2/core/src/commands/ClearHandler.ts @@ -43,7 +43,10 @@ import { v2CoreTokens } from '../ports/tokens'; import { TraceSpan } from '../ports/TraceSpan'; import { createUndoRedoCommand, type UndoRedoCommandLeafData } from '../ports/UndoRedoStore'; import * as UnitOfWorkPort from '../ports/UnitOfWork'; -import { buildSanitizedRecordConditionSpec } from '../queries/RecordFilterMapper'; +import { + buildSanitizedRecordConditionSpec, + replaceCurrentUserTagInFilter, +} from '../queries/RecordFilterMapper'; import { resolveVisibleRowSearch } from '../queries/RecordSearch'; import { ClearCommand } from './ClearCommand'; import { ClearStreamCommand } from './ClearStreamCommand'; @@ -231,7 +234,12 @@ export class ClearHandler implements ICommandHandler // 3. Build filter spec from effective view filter. Search-aware visible rows are handled // by the query repository so field-type-specific search semantics stay centralized. - const filterSpec = yield* buildSanitizedRecordConditionSpec(table, effectiveFilter); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + table, + effectiveFilter, + context.actorId.toString() + ); + const filterSpec = yield* buildSanitizedRecordConditionSpec(table, actorResolvedFilter); const visibleRowSearch = resolveVisibleRowSearch(command.search, orderedFieldIds); // 4. Get total row count for columns/rows type normalization @@ -327,8 +335,7 @@ export class ClearHandler implements ICommandHandler return ok({ updatedCount: 0 }); } - // 8. Build orderBy from group + sort for correct row mapping - // If none provided, fall back to view row order column (__row_{viewId}) + // 8. Build orderBy from group + sort to match the visible list row order. const effectiveGroup = command.ignoreViewQuery ? command.groupBy ?? undefined : mergedDefaults.group(); @@ -956,7 +963,12 @@ export class ClearStreamApplicationService extends ClearHandler { ? command.groupBy ?? undefined : mergedDefaults.group(); - const filterSpecResult = await buildSanitizedRecordConditionSpec(table, effectiveFilter); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + table, + effectiveFilter, + context.actorId.toString() + ); + const filterSpecResult = await buildSanitizedRecordConditionSpec(table, actorResolvedFilter); if (filterSpecResult.isErr()) { return err(filterSpecResult.error); } diff --git a/packages/v2/core/src/commands/CreateFieldHandler.spec.ts b/packages/v2/core/src/commands/CreateFieldHandler.spec.ts index 9411d527a3..46b3a71e43 100644 --- a/packages/v2/core/src/commands/CreateFieldHandler.spec.ts +++ b/packages/v2/core/src/commands/CreateFieldHandler.spec.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest'; import { FieldCreationSideEffectService } from '../application/services/FieldCreationSideEffectService'; import type { FieldUndoRedoSnapshotService } from '../application/services/FieldUndoRedoSnapshotService'; import { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; +import { bindPreloadedTableToExecutionContext } from '../application/services/PreloadedTableContext'; import { TableDataSafetyLimitFieldOperationPlugin } from '../application/services/TableDataSafetyLimitFieldOperationPlugin'; import { TableFieldLimitFieldOperationPlugin } from '../application/services/TableFieldLimitFieldOperationPlugin'; import { TableUpdateFlow } from '../application/services/TableUpdateFlow'; @@ -115,6 +116,7 @@ class TrackingFieldUndoRedoSnapshotService { class InMemoryTableRepository implements ITableRepository { tables: Table[] = []; + findOneCount = 0; async insert(_context: IExecutionContext, table: Table) { this.tables.push(table); @@ -130,6 +132,7 @@ class InMemoryTableRepository implements ITableRepository { _context: IExecutionContext, spec: ISpecification ): Promise> { + this.findOneCount += 1; const match = this.tables.find((table) => spec.isSatisfiedBy(table)); if (!match) return err(domainError.notFound({ message: 'Not found' })); return ok(match); @@ -249,6 +252,56 @@ const addTextFields = (table: Table, count: number, prefix: string): Table => { }; describe('CreateFieldHandler', () => { + it('reuses a preloaded table from the execution context', async () => { + const baseId = `bse${'p'.repeat(16)}`; + const tableId = `tbl${'q'.repeat(16)}`; + const primaryFieldId = `fld${'r'.repeat(16)}`; + const table = buildTable({ + baseId, + tableId, + tableName: 'Preloaded Host', + primaryFieldId, + }); + const tableRepository = new InMemoryTableRepository(); + tableRepository.tables.push(table); + const schemaRepository = new FakeTableSchemaRepository(); + const eventBus = new FakeEventBus(); + const unitOfWork = new FakeUnitOfWork(); + const tableUpdateFlow = new TableUpdateFlow( + tableRepository, + schemaRepository, + eventBus, + unitOfWork + ); + const handler = new CreateFieldHandler( + tableRepository, + tableUpdateFlow, + new FieldCreationSideEffectService(tableUpdateFlow), + new ForeignTableLoaderService(tableRepository), + createFieldOperationPluginRunner([new TableFieldLimitFieldOperationPlugin()]), + noopUndoRedoService, + noopFieldUndoRedoSnapshotService + ); + const context = createContext(); + bindPreloadedTableToExecutionContext(context, table); + + const command = CreateFieldCommand.create({ + baseId, + tableId, + field: { + id: `fld${'s'.repeat(16)}`, + type: 'singleLineText', + name: 'Created From Preload', + }, + })._unsafeUnwrap(); + + const result = await handler.handle(context, command); + result._unsafeUnwrap(); + + expect(tableRepository.findOneCount).toBe(0); + expect(tableRepository.tables[0]?.getFields()).toHaveLength(2); + }); + it('publishes view column meta events when create field changes grid visibility metadata', async () => { const baseId = `bse${'r'.repeat(16)}`; const tableId = `tbl${'s'.repeat(16)}`; diff --git a/packages/v2/core/src/commands/CreateFieldHandler.ts b/packages/v2/core/src/commands/CreateFieldHandler.ts index ce9f7af2e2..e83bfdb362 100644 --- a/packages/v2/core/src/commands/CreateFieldHandler.ts +++ b/packages/v2/core/src/commands/CreateFieldHandler.ts @@ -10,6 +10,7 @@ import { } from '../application/services/FieldOperationSideEffectPluginSupport'; import { FieldUndoRedoSnapshotService } from '../application/services/FieldUndoRedoSnapshotService'; import { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; +import { consumePreloadedTableFromExecutionContext } from '../application/services/PreloadedTableContext'; import { TableUpdateFlow } from '../application/services/TableUpdateFlow'; import { toUndoRedoStackAppendContext, @@ -25,6 +26,7 @@ import * as ExecutionContextPort from '../ports/ExecutionContext'; import { FieldOperationKind, FieldOperationTargetKind } from '../ports/FieldOperationPlugin'; import * as TableRepositoryPort from '../ports/TableRepository'; import { v2CoreTokens } from '../ports/tokens'; +import { TeableSpanAttributes, createTeableSpanAttributes, type ISpan } from '../ports/Tracer'; import { TraceSpan } from '../ports/TraceSpan'; import { createUndoRedoCommand } from '../ports/UndoRedoStore'; import type { ResolvedTableFieldInput } from '../schemas/field'; @@ -43,6 +45,42 @@ export class CreateFieldResult { } } +const withCreateFieldSpan = async ( + context: ExecutionContextPort.IExecutionContext, + operation: string, + callback: () => T | Promise, + extra: Record = {} +): Promise => { + const tracer = context.tracer; + let span: ISpan | undefined; + try { + span = tracer?.startSpan( + `teable.CreateFieldHandler.${operation}`, + createTeableSpanAttributes('handler', `CreateFieldHandler.${operation}`, { + [TeableSpanAttributes.HANDLER]: 'CreateFieldHandler', + ...extra, + }) + ); + } catch { + span = undefined; + } + + if (!span || !tracer) { + return await callback(); + } + + return tracer.withSpan(span, async () => { + try { + return await callback(); + } catch (error) { + span.recordError(error instanceof Error ? error.message : String(error)); + throw error; + } finally { + span.end(); + } + }); +}; + @CommandHandler(CreateFieldCommand) @injectable() export class CreateFieldHandler implements ICommandHandler { @@ -70,40 +108,83 @@ export class CreateFieldHandler implements ICommandHandler> { const handler = this; return safeTry(async function* () { - const foreignTableReferences = yield* command.foreignTableReferences(); - const foreignTables = yield* await handler.foreignTableLoaderService.load(context, { - // Foreign references may point to tables in other bases (e.g. cross-base lookup). - references: foreignTableReferences, - }); + const spanAttrs = { + [TeableSpanAttributes.TABLE_ID]: command.tableId.toString(), + }; + const foreignTableReferences = yield* await withCreateFieldSpan( + context, + 'resolveForeignTableReferences', + async () => command.foreignTableReferences(), + spanAttrs + ); + const foreignTables = yield* await withCreateFieldSpan( + context, + 'loadForeignTables', + () => + handler.foreignTableLoaderService.load(context, { + // Foreign references may point to tables in other bases (e.g. cross-base lookup). + references: foreignTableReferences, + }), + spanAttrs + ); const tableSpec = yield* TableAggregate.specs(command.baseId).byId(command.tableId).build(); - const table = yield* await handler.tableRepository.findOne(context, tableSpec); + const preloadedTable = consumePreloadedTableFromExecutionContext(context, { + baseId: command.baseId, + tableId: command.tableId, + }); + let table = preloadedTable; + if (!table) { + table = yield* await handler.tableRepository.findOne(context, tableSpec); + } const domainContext = ExecutionContextPort.getDomainContext(context); const existingNames = table.getFields().map((field) => field.name().toString()); - const resolvedField = yield* resolveTableFieldInputName(command.field, existingNames, { - t: context.$t, - hostTable: table, - foreignTables, - }); - const normalizedField = handler.populateLinkLookupFieldId(resolvedField, foreignTables); - const field = yield* parseTableFieldSpec(normalizedField, { - isPrimary: false, - executionContext: context, - }) - .andThen((spec) => spec.createField({ baseId: command.baseId, tableId: command.tableId })) - .andThen((field) => { - if (!normalizedField.dbFieldName) { - return ok(field); - } + const resolvedField = yield* await withCreateFieldSpan( + context, + 'resolveTableFieldInputName', + async () => + resolveTableFieldInputName(command.field, existingNames, { + t: context.$t, + hostTable: table, + foreignTables, + }), + spanAttrs + ); + const normalizedField = await withCreateFieldSpan( + context, + 'populateLinkLookupFieldId', + async () => handler.populateLinkLookupFieldId(resolvedField, foreignTables), + spanAttrs + ); + const field = yield* await withCreateFieldSpan( + context, + 'parseTableFieldSpec', + async () => + parseTableFieldSpec(normalizedField, { + isPrimary: false, + executionContext: context, + }) + .andThen((spec) => + spec.createField({ baseId: command.baseId, tableId: command.tableId }) + ) + .andThen((field) => { + if (!normalizedField.dbFieldName) { + return ok(field); + } - return DbFieldName.rehydrate(normalizedField.dbFieldName) - .andThen((dbFieldName) => field.setDbFieldName(dbFieldName)) - .map(() => field); - }); - const plannedSideEffects = yield* collectFieldCreationAddSideEffects( - table, - [field], - foreignTables, - domainContext + return DbFieldName.rehydrate(normalizedField.dbFieldName) + .andThen((dbFieldName) => field.setDbFieldName(dbFieldName)) + .map(() => field); + }), + spanAttrs + ); + const plannedSideEffects = yield* await withCreateFieldSpan( + context, + 'collectFieldCreationAddSideEffects', + () => collectFieldCreationAddSideEffects(table, [field], foreignTables, domainContext), + { + ...spanAttrs, + [TeableSpanAttributes.FIELD_ID]: field.id().toString(), + } ); const basePluginContext = { kind: FieldOperationKind.create, @@ -224,25 +305,43 @@ export class CreateFieldHandler implements ICommandHandler + handler.fieldUndoRedoSnapshotService.capture( + context, + updateResult.table, + createdField.id() + ), + { + ...spanAttrs, + [TeableSpanAttributes.FIELD_ID]: createdField.id().toString(), + } ); - yield* await handler.undoRedoStackService.appendEntry( - toUndoRedoStackAppendContext(context), - updateResult.table.id(), + yield* await withCreateFieldSpan( + context, + 'appendUndoRedoEntry', + () => + handler.undoRedoStackService.appendEntry( + toUndoRedoStackAppendContext(context), + updateResult.table.id(), + { + undoCommand: createUndoRedoCommand('DeleteField', { + baseId: command.baseId.toString(), + tableId: command.tableId.toString(), + fieldId: createdField.id().toString(), + }), + redoCommand: createUndoRedoCommand('ApplyFieldSnapshot', { + baseId: command.baseId.toString(), + tableId: command.tableId.toString(), + snapshot, + }), + } + ), { - undoCommand: createUndoRedoCommand('DeleteField', { - baseId: command.baseId.toString(), - tableId: command.tableId.toString(), - fieldId: createdField.id().toString(), - }), - redoCommand: createUndoRedoCommand('ApplyFieldSnapshot', { - baseId: command.baseId.toString(), - tableId: command.tableId.toString(), - snapshot, - }), + ...spanAttrs, + [TeableSpanAttributes.FIELD_ID]: createdField.id().toString(), } ); } diff --git a/packages/v2/core/src/commands/CreateTableCommand.spec.ts b/packages/v2/core/src/commands/CreateTableCommand.spec.ts index d6c7e0f37e..a62753502e 100644 --- a/packages/v2/core/src/commands/CreateTableCommand.spec.ts +++ b/packages/v2/core/src/commands/CreateTableCommand.spec.ts @@ -27,6 +27,8 @@ const createContextWithSelectOptionLimit = (maxChoicesPerField: number): IExecut }, }, }); +const createSelectOptions = (count: number) => + Array.from({ length: count }, (_, index) => `Option ${index + 1}`); const buildFromCommand = (command: CreateTableCommand) => { return buildTable(command); @@ -147,6 +149,36 @@ describe('CreateTableCommand', () => { expect(result._unsafeUnwrapErr().code).toBe('validation.field.select_options_limit'); }); + it('builds select fields above the default option limit when runtime table limits allow it', () => { + const baseId = createBaseId('n')._unsafeUnwrap(); + const options = createSelectOptions(1001); + + const result = CreateTableCommand.create( + { + baseId: baseId.toString(), + name: 'Runtime Select Limit', + fields: [ + { type: 'singleLineText', name: 'Name', isPrimary: true }, + { + type: 'singleSelect', + name: 'Status', + options, + }, + { + type: 'multipleSelect', + name: 'Tags', + options, + }, + ], + }, + { executionContext: createContextWithSelectOptionLimit(1001) } + ); + + expect(result.isOk()).toBe(true); + const tableResult = result.andThen((command) => buildTable(command)); + expect(tableResult.isOk()).toBe(true); + }); + it('creates command with all field types and view types', () => { const baseId = createBaseId('b')._unsafeUnwrap(); diff --git a/packages/v2/core/src/commands/CreateTableHandler.ts b/packages/v2/core/src/commands/CreateTableHandler.ts index 629170de94..815f59236b 100644 --- a/packages/v2/core/src/commands/CreateTableHandler.ts +++ b/packages/v2/core/src/commands/CreateTableHandler.ts @@ -3,6 +3,11 @@ import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; import { FieldCreationSideEffectService } from '../application/services/FieldCreationSideEffectService'; +import { FieldOperationPluginRunner } from '../application/services/FieldOperationPluginRunner'; +import { + collectFieldCreationAddSideEffects, + prepareFieldAddSideEffectPlugins, +} from '../application/services/FieldOperationSideEffectPluginSupport'; import { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; import { RecordWritePluginRunner } from '../application/services/RecordWritePluginRunner'; import { TableOperationPluginRunner } from '../application/services/TableOperationPluginRunner'; @@ -18,6 +23,7 @@ import type { Table } from '../domain/table/Table'; import { NoopLogger } from '../ports/defaults/NoopLogger'; import * as EventBusPort from '../ports/EventBus'; import * as ExecutionContextPort from '../ports/ExecutionContext'; +import { FieldOperationKind } from '../ports/FieldOperationPlugin'; import { DefaultTableMapper } from '../ports/mappers/defaults/DefaultTableMapper'; import { RecordWriteOperationKind } from '../ports/RecordWritePlugin'; import { TableOperationKind } from '../ports/TableOperationPlugin'; @@ -69,6 +75,12 @@ export class CreateTableHandler implements ICommandHandler @@ -147,21 +176,27 @@ export class CreateTableHandler implements ICommandHandler - safeTry<{ sideEffectEvents: ReadonlyArray }, DomainError>( - async function* () { + const previousPendingInitializationContext = context.pendingTableDataInitialization; + context.pendingTableDataInitialization = { tableIds: [persistedTable.id().toString()] }; + const dataResult = await (async () => { + try { + return await handler.unitOfWork.withTransaction( + context, + async (dataTransactionContext) => + safeTry<{ sideEffectEvents: ReadonlyArray }, DomainError>( + async function* () { yield* await handler.tableSchemaRepository.insert( dataTransactionContext, persistedTable ); + yield* await sideEffectPluginExecution.beforePersist(dataTransactionContext); const sideEffectResult = yield* await handler.fieldCreationSideEffectService.execute( dataTransactionContext, { table, fields: tableFields, foreignTables: foreignTablesForSideEffects, + domainContext, } ); if (recordsFieldValues.length > 0) { @@ -191,10 +226,14 @@ export class CreateTableHandler implements ICommandHandler - safeTry(async function* () { + const previousPendingInitializationContext = context.pendingTableDataInitialization; + context.pendingTableDataInitialization = { + tableIds: metadataResult.map((table) => table.id().toString()), + }; + const dataResult = await (async () => { + try { + return await handler.unitOfWork.withTransaction( + context, + async (dataTransactionContext) => + safeTry(async function* () { const creationResult = yield* await handler.tableCreationService.provisionData( dataTransactionContext, { @@ -447,9 +453,13 @@ export class CreateTablesHandler tableState: creationResult.tableState, sideEffectEvents: creationResult.sideEffectEvents, }); - }), - { scope: 'data' } - ); + }), + { scope: 'data' } + ); + } finally { + context.pendingTableDataInitialization = previousPendingInitializationContext; + } + })(); if (dataResult.isErr()) { yield* await failTablesSchemaOperation( diff --git a/packages/v2/core/src/commands/DeleteByRangeHandler.spec.ts b/packages/v2/core/src/commands/DeleteByRangeHandler.spec.ts index 3d5cb55f17..f00f26bf6b 100644 --- a/packages/v2/core/src/commands/DeleteByRangeHandler.spec.ts +++ b/packages/v2/core/src/commands/DeleteByRangeHandler.spec.ts @@ -41,6 +41,7 @@ import type { RecordStoredSnapshot, } from '../ports/TableRecordRepository'; import type { ITableRepository } from '../ports/TableRepository'; +import type { ISpan, ITracer, SpanAttributes } from '../ports/Tracer'; import type { IUnitOfWork, UnitOfWorkOperation } from '../ports/UnitOfWork'; import { DeleteByRangeCommand } from './DeleteByRangeCommand'; import { DeleteByRangeHandler } from './DeleteByRangeHandler'; @@ -51,9 +52,33 @@ import { } from './recordWritePluginRunnerTestUtils'; import { createNoopUndoRedoStackService } from './undoRedoStackServiceTestUtils'; -const createContext = (): IExecutionContext => { +class FakeSpan implements ISpan { + end = () => undefined; + recordError = (_message: string) => undefined; + setAttribute = (_key: string, _value: string | number | boolean) => undefined; + setAttributes = (_attributes: SpanAttributes) => undefined; +} + +class FakeTracer implements ITracer { + readonly spans: Array<{ name: string; attributes?: SpanAttributes }> = []; + + startSpan(name: string, attributes?: SpanAttributes): ISpan { + this.spans.push({ name, attributes }); + return new FakeSpan(); + } + + async withSpan(_span: ISpan, callback: () => Promise): Promise { + return callback(); + } + + getActiveSpan(): ISpan | undefined { + return undefined; + } +} + +const createContext = (tracer?: ITracer): IExecutionContext => { const actorId = ActorId.create('system')._unsafeUnwrap(); - return { actorId, requestId: 'req-delete-direct-test' }; + return { actorId, requestId: 'req-delete-direct-test', tracer }; }; const noopUndoRedoService = createNoopUndoRedoStackService(); @@ -272,18 +297,40 @@ class FakeUnitOfWork implements IUnitOfWork { class FakeTableRecordQueryRepository implements ITableRecordQueryRepository { records: TableRecordReadModel[] = []; total = 0; + findCalls: Array<{ + spec?: ISpecification; + options?: ITableRecordQueryOptions; + }> = []; async find( _context: IExecutionContext, _table: Table, - _spec?: ISpecification, + spec?: ISpecification, options?: ITableRecordQueryOptions ): Promise> { + this.findCalls.push({ spec, options }); + const scopedRecords = + spec instanceof RecordByIdsSpec + ? this.records.filter((record) => + spec.recordIds().some((recordId) => recordId.toString() === record.id) + ) + : this.records; + const orderedRecords = options?.recordIdsOrder?.length + ? options.recordIdsOrder + .map((recordId) => scopedRecords.find((record) => record.id === recordId.toString())) + .filter((record): record is TableRecordReadModel => Boolean(record)) + : scopedRecords; // Simulate pagination - OffsetPagination has methods that return value objects const offset = options?.pagination?.offset()?.toNumber() ?? 0; - const limit = options?.pagination?.limit()?.toNumber() ?? this.records.length; - const paginatedRecords = this.records.slice(offset, offset + limit); - return ok({ records: paginatedRecords, total: this.total || this.records.length }); + const limit = options?.pagination?.limit()?.toNumber() ?? orderedRecords.length; + const paginatedRecords = orderedRecords.slice(offset, offset + limit); + return ok({ + records: paginatedRecords, + total: + options?.includeTotal === false + ? paginatedRecords.length + : this.total || scopedRecords.length, + }); } async findOne( @@ -379,6 +426,13 @@ describe('DeleteByRangeHandler', () => { [textFieldId.toString()]: 'Record A', }); expect(deletedEvent?.recordSnapshots[0].displayName).toBe('Record A'); + expect( + queryRepository.findCalls[0].options?.projectionFieldIds?.map((id) => id.toString()) + ).toEqual([textFieldId.toString()]); + expect(queryRepository.findCalls[0].options?.includeTotal).toBe(false); + expect(queryRepository.findCalls.some((call) => call.spec instanceof RecordByIdsSpec)).toBe( + false + ); }); it('skips plugins that do not support deleteMany', async () => { @@ -460,6 +514,118 @@ describe('DeleteByRangeHandler', () => { }); }); + it('validates delete plugin record scope with a narrow projection', async () => { + const { table, tableId, viewId, textFieldId, numberFieldId } = buildTable(); + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table); + + const queryRepository = new FakeTableRecordQueryRepository(); + queryRepository.records = [ + { + id: `rec${'a'.repeat(16)}`, + fields: { [textFieldId.toString()]: 'Record A', [numberFieldId.toString()]: 42 }, + version: 1, + }, + { + id: `rec${'b'.repeat(16)}`, + fields: { [textFieldId.toString()]: 'Record B', [numberFieldId.toString()]: 42 }, + version: 1, + }, + ]; + queryRepository.total = 2; + const scopedValues: unknown[] = []; + const recordSpec: ISpecification = { + isSatisfiedBy(record) { + const value = record.fields().get(numberFieldId)?.toValue(); + scopedValues.push(value); + return value === 42; + }, + mutate(record) { + return ok(record); + }, + accept() { + return ok(undefined); + }, + }; + const plugin = { + name: 'scoped-delete-plugin', + supports: (operation: RecordWriteOperationKind) => + operation === RecordWriteOperationKind.deleteMany, + prepare: () => ok(undefined), + scope: () => ok({ recordSpec }), + guard: () => ok(undefined), + beforePersist: () => ok(undefined), + afterCommit: () => ok(undefined), + }; + + const handler = createHandler({ + tableRepository, + queryRepository, + plugins: [plugin], + }); + + const command = DeleteByRangeCommand.create({ + tableId: tableId.toString(), + viewId, + ranges: [[0, 1]], + type: 'rows', + })._unsafeUnwrap(); + + const result = await handler.handle(createContext(), command); + result._unsafeUnwrap(); + + expect( + queryRepository.findCalls[0].options?.projectionFieldIds?.map((id) => id.toString()) + ).toEqual([textFieldId.toString()]); + expect(queryRepository.findCalls[0].options?.includeTotal).toBe(false); + const scopeReadCall = queryRepository.findCalls.filter( + (call) => call.options?.includeTotal === false + )[1]; + expect(scopeReadCall?.options?.projectionFieldIds?.map((id) => id.toString())).toEqual([ + textFieldId.toString(), + ]); + expect(scopedValues).toEqual([]); + }); + + it('traces direct delete preparation, persistence, and finalization spans', async () => { + const { table, tableId, viewId } = buildTable(); + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table); + + const queryRepository = new FakeTableRecordQueryRepository(); + queryRepository.records = [ + { id: `rec${'a'.repeat(16)}`, fields: { title: 'Record A' }, version: 1 }, + { id: `rec${'b'.repeat(16)}`, fields: { title: 'Record B' }, version: 1 }, + ]; + queryRepository.total = 2; + const tracer = new FakeTracer(); + + const handler = createHandler({ + tableRepository, + queryRepository, + }); + + const command = DeleteByRangeCommand.create({ + tableId: tableId.toString(), + viewId, + ranges: [[0, 1]], + type: 'rows', + })._unsafeUnwrap(); + + const result = await handler.handle(createContext(tracer), command); + result._unsafeUnwrap(); + + expect(tracer.spans.map((span) => span.name)).toEqual( + expect.arrayContaining([ + 'teable.DeleteByRangeApplicationService.prepareDeletePlan', + 'teable.DeleteByRangeApplicationService.prepareDeletePlugins', + 'teable.DeleteByRangeApplicationService.validateDeletePluginScope', + 'teable.DeleteByRangeApplicationService.deleteRecords', + 'teable.DeleteByRangeApplicationService.finalizeDeletePlan', + ]) + ); + }); + it('returns empty result when no records in range', async () => { const { table, tableId, viewId } = buildTable(); const tableRepository = new FakeTableRepository(); diff --git a/packages/v2/core/src/commands/DeleteByRangeStreamHandler.spec.ts b/packages/v2/core/src/commands/DeleteByRangeStreamHandler.spec.ts index 4d35738873..1a963735d2 100644 --- a/packages/v2/core/src/commands/DeleteByRangeStreamHandler.spec.ts +++ b/packages/v2/core/src/commands/DeleteByRangeStreamHandler.spec.ts @@ -478,6 +478,14 @@ describe('DeleteByRangeStreamHandler', () => { expect(queryRepository.findCalls.some((call) => call.spec instanceof RecordByIdsSpec)).toBe( false ); + const chunkLoadCalls = queryRepository.findCalls.filter( + (call) => call.options?.pagination && call.options.includeTotal === false + ); + expect( + chunkLoadCalls.map((call) => + call.options?.projectionFieldIds?.map((fieldId) => fieldId.toString()) + ) + ).toEqual([[table.primaryFieldId().toString()], [table.primaryFieldId().toString()]]); expect(eventBus.publishManyCalls).toHaveLength(2); expect(undoRedoService.recordEntryCalls).toHaveLength(2); expect( @@ -622,7 +630,11 @@ describe('DeleteByRangeStreamHandler', () => { expect(tracer.spans.map((span) => span.name)).toEqual( expect.arrayContaining([ + 'teable.DeleteByRangeApplicationService.prepareDeleteStreamPlan', + 'teable.DeleteByRangeApplicationService.prepareDeleteStreamPlugins', 'teable.DeleteByRangeApplicationService.loadDeleteChunk', + 'teable.DeleteByRangeApplicationService.prepareDeleteChunkPlugins', + 'teable.DeleteByRangeApplicationService.validateDeleteChunkPluginScope', 'teable.DeleteByRangeApplicationService.deleteChunk', 'teable.DeleteByRangeApplicationService.publishDeleteChunkEvents', 'teable.DeleteByRangeApplicationService.recordDeleteChunkUndoRedo', diff --git a/packages/v2/core/src/commands/DeleteFieldCommand.spec.ts b/packages/v2/core/src/commands/DeleteFieldCommand.spec.ts index 07bcde3c2c..47c9f12a82 100644 --- a/packages/v2/core/src/commands/DeleteFieldCommand.spec.ts +++ b/packages/v2/core/src/commands/DeleteFieldCommand.spec.ts @@ -22,6 +22,41 @@ describe('DeleteFieldCommand', () => { expect(command.fieldId.toString()).toBe(fieldId); }); + it('keeps nested bulk-delete snapshot flags separate from undo recording', () => { + const commandResult = DeleteFieldCommand.create( + { + baseId, + tableId, + fieldId, + }, + { + skipUndoRedo: true, + skipTargetSnapshot: true, + } + ); + + const command = commandResult._unsafeUnwrap(); + expect(command.skipUndoRedo()).toBe(true); + expect(command.skipTargetSnapshot()).toBe(true); + }); + + it('rejects skipping the target snapshot when undo recording is enabled', () => { + const commandResult = DeleteFieldCommand.create( + { + baseId, + tableId, + fieldId, + }, + { + skipTargetSnapshot: true, + } + ); + + expect(commandResult._unsafeUnwrapErr().message).toBe( + 'skipTargetSnapshot requires skipUndoRedo' + ); + }); + it('rejects invalid input', () => { const commandResult = DeleteFieldCommand.create({ baseId, diff --git a/packages/v2/core/src/commands/DeleteFieldCommand.ts b/packages/v2/core/src/commands/DeleteFieldCommand.ts index c7bd4078b4..88fc4d0db7 100644 --- a/packages/v2/core/src/commands/DeleteFieldCommand.ts +++ b/packages/v2/core/src/commands/DeleteFieldCommand.ts @@ -21,7 +21,8 @@ export class DeleteFieldCommand extends TableUpdateCommand { readonly baseId: BaseId, readonly tableId: TableId, readonly fieldId: FieldId, - private readonly skipUndoRedoValue = false + private readonly skipUndoRedoValue = false, + private readonly skipTargetSnapshotValue = false ) { super(baseId, tableId); } @@ -30,6 +31,7 @@ export class DeleteFieldCommand extends TableUpdateCommand { raw: unknown, options?: { skipUndoRedo?: boolean; + skipTargetSnapshot?: boolean; } ): Result { const parsed = deleteFieldInputSchema.safeParse(raw); @@ -40,12 +42,25 @@ export class DeleteFieldCommand extends TableUpdateCommand { details: z.formatError(parsed.error), }) ); + if (options?.skipTargetSnapshot === true && options?.skipUndoRedo !== true) { + return err( + domainError.validation({ + message: 'skipTargetSnapshot requires skipUndoRedo', + }) + ); + } return BaseId.create(parsed.data.baseId).andThen((baseId) => TableId.create(parsed.data.tableId).andThen((tableId) => FieldId.create(parsed.data.fieldId).map( (fieldId) => - new DeleteFieldCommand(baseId, tableId, fieldId, options?.skipUndoRedo === true) + new DeleteFieldCommand( + baseId, + tableId, + fieldId, + options?.skipUndoRedo === true, + options?.skipTargetSnapshot === true + ) ) ) ); @@ -54,4 +69,8 @@ export class DeleteFieldCommand extends TableUpdateCommand { skipUndoRedo(): boolean { return this.skipUndoRedoValue; } + + skipTargetSnapshot(): boolean { + return this.skipTargetSnapshotValue; + } } diff --git a/packages/v2/core/src/commands/DeleteFieldHandler.spec.ts b/packages/v2/core/src/commands/DeleteFieldHandler.spec.ts index b159903080..2808c39c61 100644 --- a/packages/v2/core/src/commands/DeleteFieldHandler.spec.ts +++ b/packages/v2/core/src/commands/DeleteFieldHandler.spec.ts @@ -29,6 +29,7 @@ import type { IFindOptions } from '../ports/RepositoryQuery'; import type { ITableRepository } from '../ports/TableRepository'; import type { ITableSchemaRepository } from '../ports/TableSchemaRepository'; import type { IUnitOfWork, UnitOfWorkOperation } from '../ports/UnitOfWork'; +import { flattenUndoRedoCommands } from '../ports/UndoRedoStore'; import { DeleteFieldCommand } from './DeleteFieldCommand'; import { DeleteFieldHandler } from './DeleteFieldHandler'; import { @@ -321,6 +322,43 @@ describe('DeleteFieldHandler', () => { expect(result._unsafeUnwrapErr().message).toBe('Field not found'); }); + it('skips the target undo snapshot when the caller already captured it', async () => { + const { table, baseId, tableId, secondaryFieldId } = buildTable(); + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table); + + const snapshotService = new FakeFieldUndoRedoSnapshotService(); + const handler = new DeleteFieldHandler( + tableRepository, + new TableUpdateFlow( + tableRepository, + new FakeTableSchemaRepository(), + new FakeEventBus(), + new FakeUnitOfWork() + ), + new FakeFieldDeletionSideEffectService() as unknown as FieldDeletionSideEffectService, + new FakeForeignTableLoaderService() as unknown as ForeignTableLoaderService, + createFieldOperationPluginRunner(), + noopUndoRedoService, + snapshotService as unknown as FieldUndoRedoSnapshotService + ); + + const command = DeleteFieldCommand.create( + { + baseId: baseId.toString(), + tableId: tableId.toString(), + fieldId: secondaryFieldId.toString(), + }, + { skipUndoRedo: true, skipTargetSnapshot: true } + )._unsafeUnwrap(); + + const result = await handler.handle(createContext(), command); + + expect(result.isOk()).toBe(true); + expect(snapshotService.captured).toHaveLength(0); + expect(flattenUndoRedoCommands(result._unsafeUnwrap().undoCommand)).toEqual([]); + }); + it('captures related undo snapshots from explicit field deletion reactions', async () => { const baseId = BaseId.create(`bse${'f'.repeat(16)}`)._unsafeUnwrap(); const sourceTableId = TableId.create(`tbl${'g'.repeat(16)}`)._unsafeUnwrap(); diff --git a/packages/v2/core/src/commands/DeleteFieldHandler.ts b/packages/v2/core/src/commands/DeleteFieldHandler.ts index 2f0888629a..dd5788cd05 100644 --- a/packages/v2/core/src/commands/DeleteFieldHandler.ts +++ b/packages/v2/core/src/commands/DeleteFieldHandler.ts @@ -107,11 +107,13 @@ export class DeleteFieldHandler implements ICommandHandler createUndoRedoCommand('ApplyFieldSnapshot', { baseId: relatedSnapshot.baseId, diff --git a/packages/v2/core/src/commands/DeleteFieldsHandler.spec.ts b/packages/v2/core/src/commands/DeleteFieldsHandler.spec.ts index 9191bf070b..f9c82b6c7b 100644 --- a/packages/v2/core/src/commands/DeleteFieldsHandler.spec.ts +++ b/packages/v2/core/src/commands/DeleteFieldsHandler.spec.ts @@ -2,7 +2,10 @@ import { err, ok } from 'neverthrow'; import type { Result } from 'neverthrow'; import { describe, expect, it } from 'vitest'; +import type { FieldDeletionSideEffectService } from '../application/services/FieldDeletionSideEffectService'; import type { FieldUndoRedoSnapshotService } from '../application/services/FieldUndoRedoSnapshotService'; +import type { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; +import { TableUpdateFlow } from '../application/services/TableUpdateFlow'; import type { UndoRedoStackService } from '../application/services/UndoRedoStackService'; import { BaseId } from '../domain/base/BaseId'; import { ActorId } from '../domain/shared/ActorId'; @@ -11,6 +14,7 @@ import type { IDomainEvent } from '../domain/shared/DomainEvent'; import { DomainEventName } from '../domain/shared/DomainEventName'; import { OccurredAt } from '../domain/shared/OccurredAt'; import type { ISpecification } from '../domain/shared/specification/ISpecification'; +import type { Field } from '../domain/table/fields/Field'; import { FieldId } from '../domain/table/fields/FieldId'; import { FieldName } from '../domain/table/fields/FieldName'; import type { ITableSpecVisitor } from '../domain/table/specs/ITableSpecVisitor'; @@ -18,20 +22,21 @@ import { Table } from '../domain/table/Table'; import { TableId } from '../domain/table/TableId'; import { TableName } from '../domain/table/TableName'; import type { TableSortKey } from '../domain/table/TableSortKey'; -import type { ICommandBus } from '../ports/CommandBus'; -import type { IExecutionContext } from '../ports/ExecutionContext'; +import type { IEventBus } from '../ports/EventBus'; +import type { IExecutionContext, IUnitOfWorkTransaction } from '../ports/ExecutionContext'; +import { FieldOperationKind } from '../ports/FieldOperationPlugin'; import type { IFindOptions } from '../ports/RepositoryQuery'; import type { ITableRepository } from '../ports/TableRepository'; -import { - composeUndoRedoCommands, - createUndoRedoCommand, - flattenUndoRedoCommands, - type UndoRedoCommandData, -} from '../ports/UndoRedoStore'; -import type { DeleteFieldCommand } from './DeleteFieldCommand'; -import { DeleteFieldResult } from './DeleteFieldHandler'; +import type { TableProvisionOperationOptions, TableProvisionState } from '../ports/TableRepository'; +import type { ITableSchemaRepository } from '../ports/TableSchemaRepository'; +import type { IUnitOfWork, UnitOfWorkOperation } from '../ports/UnitOfWork'; +import { flattenUndoRedoCommands, type UndoRedoCommandData } from '../ports/UndoRedoStore'; import { DeleteFieldsCommand } from './DeleteFieldsCommand'; import { DeleteFieldsHandler } from './DeleteFieldsHandler'; +import { + createFieldOperationPluginRunner, + createTrackedFieldOperationPlugin, +} from './fieldOperationPluginRunnerTestUtils'; const createContext = (): IExecutionContext => ({ actorId: ActorId.create('system')._unsafeUnwrap(), @@ -41,12 +46,37 @@ const createBaseId = (seed: string) => BaseId.create(`bse${seed.repeat(16)}`)._u const createTableId = (seed: string) => TableId.create(`tbl${seed.repeat(16)}`)._unsafeUnwrap(); const createFieldId = (seed: string) => FieldId.create(`fld${seed.repeat(16)}`)._unsafeUnwrap(); -const buildEvent = (name = 'Field deleted'): IDomainEvent => ({ +const buildEvent = (): IDomainEvent => ({ name: DomainEventName.fieldDeleted(), occurredAt: OccurredAt.now(), - payload: { name }, }); +const fieldIdsFromDeleteContexts = ( + contexts: ReadonlyArray<{ kind: string; payload: unknown }> +): string[] => + contexts.map((context) => { + expect(context.kind).toBe(FieldOperationKind.delete); + return (context.payload as { fieldId: FieldId }).fieldId.toString(); + }); + +const snapshotFieldIds = (commands: ReadonlyArray): string[] => + commands.map((command) => { + expect(command.type).toBe('ApplyFieldSnapshot'); + if (command.type !== 'ApplyFieldSnapshot') { + throw new Error('Expected ApplyFieldSnapshot command'); + } + return command.payload.snapshot.field.id; + }); + +const deleteFieldIds = (commands: ReadonlyArray): string[] => + commands.map((command) => { + expect(command.type).toBe('DeleteField'); + if (command.type !== 'DeleteField') { + throw new Error('Expected DeleteField command'); + } + return command.payload.fieldId; + }); + const buildTable = (baseId: BaseId, tableId: TableId, fieldIds: readonly FieldId[]) => { const builder = Table.builder() .withBaseId(baseId) @@ -74,9 +104,12 @@ const buildTable = (baseId: BaseId, tableId: TableId, fieldIds: readonly FieldId }; class FakeTableRepository implements ITableRepository { - constructor(private readonly table: Table) {} + tables: Table[] = []; + updated: Table[] = []; + provisionStates: Array<{ table: Table; state: TableProvisionState }> = []; async insert(_: IExecutionContext, table: Table): Promise> { + this.tables.push(table); return ok(table); } @@ -84,60 +117,155 @@ class FakeTableRepository implements ITableRepository { _: IExecutionContext, tables: ReadonlyArray
): Promise, DomainError>> { + this.tables.push(...tables); return ok([...tables]); } async findOne( _: IExecutionContext, - __: ISpecification + spec: ISpecification ): Promise> { - return ok(this.table); + const match = this.tables.find((table) => spec.isSatisfiedBy(table)); + if (!match) return err(domainError.notFound({ message: 'Table not found' })); + return ok(match); } async find( _: IExecutionContext, - __: ISpecification, + spec: ISpecification, ___?: IFindOptions ): Promise, DomainError>> { - return ok([this.table]); + return ok(this.tables.filter((table) => spec.isSatisfiedBy(table))); } async updateOne( _: IExecutionContext, - __: Table, - ___: ISpecification + table: Table, + __: ISpecification ): Promise> { + this.updated.push(table); + this.tables = this.tables.map((current) => (current.id().equals(table.id()) ? table : current)); + return ok(undefined); + } + + async restore(_: IExecutionContext, __: Table): Promise> { return ok(undefined); } async delete(_: IExecutionContext, __: Table): Promise> { return ok(undefined); } + + async setProvisionState( + _: IExecutionContext, + table: Table, + state: TableProvisionState, + __?: TableProvisionOperationOptions + ): Promise> { + this.provisionStates.push({ table, state }); + return ok(undefined); + } } -class FakeCommandBus implements ICommandBus { - calls: DeleteFieldCommand[] = []; +class FakeTableSchemaRepository implements ITableSchemaRepository { + updated: Table[] = []; - constructor(private readonly results: Array>) {} + async insert(_: IExecutionContext, __: Table): Promise> { + return ok(undefined); + } - async execute( + async insertMany( _: IExecutionContext, - command: TCommand - ): Promise> { - this.calls.push(command as DeleteFieldCommand); - const next = this.results.shift(); - if (!next) { - return err(domainError.invariant({ message: 'Missing nested delete result' })); - } - return next as Result; + __: ReadonlyArray
+ ): Promise> { + return ok(undefined); + } + + async update( + _: IExecutionContext, + table: Table, + __: ISpecification + ): Promise> { + this.updated.push(table); + return ok(table); + } + + async delete(_: IExecutionContext, __: Table): Promise> { + return ok(undefined); + } +} + +class FakeEventBus implements IEventBus { + published: IDomainEvent[] = []; + + async publish(_: IExecutionContext, event: IDomainEvent) { + this.published.push(event); + return ok(undefined); + } + + async publishMany(_: IExecutionContext, events: ReadonlyArray) { + this.published.push(...events); + return ok(undefined); + } +} + +class FakeUnitOfWork implements IUnitOfWork { + transactions: IExecutionContext[] = []; + + async withTransaction( + context: IExecutionContext, + work: UnitOfWorkOperation + ): Promise> { + const transaction: IUnitOfWorkTransaction = { kind: 'unitOfWorkTransaction' }; + const transactionContext = { ...context, transaction }; + this.transactions.push(transactionContext); + return work(transactionContext); + } +} + +class FakeFieldDeletionSideEffectService { + calls: Array<{ table: Table; fields: ReadonlyArray }> = []; + events: IDomainEvent[] = []; + + async execute( + _: IExecutionContext, + input: { table: Table; fields: ReadonlyArray; foreignTables: ReadonlyArray
} + ): Promise; appliedDeletions: [] }, DomainError>> { + this.calls.push({ + table: input.table, + fields: input.fields.map((field) => field.id()), + }); + return ok({ events: [...this.events], appliedDeletions: [] }); + } +} + +class FakeForeignTableLoaderService { + calls: Array<{ references: ReadonlyArray }> = []; + + async load( + _: IExecutionContext, + input: { references: ReadonlyArray } + ): Promise, DomainError>> { + this.calls.push({ references: [...input.references] }); + return ok([]); } } class FakeFieldUndoRedoSnapshotService { - calls: string[] = []; + captured: Array<{ tableId: TableId; fieldId: FieldId; includeRecords?: boolean }> = []; + captureManyCalls: Array<{ tableId: TableId; fieldIds: ReadonlyArray }> = []; - async capture(_: IExecutionContext, __: Table, fieldId: FieldId) { - this.calls.push(fieldId.toString()); + async capture( + _: IExecutionContext, + table: Table, + fieldId: FieldId, + options?: { includeRecords?: boolean } + ) { + this.captured.push({ + tableId: table.id(), + fieldId, + includeRecords: options?.includeRecords, + }); return ok({ field: { id: fieldId.toString(), @@ -147,187 +275,183 @@ class FakeFieldUndoRedoSnapshotService { views: [], }); } + + async captureMany(_: IExecutionContext, table: Table, fieldIds: ReadonlyArray) { + this.captureManyCalls.push({ tableId: table.id(), fieldIds: [...fieldIds] }); + return ok( + fieldIds.map((fieldId) => ({ + field: { + id: fieldId.toString(), + name: `Snapshot ${fieldId.toString()}`, + type: 'singleLineText', + }, + views: [], + })) + ); + } } class FakeUndoRedoService { entries: Array<{ tableId: TableId; entry: { - undoCommand: unknown; - redoCommand: unknown; + undoCommand: UndoRedoCommandData; + redoCommand: UndoRedoCommandData; }; }> = []; async appendEntry( _: IExecutionContext, tableId: TableId, - entry: { undoCommand: unknown; redoCommand: unknown } + entry: { undoCommand: UndoRedoCommandData; redoCommand: UndoRedoCommandData } ) { this.entries.push({ tableId, entry }); return ok(undefined); } } -const buildApplyFieldSnapshotLeaf = (baseId: BaseId, tableId: TableId, fieldId: FieldId) => - createUndoRedoCommand('ApplyFieldSnapshot', { - baseId: baseId.toString(), - tableId: tableId.toString(), - snapshot: { - field: { - id: fieldId.toString(), - name: `Snapshot ${fieldId.toString()}`, - type: 'singleLineText', - }, - views: [], - }, - }); +const createHarness = (table: Table) => { + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table); + const schemaRepository = new FakeTableSchemaRepository(); + const eventBus = new FakeEventBus(); + const unitOfWork = new FakeUnitOfWork(); + const tableUpdateFlow = new TableUpdateFlow( + tableRepository, + schemaRepository, + eventBus, + unitOfWork + ); + const sideEffectService = new FakeFieldDeletionSideEffectService(); + sideEffectService.events = [buildEvent()]; + const foreignTableLoader = new FakeForeignTableLoaderService(); + const snapshotService = new FakeFieldUndoRedoSnapshotService(); + const undoRedoService = new FakeUndoRedoService(); + + return { + tableRepository, + schemaRepository, + eventBus, + unitOfWork, + tableUpdateFlow, + sideEffectService, + foreignTableLoader, + snapshotService, + undoRedoService, + }; +}; describe('DeleteFieldsHandler', () => { - it('deletes fields, dedupes related snapshots, and records undo/redo commands', async () => { + it('bulk deletes fields once while preserving plugins and undo/redo commands', async () => { const baseId = createBaseId('a'); const tableId = createTableId('a'); const targetFieldA = createFieldId('b'); const targetFieldB = createFieldId('c'); - const relatedFieldX = createFieldId('d'); - const relatedFieldY = createFieldId('e'); - const initialTable = buildTable(baseId, tableId, [targetFieldA, targetFieldB, relatedFieldX]); - const latestTable = buildTable(baseId, tableId, [relatedFieldY]); - - const nestedResults: Array> = [ - ok( - DeleteFieldResult.create( - initialTable, - [buildEvent('first')], - composeUndoRedoCommands([ - buildApplyFieldSnapshotLeaf(baseId, tableId, targetFieldA), - buildApplyFieldSnapshotLeaf(baseId, tableId, relatedFieldX), - createUndoRedoCommand('DeleteField', { - baseId: baseId.toString(), - tableId: tableId.toString(), - fieldId: targetFieldA.toString(), - }), - ]), - createUndoRedoCommand('DeleteField', { - baseId: baseId.toString(), - tableId: tableId.toString(), - fieldId: targetFieldA.toString(), - }) - ) - ), - ok( - DeleteFieldResult.create( - latestTable, - [buildEvent('second')], - composeUndoRedoCommands([ - buildApplyFieldSnapshotLeaf(baseId, tableId, targetFieldB), - buildApplyFieldSnapshotLeaf(baseId, tableId, relatedFieldX), - buildApplyFieldSnapshotLeaf(baseId, tableId, relatedFieldY), - ]), - createUndoRedoCommand('DeleteField', { - baseId: baseId.toString(), - tableId: tableId.toString(), - fieldId: targetFieldB.toString(), - }) - ) - ), - ]; - - const commandBus = new FakeCommandBus(nestedResults); - const tableRepository = new FakeTableRepository(initialTable); - const snapshotService = new FakeFieldUndoRedoSnapshotService(); - const undoRedoService = new FakeUndoRedoService(); + const initialTable = buildTable(baseId, tableId, [targetFieldA, targetFieldB]); + const harness = createHarness(initialTable); + const { plugin, calls } = createTrackedFieldOperationPlugin([FieldOperationKind.delete]); const handler = new DeleteFieldsHandler( - commandBus, - tableRepository, - snapshotService as unknown as FieldUndoRedoSnapshotService, - undoRedoService as unknown as UndoRedoStackService + harness.tableRepository, + harness.tableUpdateFlow, + harness.sideEffectService as unknown as FieldDeletionSideEffectService, + harness.foreignTableLoader as unknown as ForeignTableLoaderService, + createFieldOperationPluginRunner([plugin]), + harness.snapshotService as unknown as FieldUndoRedoSnapshotService, + harness.undoRedoService as unknown as UndoRedoStackService ); - const commandResult = DeleteFieldsCommand.create({ + const command = DeleteFieldsCommand.create({ baseId: baseId.toString(), tableId: tableId.toString(), fieldIds: [targetFieldA.toString(), targetFieldB.toString(), targetFieldA.toString()], - }); - expect(commandResult.isOk()).toBe(true); - if (commandResult.isErr()) { - return; - } - - const result = await handler.handle(createContext(), commandResult.value); + })._unsafeUnwrap(); + const result = await handler.handle(createContext(), command); expect(result.isOk()).toBe(true); - if (result.isErr()) { - return; - } + const payload = result._unsafeUnwrap(); + expect(payload.table.getFields((field) => field.id().equals(targetFieldA))).toHaveLength(0); + expect(payload.table.getFields((field) => field.id().equals(targetFieldB))).toHaveLength(0); + expect(payload.events.length).toBeGreaterThan(0); + + expect(harness.tableRepository.updated).toHaveLength(1); + expect(harness.schemaRepository.updated).toHaveLength(1); + expect(harness.sideEffectService.calls).toHaveLength(1); + expect(harness.sideEffectService.calls[0]?.fields.map((fieldId) => fieldId.toString())).toEqual( + [targetFieldA.toString(), targetFieldB.toString()] + ); - expect(commandBus.calls).toHaveLength(2); - expect(commandBus.calls.map((command) => command.fieldId.toString())).toEqual([ + expect(harness.snapshotService.captureManyCalls).toHaveLength(1); + expect( + harness.snapshotService.captureManyCalls[0]?.fieldIds.map((fieldId) => fieldId.toString()) + ).toEqual([targetFieldA.toString(), targetFieldB.toString()]); + expect(harness.snapshotService.captured).toHaveLength(0); + + expect(fieldIdsFromDeleteContexts(calls.prepare)).toEqual([ targetFieldA.toString(), targetFieldB.toString(), ]); - expect(commandBus.calls.every((command) => command.skipUndoRedo())).toBe(true); - expect(snapshotService.calls).toEqual([targetFieldA.toString(), targetFieldB.toString()]); - expect(result.value.table).toBe(latestTable); - expect(result.value.events).toHaveLength(2); - - expect(undoRedoService.entries).toHaveLength(1); - const entry = undoRedoService.entries[0]; - expect(entry.tableId.equals(latestTable.id())).toBe(true); - - const undoLeaves = flattenUndoRedoCommands(entry.entry.undoCommand as UndoRedoCommandData); - expect( - undoLeaves - .filter((leaf) => leaf.type === 'ApplyFieldSnapshot') - .map((leaf) => leaf.payload.snapshot.field.id) - ).toEqual([ + expect(fieldIdsFromDeleteContexts(calls.guard)).toEqual([ + targetFieldA.toString(), + targetFieldB.toString(), + ]); + expect(fieldIdsFromDeleteContexts(calls.beforePersist)).toEqual([ + targetFieldA.toString(), + targetFieldB.toString(), + ]); + expect(calls.beforePersist.every((context) => context.isTransactionBound)).toBe(true); + expect(fieldIdsFromDeleteContexts(calls.afterCommit)).toEqual([ targetFieldA.toString(), targetFieldB.toString(), - relatedFieldX.toString(), - relatedFieldY.toString(), ]); - const redoLeaves = flattenUndoRedoCommands(entry.entry.redoCommand as UndoRedoCommandData); - expect(redoLeaves.map((leaf) => leaf.type)).toEqual(['DeleteField', 'DeleteField']); - expect(redoLeaves.map((leaf) => leaf.payload.fieldId)).toEqual([ + expect(harness.undoRedoService.entries).toHaveLength(1); + const entry = harness.undoRedoService.entries[0]; + expect(entry?.tableId.equals(tableId)).toBe(true); + + const undoLeaves = flattenUndoRedoCommands(entry!.entry.undoCommand); + expect(undoLeaves.map((leaf) => leaf.type)).toEqual([ + 'ApplyFieldSnapshot', + 'ApplyFieldSnapshot', + ]); + expect(snapshotFieldIds(undoLeaves)).toEqual([ targetFieldA.toString(), targetFieldB.toString(), ]); + + const redoLeaves = flattenUndoRedoCommands(entry!.entry.redoCommand); + expect(redoLeaves.map((leaf) => leaf.type)).toEqual(['DeleteField', 'DeleteField']); + expect(deleteFieldIds(redoLeaves)).toEqual([targetFieldA.toString(), targetFieldB.toString()]); }); - it('returns the nested delete error and skips undo/redo recording', async () => { + it('returns not found before updating or recording undo/redo when a target field is missing', async () => { const baseId = createBaseId('f'); const tableId = createTableId('f'); const targetFieldA = createFieldId('g'); - const targetFieldB = createFieldId('h'); - const initialTable = buildTable(baseId, tableId, [targetFieldA, targetFieldB]); - - const commandBus = new FakeCommandBus([ - err(domainError.validation({ message: 'nested delete failed' })), - ]); - const tableRepository = new FakeTableRepository(initialTable); - const snapshotService = new FakeFieldUndoRedoSnapshotService(); - const undoRedoService = new FakeUndoRedoService(); + const missingField = createFieldId('h'); + const initialTable = buildTable(baseId, tableId, [targetFieldA]); + const harness = createHarness(initialTable); const handler = new DeleteFieldsHandler( - commandBus, - tableRepository, - snapshotService as unknown as FieldUndoRedoSnapshotService, - undoRedoService as unknown as UndoRedoStackService + harness.tableRepository, + harness.tableUpdateFlow, + harness.sideEffectService as unknown as FieldDeletionSideEffectService, + harness.foreignTableLoader as unknown as ForeignTableLoaderService, + createFieldOperationPluginRunner(), + harness.snapshotService as unknown as FieldUndoRedoSnapshotService, + harness.undoRedoService as unknown as UndoRedoStackService ); - const commandResult = DeleteFieldsCommand.create({ + const command = DeleteFieldsCommand.create({ baseId: baseId.toString(), tableId: tableId.toString(), - fieldIds: [targetFieldA.toString(), targetFieldB.toString()], - }); - expect(commandResult.isOk()).toBe(true); - if (commandResult.isErr()) { - return; - } - - const result = await handler.handle(createContext(), commandResult.value); + fieldIds: [targetFieldA.toString(), missingField.toString()], + })._unsafeUnwrap(); + const result = await handler.handle(createContext(), command); expect(result.isErr()).toBe(true); - expect(result._unsafeUnwrapErr().message).toContain('nested delete failed'); - expect(undoRedoService.entries).toHaveLength(0); + expect(result._unsafeUnwrapErr().message).toBe('Field not found'); + expect(harness.tableRepository.updated).toHaveLength(0); + expect(harness.schemaRepository.updated).toHaveLength(0); + expect(harness.snapshotService.captureManyCalls).toHaveLength(0); + expect(harness.undoRedoService.entries).toHaveLength(0); }); }); diff --git a/packages/v2/core/src/commands/DeleteFieldsHandler.ts b/packages/v2/core/src/commands/DeleteFieldsHandler.ts index 399d03a5ff..70cee42997 100644 --- a/packages/v2/core/src/commands/DeleteFieldsHandler.ts +++ b/packages/v2/core/src/commands/DeleteFieldsHandler.ts @@ -2,29 +2,53 @@ import { inject, injectable } from '@teable/v2-di'; import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; +import { FieldDeletionSideEffectService } from '../application/services/FieldDeletionSideEffectService'; import { FieldUndoRedoSnapshotService } from '../application/services/FieldUndoRedoSnapshotService'; +import { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; +import { + FieldOperationPluginRunner, + type FieldOperationPluginExecution, +} from '../application/services/FieldOperationPluginRunner'; +import { TableUpdateFlow } from '../application/services/TableUpdateFlow'; import { toUndoRedoStackAppendContext, UndoRedoStackService, } from '../application/services/UndoRedoStackService'; import { domainError, type DomainError } from '../domain/shared/DomainError'; import type { IDomainEvent } from '../domain/shared/DomainEvent'; +import { composeAndSpecsOrUndefined } from '../domain/shared/specification/composeAndSpecs'; +import type { ISpecification } from '../domain/shared/specification/ISpecification'; +import type { Field } from '../domain/table/fields/Field'; +import type { FieldId } from '../domain/table/fields/FieldId'; +import { LinkForeignTableReferenceVisitor } from '../domain/table/fields/visitors/LinkForeignTableReferenceVisitor'; +import { + implementsOnTeableFieldDeleted, + type FieldDeletionContext, + type FieldDeletionReaction, +} from '../domain/table/OnTeableFieldDeleted'; +import type { ITableSpecVisitor } from '../domain/table/specs/ITableSpecVisitor'; +import { TableUpdateViewColumnMetaSpec } from '../domain/table/specs/TableUpdateViewColumnMetaSpec'; +import { TableUpdateViewQueryDefaultsSpec } from '../domain/table/specs/TableUpdateViewQueryDefaultsSpec'; import type { Table } from '../domain/table/Table'; import { Table as TableAggregate } from '../domain/table/Table'; -import { ICommandBus } from '../ports/CommandBus'; +import { TableUpdateResult } from '../domain/table/TableMutator'; +import { implementsOnTeableViewFieldDeleted } from '../domain/table/views/OnTeableViewFieldDeleted'; import * as ExecutionContextPort from '../ports/ExecutionContext'; +import { + FieldOperationKind, + FieldOperationTargetKind, + type IFieldOperationDeleteContext, +} from '../ports/FieldOperationPlugin'; import { ITableRepository } from '../ports/TableRepository'; import { v2CoreTokens } from '../ports/tokens'; import { TraceSpan } from '../ports/TraceSpan'; import { composeUndoRedoCommands, createUndoRedoCommand, - flattenUndoRedoCommands, + type UndoRedoFieldSnapshot, type UndoRedoCommandLeafData, } from '../ports/UndoRedoStore'; import { CommandHandler, type ICommandHandler } from './CommandHandler'; -import { DeleteFieldCommand } from './DeleteFieldCommand'; -import { type DeleteFieldResult } from './DeleteFieldHandler'; import { DeleteFieldsCommand } from './DeleteFieldsCommand'; export class DeleteFieldsResult { @@ -38,16 +62,27 @@ export class DeleteFieldsResult { } } +type FieldDeletionCleanup = { + readonly spec?: ISpecification; + readonly relatedFieldIds: ReadonlyArray; +}; + @CommandHandler(DeleteFieldsCommand) @injectable() export class DeleteFieldsHandler implements ICommandHandler { constructor( - @inject(v2CoreTokens.commandBus) - private readonly commandBus: ICommandBus, @inject(v2CoreTokens.tableRepository) private readonly tableRepository: ITableRepository, + @inject(v2CoreTokens.tableUpdateFlow) + private readonly tableUpdateFlow: TableUpdateFlow, + @inject(v2CoreTokens.fieldDeletionSideEffectService) + private readonly fieldDeletionSideEffectService: FieldDeletionSideEffectService, + @inject(v2CoreTokens.foreignTableLoaderService) + private readonly foreignTableLoaderService: ForeignTableLoaderService, + @inject(v2CoreTokens.fieldOperationPluginRunner) + private readonly fieldOperationPluginRunner: FieldOperationPluginRunner, @inject(v2CoreTokens.fieldUndoRedoSnapshotService) private readonly fieldUndoRedoSnapshotService: FieldUndoRedoSnapshotService, @inject(v2CoreTokens.undoRedoService) @@ -66,17 +101,17 @@ export class DeleteFieldsHandler const relatedUndoSnapshotKeys = new Set(); const targetUndoLeaves: UndoRedoCommandLeafData[] = []; const relatedUndoLeaves: UndoRedoCommandLeafData[] = []; - let latestTable: Table | undefined; const tableSpec = yield* TableAggregate.specs(command.baseId).byId(command.tableId).build(); const initialTable = yield* await handler.tableRepository.findOne(context, tableSpec); + const targetFields = yield* handler.resolveTargetFields(initialTable, command.fieldIds); - for (const fieldId of command.fieldIds) { - const snapshot = yield* await handler.fieldUndoRedoSnapshotService.capture( - context, - initialTable, - fieldId - ); + const targetSnapshots = yield* await handler.fieldUndoRedoSnapshotService.captureMany( + context, + initialTable, + command.fieldIds + ); + for (const snapshot of targetSnapshots) { targetUndoLeaves.push( createUndoRedoCommand('ApplyFieldSnapshot', { baseId: command.baseId.toString(), @@ -86,44 +121,139 @@ export class DeleteFieldsHandler ); } - for (const fieldId of command.fieldIds) { - const nestedCommand = yield* DeleteFieldCommand.create( - { - baseId: command.baseId.toString(), - tableId: command.tableId.toString(), - fieldId: fieldId.toString(), - }, - { skipUndoRedo: true } + const relatedUndoSnapshots = yield* await handler.captureRelatedUndoSnapshots( + context, + initialTable, + targetFields + ); + for (const relatedSnapshot of relatedUndoSnapshots) { + const snapshotFieldId = relatedSnapshot.snapshot.field.id; + if (targetFieldIds.has(snapshotFieldId)) { + continue; + } + const snapshotKey = `${relatedSnapshot.baseId}:${relatedSnapshot.tableId}:${snapshotFieldId}`; + if (relatedUndoSnapshotKeys.has(snapshotKey)) { + continue; + } + relatedUndoSnapshotKeys.add(snapshotKey); + relatedUndoLeaves.push( + createUndoRedoCommand('ApplyFieldSnapshot', { + baseId: relatedSnapshot.baseId, + tableId: relatedSnapshot.tableId, + snapshot: relatedSnapshot.snapshot, + }) ); + } - const nestedResult = yield* await handler.commandBus.execute< - DeleteFieldCommand, - DeleteFieldResult - >(context, nestedCommand); + const referenceVisitor = new LinkForeignTableReferenceVisitor(); + const foreignRefs = yield* referenceVisitor.collect(targetFields); + const foreignTables = yield* await handler.foreignTableLoaderService.load(context, { + references: foreignRefs, + }); - latestTable = nestedResult.table; - events.push(...nestedResult.events); - for (const leaf of flattenUndoRedoCommands(nestedResult.undoCommand)) { - if (leaf.type !== 'ApplyFieldSnapshot') { - continue; - } - const snapshotFieldId = leaf.payload.snapshot.field.id; - if (targetFieldIds.has(snapshotFieldId)) { - continue; - } - const snapshotKey = `${leaf.payload.baseId}:${leaf.payload.tableId}:${snapshotFieldId}`; - if (relatedUndoSnapshotKeys.has(snapshotKey)) { - continue; - } - relatedUndoSnapshotKeys.add(snapshotKey); - relatedUndoLeaves.push(leaf); - } - } + const pluginContexts: IFieldOperationDeleteContext[] = targetFields.map((targetField) => ({ + kind: FieldOperationKind.delete, + executionContext: context, + table: initialTable, + target: { + kind: FieldOperationTargetKind.direct, + sourceOperation: FieldOperationKind.delete, + sourceTable: initialTable, + }, + payload: { + fieldId: targetField.id(), + targetField, + foreignTables, + skipUndoRedo: true, + }, + isTransactionBound: false, + })); - if (!latestTable) { - return err(domainError.validation({ message: 'DeleteFieldsCommand requires field ids' })); + const pluginExecutions: FieldOperationPluginExecution[] = []; + for (const pluginContext of pluginContexts) { + const pluginExecution = + yield* await handler.fieldOperationPluginRunner.prepare(pluginContext); + yield* await pluginExecution.guard(); + pluginExecutions.push(pluginExecution); } + const updateResult = yield* await handler.tableUpdateFlow.execute( + context, + { table: initialTable }, + (candidate) => + candidate.update((mutator) => { + let nextMutator = mutator; + for (const fieldId of command.fieldIds) { + nextMutator = nextMutator.removeField(fieldId); + } + return nextMutator; + }), + { + hooks: { + prepare: async (transactionContext, updatedTable) => { + for (const [index, pluginExecution] of pluginExecutions.entries()) { + const beforePersistResult = await pluginExecution.beforePersist( + transactionContext, + { + ...pluginContexts[index]!, + executionContext: transactionContext, + table: updatedTable, + result: { + deletedField: targetFields[index]!, + }, + isTransactionBound: true, + } + ); + if (beforePersistResult.isErr()) { + return err(beforePersistResult.error); + } + } + + return ok([]); + }, + afterPersist: async (transactionContext, updatedTable) => + safeTry<{ events: ReadonlyArray; table: Table }, DomainError>( + async function* () { + const sideEffectResult = + yield* await handler.fieldDeletionSideEffectService.execute( + transactionContext, + { + table: updatedTable, + fields: targetFields, + foreignTables, + } + ); + + const cleanupResult = yield* await handler.executeDeletionEntityCleanups( + transactionContext, + updatedTable, + initialTable, + targetFields + ); + const cleanupEvents: IDomainEvent[] = [...cleanupResult.events]; + + for (const appliedDeletion of sideEffectResult.appliedDeletions) { + const appliedCleanupResult = yield* await handler.executeDeletionEntityCleanups( + transactionContext, + appliedDeletion.table, + appliedDeletion.previousTable, + [appliedDeletion.deletedField] + ); + cleanupEvents.push(...appliedCleanupResult.events); + } + + return ok({ + events: [...sideEffectResult.events, ...cleanupEvents], + table: cleanupResult.sourceTable, + }); + } + ), + }, + } + ); + + events.push(...updateResult.events); + const redoLeaves = command.fieldIds.map((fieldId) => createUndoRedoCommand('DeleteField', { baseId: command.baseId.toString(), @@ -134,14 +264,275 @@ export class DeleteFieldsHandler yield* await handler.undoRedoStackService.appendEntry( toUndoRedoStackAppendContext(context), - latestTable.id(), + updateResult.table.id(), { undoCommand: composeUndoRedoCommands([...targetUndoLeaves, ...relatedUndoLeaves]), redoCommand: composeUndoRedoCommands(redoLeaves), } ); - return ok(DeleteFieldsResult.create(latestTable, events)); + for (const [index, pluginExecution] of pluginExecutions.entries()) { + await pluginExecution.afterCommit({ + ...pluginContexts[index]!, + table: updateResult.table, + result: { + deletedField: targetFields[index]!, + }, + }); + } + + return ok(DeleteFieldsResult.create(updateResult.table, events)); }); } + + private resolveTargetFields( + table: Table, + fieldIds: ReadonlyArray + ): Result, DomainError> { + const fields: Field[] = []; + for (const fieldId of fieldIds) { + const targetField = table.getFields((field) => field.id().equals(fieldId))[0]; + if (!targetField) { + return err(domainError.notFound({ message: 'Field not found' })); + } + fields.push(targetField); + } + return ok(fields); + } + + private async executeDeletionEntityCleanups( + context: ExecutionContextPort.IExecutionContext, + sourceTable: Table, + previousSourceTable: Table, + deletedFields: ReadonlyArray + ): Promise< + Result< + { + sourceTable: Table; + events: ReadonlyArray; + }, + DomainError + > + > { + const handler = this; + return safeTry< + { + sourceTable: Table; + events: ReadonlyArray; + }, + DomainError + >(async function* () { + let latestSourceTable = sourceTable; + const events: IDomainEvent[] = []; + + for (const deletedField of deletedFields) { + const cleanupResult = yield* await handler.executeDeletionEntityCleanup( + context, + latestSourceTable, + previousSourceTable, + deletedField + ); + latestSourceTable = cleanupResult.sourceTable; + events.push(...cleanupResult.events); + } + + return ok({ + sourceTable: latestSourceTable, + events, + }); + }); + } + + private async executeDeletionEntityCleanup( + context: ExecutionContextPort.IExecutionContext, + sourceTable: Table, + previousSourceTable: Table, + deletedField: Field + ): Promise< + Result< + { + sourceTable: Table; + events: ReadonlyArray; + }, + DomainError + > + > { + const handler = this; + return safeTry< + { + sourceTable: Table; + events: ReadonlyArray; + }, + DomainError + >(async function* () { + const allTablesSpec = yield* TableAggregate.specs(sourceTable.baseId()).build(); + const allTables = yield* await handler.tableRepository.find(context, allTablesSpec); + const orderedTables = [ + sourceTable, + ...allTables.filter((table) => !table.id().equals(sourceTable.id())), + ]; + + let latestSourceTable = sourceTable; + const events: IDomainEvent[] = []; + + for (const table of orderedTables) { + const candidateTable = table.id().equals(latestSourceTable.id()) + ? latestSourceTable + : table; + + const cleanupResult = handler.buildDeletionCleanup(candidateTable, deletedField, { + table: candidateTable, + sourceTable: latestSourceTable, + previousSourceTable, + }); + if (cleanupResult.isErr()) return err(cleanupResult.error); + const cleanupSpec = cleanupResult.value.spec; + if (!cleanupSpec) { + continue; + } + const updateResult = yield* await handler.tableUpdateFlow.execute( + context, + { table: candidateTable }, + (table) => { + const updated = cleanupSpec.mutate(table); + if (updated.isErr()) return err(updated.error); + return ok(TableUpdateResult.create(updated.value, cleanupSpec)); + }, + { publishEvents: false } + ); + if (candidateTable.id().equals(latestSourceTable.id())) { + latestSourceTable = updateResult.table; + } + events.push(...updateResult.events); + } + + return ok({ + sourceTable: latestSourceTable, + events, + }); + }); + } + + private async captureRelatedUndoSnapshots( + context: ExecutionContextPort.IExecutionContext, + sourceTable: Table, + deletedFields: ReadonlyArray + ): Promise< + Result< + ReadonlyArray<{ + baseId: string; + tableId: string; + snapshot: UndoRedoFieldSnapshot; + }>, + DomainError + > + > { + const handler = this; + return safeTry(async function* () { + const allTablesSpec = yield* TableAggregate.specs(sourceTable.baseId()).build(); + const allTables = yield* await handler.tableRepository.find(context, allTablesSpec); + const orderedTables = [ + sourceTable, + ...allTables.filter((table) => !table.id().equals(sourceTable.id())), + ]; + const relatedSnapshots: Array<{ + baseId: string; + tableId: string; + snapshot: UndoRedoFieldSnapshot; + }> = []; + + for (const deletedField of deletedFields) { + for (const candidateTable of orderedTables) { + const cleanupResult = handler.buildDeletionCleanup(candidateTable, deletedField, { + table: candidateTable, + sourceTable, + previousSourceTable: sourceTable, + }); + if (cleanupResult.isErr()) { + return err(cleanupResult.error); + } + + for (const relatedFieldId of cleanupResult.value.relatedFieldIds) { + const snapshot = yield* await handler.fieldUndoRedoSnapshotService.capture( + context, + candidateTable, + relatedFieldId, + { includeRecords: false } + ); + relatedSnapshots.push({ + baseId: candidateTable.baseId().toString(), + tableId: candidateTable.id().toString(), + snapshot, + }); + } + } + } + + return ok(relatedSnapshots); + }); + } + + private buildDeletionCleanup( + candidateTable: Table, + deletedField: Field, + context: FieldDeletionContext + ): Result { + const specs: Array> = []; + const relatedFieldIds = new Map(); + + for (const view of candidateTable.views()) { + if (!implementsOnTeableViewFieldDeleted(view)) { + continue; + } + const result = view.onFieldDeleted(deletedField, context); + if (result.isErr()) return err(result.error); + if (result.value?.columnMeta) { + specs.push( + TableUpdateViewColumnMetaSpec.create([ + { + viewId: result.value.viewId, + fieldId: result.value.fieldId, + columnMeta: result.value.columnMeta, + }, + ]) + ); + } + if (result.value?.queryDefaults) { + specs.push( + TableUpdateViewQueryDefaultsSpec.create([ + { + viewId: result.value.viewId, + queryDefaults: result.value.queryDefaults, + }, + ]) + ); + } + } + + for (const field of candidateTable.getFields()) { + if (!implementsOnTeableFieldDeleted(field)) { + continue; + } + const result = field.onFieldDeleted(deletedField, context); + if (result.isErr()) return err(result.error); + if (result.value) { + specs.push(result.value.spec); + this.collectRelatedFieldIds(relatedFieldIds, result.value); + } + } + + return ok({ + spec: composeAndSpecsOrUndefined(specs), + relatedFieldIds: [...relatedFieldIds.values()], + }); + } + + private collectRelatedFieldIds( + accumulator: Map, + reaction: FieldDeletionReaction + ): void { + for (const fieldId of reaction.relatedFieldIds) { + accumulator.set(fieldId.toString(), fieldId); + } + } } diff --git a/packages/v2/core/src/commands/DuplicateBaseCommand.ts b/packages/v2/core/src/commands/DuplicateBaseCommand.ts index 162793d29c..ec19d63d9f 100644 --- a/packages/v2/core/src/commands/DuplicateBaseCommand.ts +++ b/packages/v2/core/src/commands/DuplicateBaseCommand.ts @@ -15,13 +15,22 @@ export interface DuplicateBaseRecordInput { autoNumber?: number; createdTime?: string; createdBy?: string; - lastModifiedTime?: string; - lastModifiedBy?: string; + lastModifiedTime?: string | null; + lastModifiedBy?: string | null; +} + +export type DuplicateBaseRecordReadPhase = 'insert' | 'linkRestore'; + +export interface DuplicateBaseRecordReadOptions { + phase?: DuplicateBaseRecordReadPhase; } export interface DuplicateBaseSource { structure: NormalizedDotTeaStructure; - records(tableId: string): AsyncIterable; + records( + tableId: string, + options?: DuplicateBaseRecordReadOptions + ): AsyncIterable; } export interface DuplicateBaseProgressEvent { diff --git a/packages/v2/core/src/commands/DuplicateBaseHandler.spec.ts b/packages/v2/core/src/commands/DuplicateBaseHandler.spec.ts index 6b5d656d14..3caba3387b 100644 --- a/packages/v2/core/src/commands/DuplicateBaseHandler.spec.ts +++ b/packages/v2/core/src/commands/DuplicateBaseHandler.spec.ts @@ -1,26 +1,379 @@ -import { ok, type Result } from 'neverthrow'; +import { err, ok, type Result } from 'neverthrow'; import { describe, expect, it, vi } from 'vitest'; import { BaseId } from '../domain/base/BaseId'; +import { domainError } from '../domain/shared/DomainError'; import { DuplicateBaseCommand } from './DuplicateBaseCommand'; import { DuplicateBaseHandler } from './DuplicateBaseHandler'; const baseId = `bse${'d'.repeat(16)}`; -const fakeTable = (id: string, name: string) => ({ +const fakeTable = (id: string, name: string, fields: unknown[] = []) => ({ id: () => ({ toString: () => id }), name: () => ({ toString: () => name }), + getFields: () => fields, }); +const createHandler = ( + overrides: { + foreignTableLoaderService?: unknown; + tableCreationService?: unknown; + tableRecordRepository?: unknown; + eventBus?: unknown; + unitOfWork?: unknown; + computedFieldBackfillService?: unknown; + } = {} +) => + new DuplicateBaseHandler( + (overrides.foreignTableLoaderService ?? {}) as never, + (overrides.tableCreationService ?? {}) as never, + (overrides.tableRecordRepository ?? {}) as never, + (overrides.eventBus ?? {}) as never, + (overrides.unitOfWork ?? {}) as never, + (overrides.computedFieldBackfillService ?? { + executeSyncMany: vi.fn(async () => ok(undefined)), + }) as never + ); + describe('DuplicateBaseHandler', () => { - it('streams table structure progress with table counts', async () => { - const handler = new DuplicateBaseHandler( - {} as never, - {} as never, - {} as never, - {} as never, - {} as never + it('restores link fields after streaming non-link record inserts', async () => { + type LinkUpdateBatch = Array<{ recordId: unknown; fieldValues: Map }>; + const operationOrder: string[] = []; + const insertedFieldIds: string[] = []; + const linkUpdateItems: LinkUpdateBatch = []; + const targetTableId = 'tblcccccccccccccccc'; + const targetTextFieldId = 'fldcccccccccccccccc'; + const targetLinkFieldId = 'flddddddddddddddddd'; + const tableRecordRepository = { + insertManyStream: vi.fn(async (_context, _table, batches: Array<{ records: unknown[] }>) => { + operationOrder.push('insert'); + for (const record of batches[0]?.records ?? []) { + insertedFieldIds.push( + ...( + record as { + fields(): { + entries(): Array<{ fieldId: { toString(): string } }>; + }; + } + ) + .fields() + .entries() + .map((entry) => entry.fieldId.toString()) + ); + } + return ok({ totalInserted: batches[0]?.records.length ?? 0 }); + }), + updateManyStream: vi.fn( + async (_context, _table, batches: AsyncIterable<{ _unsafeUnwrap(): LinkUpdateBatch }>) => { + operationOrder.push('update'); + for await (const batchResult of batches) { + linkUpdateItems.push(...batchResult._unsafeUnwrap()); + } + return ok({ totalUpdated: linkUpdateItems.length }); + } + ), + }; + const unitOfWork = { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }; + const targetTable = { + ...fakeTable(targetTableId, 'Tasks'), + updateRecordsStream: vi.fn(function* ( + updates: Array<{ recordId: unknown; fieldValues: Map }> + ) { + yield ok(updates); + }), + }; + const handler = createHandler({ + tableRecordRepository, + unitOfWork, + }); + const sourceTextFieldId = 'fldaaaaaaaaaaaaaaaa'; + const sourceLinkFieldId = 'fldbbbbbbbbbbbbbbbb'; + const sourceTableId = 'tblaaaaaaaaaaaaaaaa'; + const recordId = 'recaaaaaaaaaaaaaaaa'; + const records = vi.fn(async function* () { + yield { + recordId, + fields: { + [sourceTextFieldId]: 'Task A', + [sourceLinkFieldId]: [{ id: 'recbbbbbbbbbbbbbbbb' }], + }, + }; + }); + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'Tasks', + fields: [ + { id: sourceTextFieldId, name: 'Title', type: 'singleLineText' }, + { id: sourceLinkFieldId, name: 'Owner', type: 'link' }, + ], + }, + ], + }, + records, + }, + withRecords: true, + batchSize: 500, + })._unsafeUnwrap(); + + vi.spyOn( + handler as unknown as { createTables: (...args: unknown[]) => Promise }, + 'createTables' + ).mockResolvedValue( + ok({ + result: { + tableIdMap: { [sourceTableId]: targetTableId }, + fieldIdMap: { + [sourceTextFieldId]: targetTextFieldId, + [sourceLinkFieldId]: targetLinkFieldId, + }, + viewIdMap: {}, + }, + tablesBySourceId: new Map([[sourceTableId, targetTable]]), + }) as never ); + + const result = await handler.handle({} as never, command); + for await (const event of result._unsafeUnwrap()) { + if (event.id === 'error') throw new Error(event.message); + } + + expect(operationOrder).toEqual(['insert', 'update']); + expect(records).toHaveBeenNthCalledWith(1, sourceTableId, { phase: 'insert' }); + expect(records).toHaveBeenNthCalledWith(2, sourceTableId, { phase: 'linkRestore' }); + expect(insertedFieldIds).toEqual([targetTextFieldId]); + expect(linkUpdateItems).toHaveLength(1); + expect(linkUpdateItems[0]?.fieldValues).toEqual( + new Map([[targetLinkFieldId, [{ id: 'recbbbbbbbbbbbbbbbb' }]]]) + ); + expect(tableRecordRepository.insertManyStream).toHaveBeenCalledWith( + expect.anything(), + targetTable, + expect.anything(), + expect.objectContaining({ skipComputedUpdates: true, skipChangedFields: true }) + ); + expect(tableRecordRepository.updateManyStream).toHaveBeenCalledWith( + expect.anything(), + targetTable, + expect.anything(), + expect.objectContaining({ + skipComputedUpdates: true, + fillLinkTitles: true, + assumeEmptyLinkState: true, + }) + ); + }); + + it('copies records with insertManyStream in command batches', async () => { + const tableRecordRepository = { + insertManyStream: vi.fn((_context, _table, batches: Array<{ records: unknown[] }>) => + Promise.resolve(ok({ totalInserted: batches[0]?.records.length ?? 0 })) + ), + }; + const unitOfWork = { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }; + const handler = createHandler({ + tableRecordRepository, + unitOfWork, + }); + const sourceFieldId = 'fldaaaaaaaaaaaaaaaa'; + const targetFieldId = 'fldbbbbbbbbbbbbbbbb'; + const sourceTableId = 'tblaaaaaaaaaaaaaaaa'; + const targetTableId = 'tblbbbbbbbbbbbbbbbb'; + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'People', + fields: [{ id: sourceFieldId, name: 'Name', type: 'singleLineText' }], + }, + ], + }, + records: async function* () { + yield { fields: { [sourceFieldId]: 'Alice' }, autoNumber: 1 }; + yield { fields: { [sourceFieldId]: 'Bob' }, autoNumber: 2 }; + yield { fields: { [sourceFieldId]: 'Cara' }, autoNumber: 3 }; + }, + }, + withRecords: true, + batchSize: 2, + })._unsafeUnwrap(); + + vi.spyOn( + handler as unknown as { createTables: (...args: unknown[]) => Promise }, + 'createTables' + ).mockResolvedValue( + ok({ + result: { + tableIdMap: { [sourceTableId]: targetTableId }, + fieldIdMap: { [sourceFieldId]: targetFieldId }, + viewIdMap: {}, + }, + tablesBySourceId: new Map([[sourceTableId, fakeTable(targetTableId, 'People')]]), + }) as never + ); + + const result = await handler.handle({} as never, command); + const events = []; + for await (const event of result._unsafeUnwrap()) { + events.push(event); + } + + expect(tableRecordRepository.insertManyStream).toHaveBeenCalledTimes(2); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'progress', + phase: 'table_data_progress', + tableId: targetTableId, + processedRows: 2, + batchProcessedRows: 2, + currentBatch: 1, + }), + expect.objectContaining({ + id: 'progress', + phase: 'table_data_progress', + tableId: targetTableId, + processedRows: 3, + batchProcessedRows: 1, + currentBatch: 2, + }), + expect.objectContaining({ + id: 'progress', + phase: 'table_data_done', + tableId: targetTableId, + processedRows: 3, + }), + expect.objectContaining({ + id: 'done', + recordsLength: 3, + }), + ]) + ); + }); + + it('scales duplicate record batches down for wide tables', async () => { + const observedBatchSizes: number[] = []; + const tableRecordRepository = { + insertManyStream: vi.fn((_context, _table, batches: Array<{ records: unknown[] }>) => { + const batchSize = batches[0]?.records.length ?? 0; + observedBatchSizes.push(batchSize); + return Promise.resolve(ok({ totalInserted: batchSize })); + }), + }; + const unitOfWork = { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }; + const handler = createHandler({ + tableRecordRepository, + unitOfWork, + }); + const sourceTableId = 'tblaaaaaaaaaaaaaaaa'; + const targetTableId = 'tblbbbbbbbbbbbbbbbb'; + const sourceFieldIds = Array.from( + { length: 250 }, + (_, index) => `fld${index.toString().padStart(16, 'a')}` + ); + const fieldIdMap = Object.fromEntries( + sourceFieldIds.map((sourceFieldId, index) => [ + sourceFieldId, + `fld${index.toString().padStart(16, 'b')}`, + ]) + ); + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'Wide table', + fields: sourceFieldIds.map((id, index) => ({ + id, + name: `Field ${index}`, + type: 'singleLineText', + })), + }, + ], + }, + records: async function* () { + for (let rowIndex = 0; rowIndex < 600; rowIndex++) { + yield { + fields: Object.fromEntries( + sourceFieldIds.map((fieldId, fieldIndex) => [ + fieldId, + `row-${rowIndex}-field-${fieldIndex}`, + ]) + ), + }; + } + }, + }, + withRecords: true, + batchSize: 500, + })._unsafeUnwrap(); + + vi.spyOn( + handler as unknown as { createTables: (...args: unknown[]) => Promise }, + 'createTables' + ).mockResolvedValue( + ok({ + result: { + tableIdMap: { [sourceTableId]: targetTableId }, + fieldIdMap, + viewIdMap: {}, + }, + tablesBySourceId: new Map([ + [sourceTableId, fakeTable(targetTableId, 'Wide table', sourceFieldIds)], + ]), + }) as never + ); + + const result = await handler.handle({} as never, command); + const events = []; + for await (const event of result._unsafeUnwrap()) { + events.push(event); + } + + expect(observedBatchSizes).toEqual([232, 232, 136]); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'progress', + phase: 'table_data_progress', + tableId: targetTableId, + processedRows: 232, + batchProcessedRows: 232, + currentBatch: 1, + }), + expect.objectContaining({ + id: 'progress', + phase: 'table_data_done', + tableId: targetTableId, + processedRows: 600, + }), + ]) + ); + }); + + it('streams table structure progress with table counts', async () => { + const handler = createHandler(); const command = DuplicateBaseCommand.createFromSource({ baseId, source: { @@ -79,14 +432,64 @@ describe('DuplicateBaseHandler', () => { ); }); - it('remaps aiConfig references when duplicating fields', async () => { - const handler = new DuplicateBaseHandler( - {} as never, - {} as never, - {} as never, - {} as never, - {} as never + it('creates duplicated table structures with duplicate-only schema options', async () => { + const tableCreationService = { + execute: vi.fn(async (_context: unknown, input: { tables: ReadonlyArray }) => + ok({ + persistedTables: input.tables, + tableState: new Map(), + sideEffectEvents: [], + }) + ), + }; + const unitOfWork = { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }; + const handler = createHandler({ + foreignTableLoaderService: { load: vi.fn(async () => ok([])) }, + tableCreationService, + eventBus: { publishMany: vi.fn(async () => ok(undefined)) }, + unitOfWork, + }); + const sourceTableId = 'tblSourceA'; + const sourceFieldId = 'fldaaaaaaaaaaaaaaaa'; + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'People', + fields: [{ id: sourceFieldId, name: 'Name', type: 'singleLineText' }], + }, + ], + }, + records: async function* () { + yield undefined as never; + }, + }, + withRecords: false, + })._unsafeUnwrap(); + + const result = await handler.handle({} as never, command); + for await (const event of result._unsafeUnwrap()) { + if (event.id === 'error') throw new Error(event.message); + } + + expect(tableCreationService.execute).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + schemaOptions: { optimizeForEmptyTables: true, skipUndoCaptureSetup: true }, + sideEffectOptions: { skipFieldCreationSideEffects: true }, + }) ); + }); + + it('remaps aiConfig references when duplicating fields', async () => { + const handler = createHandler(); const sourceBaseId = 'bseSource'; const sourcePrimaryFieldId = 'fldPrimary'; const sourceAiFieldId = 'fldAi'; @@ -156,4 +559,255 @@ describe('DuplicateBaseHandler', () => { sourceFieldId: fieldIdMap[sourcePrimaryFieldId], }); }); + + it('backfills computed fields after record and link restore complete', async () => { + const operationOrder: string[] = []; + const targetTable = fakeTable('tblbbbbbbbbbbbbbbbb', 'People', [{ id: 'fldComputed' }]); + const tableRecordRepository = { + insertManyStream: vi.fn(async () => { + operationOrder.push('insert'); + return ok({ totalInserted: 1 }); + }), + updateManyStream: vi.fn(async () => { + operationOrder.push('linkRestore'); + return ok({ totalUpdated: 0 }); + }), + }; + const computedFieldBackfillService = { + executeSyncMany: vi.fn(async () => { + operationOrder.push('backfill'); + return ok(undefined); + }), + }; + const handler = createHandler({ + tableRecordRepository, + unitOfWork: { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }, + computedFieldBackfillService, + }); + const sourceTableId = 'tblaaaaaaaaaaaaaaaa'; + const sourceFieldId = 'fldaaaaaaaaaaaaaaaa'; + const sourceLinkFieldId = 'fldcccccccccccccccc'; + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'People', + fields: [ + { id: sourceFieldId, name: 'Name', type: 'singleLineText' }, + { id: sourceLinkFieldId, name: 'Friend', type: 'link' }, + ], + }, + ], + }, + records: async function* () { + yield { fields: { [sourceFieldId]: 'Alice', [sourceLinkFieldId]: [] } }; + }, + }, + withRecords: true, + })._unsafeUnwrap(); + + vi.spyOn( + handler as unknown as { createTables: (...args: unknown[]) => Promise }, + 'createTables' + ).mockResolvedValue( + ok({ + result: { + tableIdMap: { [sourceTableId]: 'tblbbbbbbbbbbbbbbbb' }, + fieldIdMap: { + [sourceFieldId]: 'fldbbbbbbbbbbbbbbbb', + [sourceLinkFieldId]: 'flddddddddddddddddd', + }, + viewIdMap: {}, + }, + tablesBySourceId: new Map([[sourceTableId, targetTable]]), + }) as never + ); + + const result = await handler.handle({ requestId: 'ctx' } as never, command); + const events = []; + for await (const event of result._unsafeUnwrap()) { + events.push(event); + } + + expect(operationOrder).toEqual(['insert', 'linkRestore', 'backfill']); + expect(computedFieldBackfillService.executeSyncMany).toHaveBeenCalledWith( + expect.objectContaining({ requestId: 'ctx' }), + { + table: targetTable, + fields: targetTable.getFields(), + skipDistinctFilter: true, + includeOneManyTwoWay: true, + } + ); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'done', + recordsLength: 1, + }), + ]) + ); + }); + + it('returns an error event when duplicate computed backfill fails', async () => { + const failure = domainError.infrastructure({ + message: 'computed backfill failed', + code: 'computed.backfill_failed', + }); + const targetTable = fakeTable('tblbbbbbbbbbbbbbbbb', 'People', [{ id: 'fldComputed' }]); + const handler = createHandler({ + tableRecordRepository: { + insertManyStream: vi.fn(async () => ok({ totalInserted: 1 })), + updateManyStream: vi.fn(async () => ok({ totalUpdated: 0 })), + }, + unitOfWork: { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }, + computedFieldBackfillService: { + executeSyncMany: vi.fn(async () => err(failure)), + }, + }); + const sourceTableId = 'tblaaaaaaaaaaaaaaaa'; + const sourceFieldId = 'fldaaaaaaaaaaaaaaaa'; + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'People', + fields: [{ id: sourceFieldId, name: 'Name', type: 'singleLineText' }], + }, + ], + }, + records: async function* () { + yield { fields: { [sourceFieldId]: 'Alice' } }; + }, + }, + withRecords: true, + })._unsafeUnwrap(); + + vi.spyOn( + handler as unknown as { createTables: (...args: unknown[]) => Promise }, + 'createTables' + ).mockResolvedValue( + ok({ + result: { + tableIdMap: { [sourceTableId]: 'tblbbbbbbbbbbbbbbbb' }, + fieldIdMap: { [sourceFieldId]: 'fldbbbbbbbbbbbbbbbb' }, + viewIdMap: {}, + }, + tablesBySourceId: new Map([[sourceTableId, targetTable]]), + }) as never + ); + + const result = await handler.handle({} as never, command); + const events = []; + for await (const event of result._unsafeUnwrap()) { + events.push(event); + } + + expect(events).toContainEqual({ + id: 'error', + message: 'computed backfill failed', + code: 'computed.backfill_failed', + }); + expect(events).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'done' })])); + }); + + it('stops before computed backfill when link restore fails', async () => { + const failure = domainError.infrastructure({ + message: 'link restore failed', + code: 'duplicate_base.link_restore_failed', + }); + const targetTable = { + ...fakeTable('tblbbbbbbbbbbbbbbbb', 'People'), + updateRecordsStream: vi.fn(function* () { + yield ok([]); + }), + }; + const computedFieldBackfillService = { + executeSyncMany: vi.fn(async () => ok(undefined)), + }; + const handler = createHandler({ + tableRecordRepository: { + insertManyStream: vi.fn(async () => ok({ totalInserted: 1 })), + updateManyStream: vi.fn(async () => err(failure)), + }, + unitOfWork: { + withTransaction: vi.fn((_context, callback: (tx: unknown) => Promise) => + callback({ tx: true }) + ), + }, + computedFieldBackfillService, + }); + const sourceTableId = 'tblaaaaaaaaaaaaaaaa'; + const sourceFieldId = 'fldaaaaaaaaaaaaaaaa'; + const sourceLinkFieldId = 'fldcccccccccccccccc'; + const command = DuplicateBaseCommand.createFromSource({ + baseId, + source: { + structure: { + tables: [ + { + id: sourceTableId, + name: 'People', + fields: [ + { id: sourceFieldId, name: 'Name', type: 'singleLineText' }, + { id: sourceLinkFieldId, name: 'Friend', type: 'link' }, + ], + }, + ], + }, + records: async function* () { + yield { + recordId: 'recaaaaaaaaaaaaaaaa', + fields: { [sourceFieldId]: 'Alice', [sourceLinkFieldId]: [] }, + }; + }, + }, + withRecords: true, + })._unsafeUnwrap(); + + vi.spyOn( + handler as unknown as { createTables: (...args: unknown[]) => Promise }, + 'createTables' + ).mockResolvedValue( + ok({ + result: { + tableIdMap: { [sourceTableId]: 'tblbbbbbbbbbbbbbbbb' }, + fieldIdMap: { + [sourceFieldId]: 'fldbbbbbbbbbbbbbbbb', + [sourceLinkFieldId]: 'flddddddddddddddddd', + }, + viewIdMap: {}, + }, + tablesBySourceId: new Map([[sourceTableId, targetTable]]), + }) as never + ); + + const result = await handler.handle({} as never, command); + const events = []; + for await (const event of result._unsafeUnwrap()) { + events.push(event); + } + + expect(events).toContainEqual({ + id: 'error', + message: 'link restore failed', + code: 'duplicate_base.link_restore_failed', + }); + expect(computedFieldBackfillService.executeSyncMany).not.toHaveBeenCalled(); + expect(events).not.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'done' })])); + }); }); diff --git a/packages/v2/core/src/commands/DuplicateBaseHandler.ts b/packages/v2/core/src/commands/DuplicateBaseHandler.ts index 77f05b32f1..b91452f113 100644 --- a/packages/v2/core/src/commands/DuplicateBaseHandler.ts +++ b/packages/v2/core/src/commands/DuplicateBaseHandler.ts @@ -1,5 +1,5 @@ import { inject, injectable } from '@teable/v2-di'; -import { ok, safeTry } from 'neverthrow'; +import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; import { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; @@ -9,18 +9,21 @@ import { domainError, isDomainError, type DomainError } from '../domain/shared/D import { FieldId } from '../domain/table/fields/FieldId'; import { validateForeignTablesForFields } from '../domain/table/fields/ForeignTableRelatedField'; import type { LinkForeignTableReference } from '../domain/table/fields/visitors/LinkForeignTableReferenceVisitor'; +import { calculateBatchSize } from '../domain/table/methods/records/calculateBatchSize'; import { RecordId } from '../domain/table/records/RecordId'; import { TableRecord } from '../domain/table/records/TableRecord'; import { TableRecordCellValue } from '../domain/table/records/TableRecordFields'; import type { Table } from '../domain/table/Table'; import { TableId } from '../domain/table/TableId'; import { ViewId } from '../domain/table/views/ViewId'; +import type { IComputedFieldBackfillService } from '../ports/ComputedFieldBackfillService'; import type { NormalizedDotTeaStructure } from '../ports/DotTeaParser'; import * as EventBusPort from '../ports/EventBus'; import type { IExecutionContext } from '../ports/ExecutionContext'; import type { InsertManyStreamBatch, RecordRestoreSystemValues, + UpdateManyStreamBatchInput, } from '../ports/TableRecordRepository'; import * as TableRecordRepositoryPort from '../ports/TableRecordRepository'; import { v2CoreTokens } from '../ports/tokens'; @@ -93,7 +96,9 @@ export class DuplicateBaseHandler @inject(v2CoreTokens.eventBus) private readonly eventBus: EventBusPort.IEventBus, @inject(v2CoreTokens.unitOfWork) - private readonly unitOfWork: UnitOfWorkPort.IUnitOfWork + private readonly unitOfWork: UnitOfWorkPort.IUnitOfWork, + @inject(v2CoreTokens.computedFieldBackfillService) + private readonly computedFieldBackfillService: IComputedFieldBackfillService ) {} async handle( @@ -148,11 +153,35 @@ export class DuplicateBaseHandler viewIdMap: result.viewIdMap, })) { yield event; + if (event.id === 'error') return; if (event.id === 'progress' && event.phase === 'table_data_done') { recordsLength += event.processedRows ?? 0; } } } + + for (const table of command.source.structure.tables) { + const targetTable = tablesBySourceId.get(table.id ?? ''); + if (!targetTable || !table.id) continue; + + for await (const event of this.restoreLinkFieldsStream(context, command, { + sourceTableId: table.id, + targetTable, + fieldIdMap: result.fieldIdMap, + })) { + yield event; + if (event.id === 'error') return; + } + } + + const backfillResult = await this.backfillComputedFields( + context, + Array.from(tablesBySourceId.values()) + ); + if (backfillResult.isErr()) { + yield this.errorEvent(backfillResult.error); + return; + } } yield { @@ -254,6 +283,13 @@ export class DuplicateBaseHandler tables: builtTables, externalTables, referencesByTable, + schemaOptions: { + optimizeForEmptyTables: true, + skipUndoCaptureSetup: true, + }, + sideEffectOptions: { + skipFieldCreationSideEffects: true, + }, }) ); @@ -279,6 +315,26 @@ export class DuplicateBaseHandler }); } + private async backfillComputedFields( + context: IExecutionContext, + targetTables: ReadonlyArray
+ ): Promise> { + return safeTry( + async function* (this: DuplicateBaseHandler) { + for (const table of targetTables) { + yield* await this.computedFieldBackfillService.executeSyncMany(context, { + table, + fields: table.getFields(), + skipDistinctFilter: true, + includeOneManyTwoWay: true, + }); + } + + return ok(undefined); + }.bind(this) + ); + } + private async remapStructure(baseId: BaseId, normalized: NormalizedDotTeaStructure) { return safeTry< { @@ -358,6 +414,7 @@ export class DuplicateBaseHandler ): AsyncGenerator { let totalInserted = 0; let batchIndex = 0; + const batchSize = this.resolveTableRecordBatchSize(params.targetTable, command.batchSize); yield { id: 'progress', @@ -369,10 +426,11 @@ export class DuplicateBaseHandler for await (const batch of this.buildInsertBatches( params.targetTable, - command.source.records(params.sourceTableId), + command.source.records(params.sourceTableId, { phase: 'insert' }), params.fieldIdMap, params.viewIdMap, - command.batchSize + batchSize, + this.getSourceLinkFieldIds(command, params.sourceTableId) )) { const currentBatchIndex = batchIndex; const result = await this.unitOfWork.withTransaction(context, async (transactionContext) => @@ -381,8 +439,8 @@ export class DuplicateBaseHandler params.targetTable, [batch], { - deferComputedUpdates: true, - enqueueDeferredComputedUpdates: true, + skipComputedUpdates: true, + skipChangedFields: true, } ) ); @@ -413,23 +471,108 @@ export class DuplicateBaseHandler }; } + private async *restoreLinkFieldsStream( + context: IExecutionContext, + command: DuplicateBaseCommand, + params: { + sourceTableId: string; + targetTable: Table; + fieldIdMap: Record; + } + ): AsyncGenerator { + const sourceLinkFieldIds = this.getSourceLinkFieldIds(command, params.sourceTableId); + if (!sourceLinkFieldIds.size) { + return; + } + + const result = await this.unitOfWork.withTransaction(context, async (transactionContext) => + this.tableRecordRepository.updateManyStream( + transactionContext, + params.targetTable, + this.buildLinkUpdateBatches( + params.targetTable, + command.source.records(params.sourceTableId, { phase: 'linkRestore' }), + params.fieldIdMap, + sourceLinkFieldIds, + this.resolveTableRecordBatchSize(params.targetTable, command.batchSize) + ), + { + skipComputedUpdates: true, + fillLinkTitles: true, + assumeEmptyLinkState: true, + } + ) + ); + if (result.isErr()) { + yield this.errorEvent(result.error); + } + } + + private resolveTableRecordBatchSize(table: Table, requestedBatchSize: number): number { + return Math.min(requestedBatchSize, calculateBatchSize(table.getFields().length)); + } + private async *buildInsertBatches( table: Table, records: AsyncIterable, fieldIdMap: Record, viewIdMap: Record, - batchSize: number + batchSize: number, + excludedSourceFieldIds: ReadonlySet = new Set() ): AsyncGenerator { let batch: DuplicateBaseRecordInput[] = []; for await (const record of records) { batch.push(record); if (batch.length >= batchSize) { - yield this.toInsertBatch(table, batch, fieldIdMap, viewIdMap); + yield this.toInsertBatch(table, batch, fieldIdMap, viewIdMap, excludedSourceFieldIds); batch = []; } } if (batch.length > 0) { - yield this.toInsertBatch(table, batch, fieldIdMap, viewIdMap); + yield this.toInsertBatch(table, batch, fieldIdMap, viewIdMap, excludedSourceFieldIds); + } + } + + private async *buildLinkUpdateBatches( + table: Table, + records: AsyncIterable, + fieldIdMap: Record, + sourceLinkFieldIds: ReadonlySet, + batchSize: number + ): AsyncGenerator> { + let batch: Array<{ recordId: RecordId; fieldValues: Map }> = []; + for await (const record of records) { + const fieldValues = new Map(); + for (const [sourceFieldId, rawValue] of Object.entries(record.fields)) { + if (!sourceLinkFieldIds.has(sourceFieldId)) continue; + const targetFieldId = fieldIdMap[sourceFieldId]; + if (targetFieldId) { + fieldValues.set(targetFieldId, rawValue); + } + } + if (!fieldValues.size) continue; + if (!record.recordId) { + yield err( + domainError.validation({ + code: 'duplicate_base.record_id_required', + message: 'Duplicate base link restore requires record ids in source records', + }) + ); + return; + } + const recordId = RecordId.create(record.recordId); + if (recordId.isErr()) { + yield err(recordId.error); + return; + } + batch.push({ recordId: recordId.value, fieldValues }); + if (batch.length >= batchSize) { + yield* table.updateRecordsStream(batch, { typecast: false, batchSize }); + batch = []; + } + } + if (batch.length > 0) { + yield* table.updateRecordsStream(batch, { typecast: false, batchSize }); } } @@ -437,9 +580,12 @@ export class DuplicateBaseHandler table: Table, batch: ReadonlyArray, fieldIdMap: Record, - viewIdMap: Record + viewIdMap: Record, + excludedSourceFieldIds: ReadonlySet = new Set() ): InsertManyStreamBatch { - const records = batch.map((record) => this.toTableRecord(table, record, fieldIdMap)); + const records = batch.map((record) => + this.toTableRecord(table, record, fieldIdMap, excludedSourceFieldIds) + ); return { records, restoreRecordsById: new Map( @@ -454,11 +600,13 @@ export class DuplicateBaseHandler private toTableRecord( table: Table, record: DuplicateBaseRecordInput, - fieldIdMap: Record + fieldIdMap: Record, + excludedSourceFieldIds: ReadonlySet = new Set() ): TableRecord { const recordIdResult = record.recordId ? RecordId.create(record.recordId) : RecordId.generate(); if (recordIdResult.isErr()) throw recordIdResult.error; const fieldValues = Object.entries(record.fields).flatMap(([sourceFieldId, rawValue]) => { + if (excludedSourceFieldIds.has(sourceFieldId)) return []; const targetFieldId = fieldIdMap[sourceFieldId]; if (!targetFieldId) return []; const fieldId = FieldId.create(targetFieldId); @@ -476,6 +624,18 @@ export class DuplicateBaseHandler return tableRecord.value; } + private getSourceLinkFieldIds( + command: DuplicateBaseCommand, + sourceTableId: string + ): ReadonlySet { + const sourceTable = command.source.structure.tables.find((table) => table.id === sourceTableId); + return new Set( + sourceTable?.fields.flatMap((field) => + field.id && field.type === 'link' ? [field.id] : [] + ) ?? [] + ); + } + private toRestoreValues( record: DuplicateBaseRecordInput, viewIdMap: Record @@ -495,8 +655,8 @@ export class DuplicateBaseHandler ...(record.autoNumber !== undefined ? { autoNumber: record.autoNumber } : {}), ...(record.createdTime ? { createdTime: record.createdTime } : {}), ...(record.createdBy ? { createdBy: record.createdBy } : {}), - ...(record.lastModifiedTime ? { lastModifiedTime: record.lastModifiedTime } : {}), - ...(record.lastModifiedBy ? { lastModifiedBy: record.lastModifiedBy } : {}), + lastModifiedTime: record.lastModifiedTime ?? null, + lastModifiedBy: record.lastModifiedBy ?? null, }; } diff --git a/packages/v2/core/src/commands/DuplicateTableHandler.ts b/packages/v2/core/src/commands/DuplicateTableHandler.ts index c97204a36c..f8df880166 100644 --- a/packages/v2/core/src/commands/DuplicateTableHandler.ts +++ b/packages/v2/core/src/commands/DuplicateTableHandler.ts @@ -155,42 +155,55 @@ export class DuplicateTableHandler { scope: 'meta' } ); - const duplicateResult = await handler.unitOfWork.withTransaction( - context, - async (dataTransactionContext) => - safeTry(async function* () { - yield* await handler.tableSchemaRepository.insert( - dataTransactionContext, - persistedTable - ); + const previousDuplicateContext = context.duplicateTable; + context.duplicateTable = { + sourceTableId: command.tableId.toString(), + duplicatedTableId: persistedTable.id().toString(), + includeRecords: command.includeRecords, + }; - if (records.length > 0) { - const pluginExecution = yield* await handler.recordWritePluginRunner.prepare({ - kind: RecordWriteOperationKind.createMany, - executionContext: dataTransactionContext, - table: persistedTable, - payload: { - recordsFieldValues: records.map(tableRecordToRecordWriteFieldValues), - fieldKeyType: FieldKeyType.Id, - typecast: false, - recordCount: records.length, - }, - isTransactionBound: true, - }); - yield* await pluginExecution.guard(); - yield* await pluginExecution.beforePersist(dataTransactionContext); - yield* await handler.tableRecordRepository.insertMany( - dataTransactionContext, - persistedTable, - records, - restoreRecordsById ? { restoreRecordsById } : undefined - ); - } - - return ok(undefined); - }), - { scope: 'data' } - ); + const duplicateResult = await (async () => { + try { + return await handler.unitOfWork.withTransaction( + context, + async (dataTransactionContext) => + safeTry(async function* () { + yield* await handler.tableSchemaRepository.insert( + dataTransactionContext, + persistedTable + ); + + if (records.length > 0) { + const pluginExecution = yield* await handler.recordWritePluginRunner.prepare({ + kind: RecordWriteOperationKind.createMany, + executionContext: dataTransactionContext, + table: persistedTable, + payload: { + recordsFieldValues: records.map(tableRecordToRecordWriteFieldValues), + fieldKeyType: FieldKeyType.Id, + typecast: false, + recordCount: records.length, + }, + isTransactionBound: true, + }); + yield* await pluginExecution.guard(); + yield* await pluginExecution.beforePersist(dataTransactionContext); + yield* await handler.tableRecordRepository.insertMany( + dataTransactionContext, + persistedTable, + records, + restoreRecordsById ? { restoreRecordsById } : undefined + ); + } + + return ok(undefined); + }), + { scope: 'data' } + ); + } finally { + context.duplicateTable = previousDuplicateContext; + } + })(); if (duplicateResult.isErr()) { yield* await failTableSchemaOperation( handler.unitOfWork, @@ -219,7 +232,6 @@ export class DuplicateTableHandler restoreRecordsById ); - const previousDuplicateContext = context.duplicateTable; context.duplicateTable = { sourceTableId: command.tableId.toString(), duplicatedTableId: persistedTable.id().toString(), diff --git a/packages/v2/core/src/commands/ImportCsvCommand.spec.ts b/packages/v2/core/src/commands/ImportCsvCommand.spec.ts index 31339f1c0f..064f35a6f4 100644 --- a/packages/v2/core/src/commands/ImportCsvCommand.spec.ts +++ b/packages/v2/core/src/commands/ImportCsvCommand.spec.ts @@ -10,25 +10,36 @@ describe('ImportCsvCommand', () => { baseId, csvData: 'Name,Age\nAlice,30', tableName: 'People', + columns: [ + { name: 'Display Name', sourceColumnIndex: 0, type: 'singleLineText' }, + { name: 'Years', sourceColumnIndex: 1, type: 'number' }, + ], }); expect(result.isOk()).toBe(true); const command = result._unsafeUnwrap(); expect(command.csvSource.type).toBe('string'); expect(command.tableName?.toString()).toBe('People'); + expect(command.importData).toBe(true); expect(command.batchSize).toBe(500); + expect(command.columns).toEqual([ + { name: 'Display Name', sourceColumnIndex: 0, type: 'singleLineText' }, + { name: 'Years', sourceColumnIndex: 1, type: 'number' }, + ]); }); it('creates from csvUrl input', () => { const result = ImportCsvCommand.create({ baseId, csvUrl: 'https://example.com/data.csv', + importData: false, batchSize: 1000, }); expect(result.isOk()).toBe(true); const command = result._unsafeUnwrap(); expect(command.csvSource.type).toBe('url'); + expect(command.importData).toBe(false); expect(command.batchSize).toBe(1000); }); @@ -57,11 +68,13 @@ describe('ImportCsvCommand', () => { baseId, csvData: new Uint8Array([65, 44, 66]), tableName: 'Buffer Table', + importData: false, }); expect(bufferResult.isOk()).toBe(true); expect(bufferResult._unsafeUnwrap().csvSource.type).toBe('buffer'); expect(bufferResult._unsafeUnwrap().tableName?.toString()).toBe('Buffer Table'); + expect(bufferResult._unsafeUnwrap().importData).toBe(false); expect(bufferResult._unsafeUnwrap().batchSize).toBe(500); const streamResult = ImportCsvCommand.createFromStream({ diff --git a/packages/v2/core/src/commands/ImportCsvCommand.ts b/packages/v2/core/src/commands/ImportCsvCommand.ts index 9d7a7b6184..1c2cb8627f 100644 --- a/packages/v2/core/src/commands/ImportCsvCommand.ts +++ b/packages/v2/core/src/commands/ImportCsvCommand.ts @@ -13,9 +13,26 @@ import type { CsvSource } from '../ports/CsvParser'; const importCsvBaseSchema = z.object({ baseId: z.string(), tableName: z.string().optional(), + importData: z.boolean().default(true), batchSize: z.number().min(1).max(5000).default(500), + maxRowCount: z.number().int().positive().optional(), + columns: z + .array( + z.object({ + name: z.string(), + sourceColumnIndex: z.number().int().nonnegative(), + type: z.string().optional(), + }) + ) + .optional(), }); +export type ImportCsvColumn = { + readonly name: string; + readonly sourceColumnIndex: number; + readonly type?: string; +}; + /** * CSV 导入配置 Schema（用于 HTTP API） * 支持两种导入方式： @@ -47,7 +64,10 @@ export class ImportCsvCommand { readonly baseId: BaseId, readonly csvSource: CsvSource, readonly tableName: TableName | undefined, - readonly batchSize: number + readonly importData: boolean, + readonly batchSize: number, + readonly maxRowCount: number | undefined, + readonly columns: ReadonlyArray | undefined ) {} /** @@ -65,7 +85,7 @@ export class ImportCsvCommand { ); } - const { baseId, tableName, batchSize } = parsed.data; + const { baseId, tableName, importData, batchSize, maxRowCount, columns } = parsed.data; // 根据输入类型选择创建方式 if ('csvUrl' in parsed.data && parsed.data.csvUrl) { @@ -73,7 +93,10 @@ export class ImportCsvCommand { baseId, csvUrl: parsed.data.csvUrl, tableName, + importData, batchSize, + maxRowCount, + columns, }); } @@ -82,7 +105,10 @@ export class ImportCsvCommand { baseId, csvData: parsed.data.csvData, tableName, + importData, batchSize, + maxRowCount, + columns, }); } @@ -101,7 +127,10 @@ export class ImportCsvCommand { baseId: string; csvData: Uint8Array; tableName?: string; + importData?: boolean; batchSize?: number; + maxRowCount?: number; + columns?: ReadonlyArray; }): Result { const baseIdResult = BaseId.create(input.baseId); if (baseIdResult.isErr()) { @@ -131,7 +160,10 @@ export class ImportCsvCommand { baseIdResult.value, { type: 'buffer', data: input.csvData }, tableNameVo, - batchSize + input.importData ?? true, + batchSize, + input.maxRowCount, + input.columns ) ); } @@ -143,7 +175,10 @@ export class ImportCsvCommand { baseId: string; csvData: string; tableName?: string; + importData?: boolean; batchSize?: number; + maxRowCount?: number; + columns?: ReadonlyArray; }): Result { const baseIdResult = BaseId.create(input.baseId); if (baseIdResult.isErr()) { @@ -173,7 +208,10 @@ export class ImportCsvCommand { baseIdResult.value, { type: 'string', data: input.csvData }, tableNameVo, - batchSize + input.importData ?? true, + batchSize, + input.maxRowCount, + input.columns ) ); } @@ -185,7 +223,10 @@ export class ImportCsvCommand { baseId: string; csvStream: AsyncIterable; tableName?: string; + importData?: boolean; batchSize?: number; + maxRowCount?: number; + columns?: ReadonlyArray; }): Result { const baseIdResult = BaseId.create(input.baseId); if (baseIdResult.isErr()) { @@ -215,7 +256,10 @@ export class ImportCsvCommand { baseIdResult.value, { type: 'stream', data: input.csvStream }, tableNameVo, - batchSize + input.importData ?? true, + batchSize, + input.maxRowCount, + input.columns ) ); } @@ -228,7 +272,10 @@ export class ImportCsvCommand { baseId: string; csvUrl: string; tableName?: string; + importData?: boolean; batchSize?: number; + maxRowCount?: number; + columns?: ReadonlyArray; }): Result { const baseIdResult = BaseId.create(input.baseId); if (baseIdResult.isErr()) { @@ -270,7 +317,10 @@ export class ImportCsvCommand { baseIdResult.value, { type: 'url', url: input.csvUrl }, tableNameVo, - batchSize + input.importData ?? true, + batchSize, + input.maxRowCount, + input.columns ) ); } diff --git a/packages/v2/core/src/commands/ImportCsvHandler.spec.ts b/packages/v2/core/src/commands/ImportCsvHandler.spec.ts index d1f297ae7d..80f825de5b 100644 --- a/packages/v2/core/src/commands/ImportCsvHandler.spec.ts +++ b/packages/v2/core/src/commands/ImportCsvHandler.spec.ts @@ -8,6 +8,7 @@ import { ActorId } from '../domain/shared/ActorId'; import type { DomainError } from '../domain/shared/DomainError'; import type { IDomainEvent } from '../domain/shared/DomainEvent'; import type { ISpecification } from '../domain/shared/specification/ISpecification'; +import { isRecordsBatchCreatedEvent } from '../domain/table/events/RecordsBatchCreated'; import type { RecordId } from '../domain/table/records/RecordId'; import type { RecordUpdateResult } from '../domain/table/records/RecordUpdateResult'; import type { ITableRecordConditionSpecVisitor } from '../domain/table/records/specs/ITableRecordConditionSpecVisitor'; @@ -288,10 +289,10 @@ describe('ImportCsvHandler', () => { it('imports csv data and creates table/records', async () => { const parser = new FakeCsvParser( ok({ - headers: ['Name', 'Age'], + headers: ['Name', 'Age', 'Note'], rows: [ - { Name: 'Alice', Age: '30' }, - { Name: 'Bob', Age: '40' }, + { Name: 'Alice', Age: '30', Note: 'hello' }, + { Name: 'Bob', Age: '40', Note: '' }, ], }) ); @@ -314,7 +315,7 @@ describe('ImportCsvHandler', () => { const command = ImportCsvCommand.createFromString({ baseId, - csvData: 'Name,Age\nAlice,30\nBob,40', + csvData: 'Name,Age,Note\nAlice,30,hello\nBob,40,', tableName: 'People', batchSize: 2, })._unsafeUnwrap(); @@ -325,9 +326,27 @@ describe('ImportCsvHandler', () => { const value = result._unsafeUnwrap(); expect(value.totalImported).toBe(2); expect(value.table.name().toString()).toBe('People'); - expect(value.table.getFields()).toHaveLength(2); + expect(value.table.getFields()).toHaveLength(3); + expect(value.table.getFields().map((field) => field.type().toString())).toEqual([ + 'singleLineText', + 'number', + 'singleLineText', + ]); expect(value.table.primaryField()._unsafeUnwrap().name().toString()).toBe('Name'); expect(tableRecordRepository.inserted).toHaveLength(2); + const noteFieldId = value.table.getFields()[2].id().toString(); + const insertedFieldValues = tableRecordRepository.inserted.map( + (record) => + new Map( + record + .fields() + .entries() + .map((entry) => [entry.fieldId.toString(), entry.value]) + ) + ); + expect(insertedFieldValues[0].get(noteFieldId)?.toValue()).toBe('hello'); + expect(insertedFieldValues[1].has(noteFieldId)).toBe(false); + expect(eventBus.published.some(isRecordsBatchCreatedEvent)).toBe(true); expect(eventBus.published.length).toBeGreaterThan(0); expect(tableRepository.provisionStateChanges.map(({ state }) => state)).toEqual([ 'pending', @@ -335,6 +354,128 @@ describe('ImportCsvHandler', () => { ]); }); + it('creates table schema only when importData is false', async () => { + const parser = new FakeCsvParser( + ok({ + headers: ['Name', 'Age'], + rows: [ + { Name: 'Alice', Age: '30' }, + { Name: 'Bob', Age: '40' }, + ], + }) + ); + const tableRepository = new FakeTableRepository(); + const tableSchemaRepository = new FakeTableSchemaRepository(); + const tableRecordRepository = new FakeTableRecordRepository(); + const eventBus = new FakeEventBus(); + const unitOfWork = new FakeUnitOfWork(); + + const handler = new ImportCsvHandler( + parser, + tableRepository, + tableSchemaRepository, + tableRecordRepository, + eventBus, + unitOfWork, + undefined, + createTableLimitPluginRunner(tableRepository) + ); + + const command = ImportCsvCommand.createFromString({ + baseId, + csvData: 'Name,Age\nAlice,30\nBob,40', + tableName: 'People Schema Only', + importData: false, + maxRowCount: 1, + })._unsafeUnwrap(); + + const result = await handler.handle(createContext(), command); + expect(result.isOk()).toBe(true); + + const value = result._unsafeUnwrap(); + expect(value.totalImported).toBe(0); + expect(value.table.name().toString()).toBe('People Schema Only'); + expect(value.table.getFields().map((field) => field.type().toString())).toEqual([ + 'singleLineText', + 'number', + ]); + expect(tableSchemaRepository.inserted).toHaveLength(1); + expect(tableRecordRepository.inserted).toHaveLength(0); + expect(eventBus.published.some(isRecordsBatchCreatedEvent)).toBe(false); + expect(tableRepository.provisionStateChanges.map(({ state }) => state)).toEqual([ + 'pending', + 'ready', + ]); + }); + + it('uses provided columns for field names and source column mapping', async () => { + const parser = new FakeCsvParser( + ok({ + headers: ['registration_date', 'total_registered_users', 'active_d1_users'], + rows: [ + { + registration_date: '2024-01-01', + total_registered_users: '100', + active_d1_users: '75', + }, + ], + }) + ); + const tableRepository = new FakeTableRepository(); + const tableRecordRepository = new FakeTableRecordRepository(); + const handler = new ImportCsvHandler( + parser, + tableRepository, + new FakeTableSchemaRepository(), + tableRecordRepository, + new FakeEventBus(), + new FakeUnitOfWork(), + undefined, + createTableLimitPluginRunner(tableRepository) + ); + + const command = ImportCsvCommand.createFromString({ + baseId, + csvData: 'registration_date,total_registered_users,active_d1_users\n2024-01-01,100,75', + tableName: 'Custom Mapping', + columns: [ + { name: '总注册用户', sourceColumnIndex: 1, type: 'number' }, + { name: '次日活跃用户', sourceColumnIndex: 2, type: 'number' }, + { name: '注册日期', sourceColumnIndex: 0, type: 'date' }, + ], + })._unsafeUnwrap(); + + const result = await handler.handle(createContext(), command); + expect(result.isOk()).toBe(true); + + const table = result._unsafeUnwrap().table; + expect(table.getFields().map((field) => field.name().toString())).toEqual([ + '总注册用户', + '次日活跃用户', + '注册日期', + ]); + expect(table.getFields().map((field) => field.type().toString())).toEqual([ + 'number', + 'number', + 'date', + ]); + + const insertedRecord = tableRecordRepository.inserted[0]; + const fieldsByName = new Map( + table.getFields().map((field) => [field.name().toString(), field.id().toString()]) + ); + const valuesByFieldId = new Map( + insertedRecord + .fields() + .entries() + .map((entry) => [entry.fieldId.toString(), entry.value.toValue()]) + ); + + expect(valuesByFieldId.get(fieldsByName.get('总注册用户')!)).toBe(100); + expect(valuesByFieldId.get(fieldsByName.get('次日活跃用户')!)).toBe(75); + expect(valuesByFieldId.get(fieldsByName.get('注册日期')!)).toBeDefined(); + }); + it('returns error when csv has no headers', async () => { const parser = new FakeCsvParser( ok({ diff --git a/packages/v2/core/src/commands/ImportCsvHandler.ts b/packages/v2/core/src/commands/ImportCsvHandler.ts index eb8f5ba4e8..e974fff3e1 100644 --- a/packages/v2/core/src/commands/ImportCsvHandler.ts +++ b/packages/v2/core/src/commands/ImportCsvHandler.ts @@ -2,20 +2,19 @@ import { inject, injectable } from '@teable/v2-di'; import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; +import { RecordWritePluginRunner } from '../application/services/RecordWritePluginRunner'; +import type { RecordWritePluginExecution } from '../application/services/RecordWritePluginRunner'; +import { TableOperationPluginRunner } from '../application/services/TableOperationPluginRunner'; import { beginTableSchemaOperation, completeTableSchemaOperation, failTableSchemaOperation, } from '../application/services/TableSchemaOperationLifecycleService'; -import { - RecordWritePluginExecution, - RecordWritePluginRunner, -} from '../application/services/RecordWritePluginRunner'; -import { TableOperationPluginRunner } from '../application/services/TableOperationPluginRunner'; import type { DomainError } from '../domain/shared/DomainError'; import { domainError } from '../domain/shared/DomainError'; import type { IDomainEvent } from '../domain/shared/DomainEvent'; -import { FieldKeyType } from '../domain/table/fields/FieldKeyType'; +import type { RecordValuesDTO } from '../domain/table/events/RecordFieldValuesDTO'; +import { RecordsBatchCreated } from '../domain/table/events/RecordsBatchCreated'; import { FieldName } from '../domain/table/fields/FieldName'; import type { TableRecord } from '../domain/table/records/TableRecord'; import { Table } from '../domain/table/Table'; @@ -34,15 +33,44 @@ import { v2CoreTokens } from '../ports/tokens'; import { TraceSpan } from '../ports/TraceSpan'; import * as UnitOfWorkPort from '../ports/UnitOfWork'; import { CommandHandler, type ICommandHandler } from './CommandHandler'; -import { ImportCsvCommand } from './ImportCsvCommand'; +import { ImportCsvCommand, type ImportCsvColumn } from './ImportCsvCommand'; type ChunkPluginOptions = { readonly table: Table; readonly batchSize: number; readonly operationId: string; readonly totalRecordCount: number; + readonly events: IDomainEvent[]; +}; + +type InferredCsvFieldType = 'checkbox' | 'number' | 'date' | 'longText' | 'singleLineText'; + +type ResolvedImportColumn = { + readonly name: string; + readonly sourceColumnIndex: number; + readonly type: InferredCsvFieldType; }; +const csvInferenceSampleSize = 500; +const inferredCsvFieldTypeOrder: InferredCsvFieldType[] = [ + 'checkbox', + 'number', + 'date', + 'longText', + 'singleLineText', +]; +const dateFormatPatterns: RegExp[] = [ + /^\d{4}-\d{2}-\d{2}$/, + /^\d{4}-\d{2}-\d{2}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?$/, + /^\d{4}-\d{2}-\d{2}T\d{1,2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?(?:Z|[+-]\d{2}:?\d{2})?$/, + /^\d{1,2}-\d{1,2}-\d{4}$/, + /^\d{4}\/\d{1,2}\/\d{1,2}$/, + /^\d{1,2}\/\d{1,2}\/\d{4}$/, + /^\d{1,2}\/\d{1,2}\/\d{4}\s+\d{1,2}:\d{2}(?::\d{2})?$/, +]; +const reasonableYearMin = 1; +const reasonableYearMax = 9999; + const tableRecordToRecordWriteFieldValues = (record: TableRecord): RecordWriteFieldValues => new Map( record @@ -116,7 +144,12 @@ export class ImportCsvHandler implements ICommandHandler(async function* () { // 1. 解析 CSV（根据数据源类型选择同步或异步） const parseResult = yield* await handler.parseCsvSource(command.csvSource); + const asyncRows = parseResult.rowsAsync + ? await handler.sampleAsyncRows(parseResult.rowsAsync, csvInferenceSampleSize) + : undefined; const rows = parseResult.rowsAsync ? undefined : [...parseResult.rows]; + const inferenceRows = asyncRows?.sampleRows ?? rows?.slice(0, csvInferenceSampleSize) ?? []; + const rowsAsync = asyncRows?.rowsAsync; if (parseResult.headers.length === 0) { return err( @@ -134,12 +167,13 @@ export class ImportCsvHandler implements ICommandHandler { - return safeTry<{ totalImported: number }, DomainError>(async function* () { - yield* await handler.tableSchemaRepository.insert( - dataTransactionContext, - persistedTable - ); - const totalRecordCount = rows?.length ?? 0; - const operationId = `import-csv:${persistedTable.id().toString()}`; - const pluginExecution = yield* await handler.recordWritePluginRunner.prepare({ - kind: RecordWriteOperationKind.createStream, - executionContext: dataTransactionContext, - table: persistedTable, - payload: { - recordsFieldValues: [], - batchSize: command.batchSize, - recordCount: totalRecordCount, - }, - orchestration: { - mode: 'stream', - scope: 'operation', - operationId, - totalRecordCount, - }, - isTransactionBound: true, - }); - yield* await pluginExecution.guard(); - - const fieldIdMap = handler.buildFieldIdMap(persistedTable, parseResult.headers); - - const recordsIterable = parseResult.rowsAsync - ? handler.createRecordsIterableAsync(parseResult.rowsAsync, fieldIdMap) - : handler.createRecordsIterable(rows ?? [], fieldIdMap); - - const batchGenerator = parseResult.rowsAsync - ? persistedTable.createRecordsStreamAsync( - recordsIterable as AsyncIterable>, - { - batchSize: command.batchSize, - } - ) - : persistedTable.createRecordsStream( - recordsIterable as Iterable>, - { - batchSize: command.batchSize, - } + return safeTry<{ totalImported: number; events: IDomainEvent[] }, DomainError>( + async function* () { + yield* await handler.tableSchemaRepository.insert( + dataTransactionContext, + persistedTable + ); + if (!command.importData) { + return ok({ totalImported: 0, events: [] }); + } + + const totalRecordCount = rows?.length ?? 0; + const operationId = `import-csv:${persistedTable.id().toString()}`; + const pluginExecution = yield* await handler.recordWritePluginRunner.prepare({ + kind: RecordWriteOperationKind.createStream, + executionContext: dataTransactionContext, + table: persistedTable, + payload: { + recordsFieldValues: [], + batchSize: command.batchSize, + recordCount: totalRecordCount, + }, + orchestration: { + mode: 'stream', + scope: 'operation', + operationId, + totalRecordCount, + }, + isTransactionBound: true, + }); + yield* await pluginExecution.guard(); + + if (command.maxRowCount !== undefined && rows && rows.length > command.maxRowCount) { + return err( + domainError.validation({ + code: 'validation.limit.rows_per_table_max', + message: `Exceed max row limit: ${command.maxRowCount}`, + details: { + max: command.maxRowCount, + maxRowCount: command.maxRowCount, + rowCount: rows.length, + }, + }) ); + } - const insertResult = yield* await handler.tableRecordRepository.insertManyStream( - dataTransactionContext, - persistedTable, - parseResult.rowsAsync - ? handler.consumeBatchesAsync( - batchGenerator as AsyncGenerator< - Result, DomainError> - >, - pluginExecution, - dataTransactionContext, + const fieldIdMap = handler.buildFieldIdMap( + persistedTable, + parseResult.headers, + importColumns + ); + const recordEvents: IDomainEvent[] = []; + + const recordsIterable = rowsAsync + ? handler.createRecordsIterableAsync(rowsAsync, fieldIdMap, command.maxRowCount) + : handler.createRecordsIterable(rows ?? [], fieldIdMap); + + const batchGenerator = rowsAsync + ? persistedTable.createRecordsStreamAsync( + recordsIterable as AsyncIterable>, { - table: persistedTable, batchSize: command.batchSize, - operationId, - totalRecordCount, + typecast: true, } ) - : handler.consumeBatches( - batchGenerator as Generator, DomainError>>, - pluginExecution, - dataTransactionContext, + : persistedTable.createRecordsStream( + recordsIterable as Iterable>, { - table: persistedTable, batchSize: command.batchSize, - operationId, - totalRecordCount, + typecast: true, } - ) - ); - - return ok({ totalImported: insertResult.totalInserted }); - }); + ); + + const insertResult = yield* await handler.tableRecordRepository.insertManyStream( + dataTransactionContext, + persistedTable, + parseResult.rowsAsync + ? handler.consumeBatchesAsync( + batchGenerator as AsyncGenerator< + Result, DomainError> + >, + pluginExecution, + dataTransactionContext, + { + table: persistedTable, + batchSize: command.batchSize, + operationId, + totalRecordCount, + events: recordEvents, + } + ) + : handler.consumeBatches( + batchGenerator as Generator, DomainError>>, + pluginExecution, + dataTransactionContext, + { + table: persistedTable, + batchSize: command.batchSize, + operationId, + totalRecordCount, + events: recordEvents, + } + ) + ); + + return ok({ totalImported: insertResult.totalInserted, events: recordEvents }); + } + ); }, { scope: 'data' } ); @@ -292,32 +355,82 @@ export class ImportCsvHandler implements ICommandHandler, + sampleRows: ReadonlyArray>, + columns: ReadonlyArray | undefined + ): Result, DomainError> { + const fieldTypes = this.inferFieldTypes(headers, sampleRows); + + if (!columns?.length) { + return ok( + headers.map((header, index) => ({ + name: header || `Column_${index + 1}`, + sourceColumnIndex: index, + type: fieldTypes[index] ?? 'singleLineText', + })) + ); + } + + const resolvedColumns: ResolvedImportColumn[] = []; + for (const column of columns) { + if (column.sourceColumnIndex < 0 || column.sourceColumnIndex >= headers.length) { + return err( + domainError.validation({ + code: 'import.column_index_out_of_range', + message: `Column index ${column.sourceColumnIndex} is out of range`, + details: { + sourceColumnIndex: column.sourceColumnIndex, + columnCount: headers.length, + }, + }) + ); + } + + const fallbackName = `Column_${column.sourceColumnIndex + 1}`; + resolvedColumns.push({ + name: column.name || headers[column.sourceColumnIndex] || fallbackName, + sourceColumnIndex: column.sourceColumnIndex, + type: + this.resolveFieldType(column.type) ?? + fieldTypes[column.sourceColumnIndex] ?? + 'singleLineText', + }); + } + + return ok(resolvedColumns); + } + /** - * 从 CSV 头部构建表 - * 所有列都是 SingleLineText 类型，第一列为主键 + * 从导入列定义构建表。 */ - private buildTableFromHeaders( + private buildTableFromColumns( baseId: ImportCsvCommand['baseId'], tableName: TableName, - headers: ReadonlyArray + columns: ReadonlyArray ): Result { const builder = Table.builder().withBaseId(baseId).withName(tableName); + const seenFieldNames: string[] = []; - for (let i = 0; i < headers.length; i++) { - const header = headers[i]; - const fieldNameResult = FieldName.create(header || `Column_${i + 1}`); + for (let i = 0; i < columns.length; i++) { + const column = columns[i]; + const fieldName = this.getUniqueFieldName(column.name, seenFieldNames); + seenFieldNames.push(fieldName); + const fieldNameResult = FieldName.create(fieldName); if (fieldNameResult.isErr()) { return err(fieldNameResult.error); } - const fieldBuilder = builder.field().singleLineText().withName(fieldNameResult.value); + const fieldBuilder = this.createFieldBuilder(builder, column.type).withName( + fieldNameResult.value + ); // 第一列设为主键 if (i === 0) { @@ -333,16 +446,126 @@ export class ImportCsvHandler implements ICommandHandler, + sampleRows: ReadonlyArray> + ): InferredCsvFieldType[] { + return headers.map((header) => { + const values = sampleRows + .map((row) => row[header]) + .filter((value): value is string => value != null && value !== ''); + if (values.length === 0) { + return 'singleLineText'; + } + + let candidates = [...inferredCsvFieldTypeOrder]; + for (const value of values) { + if (candidates.length <= 1) { + break; + } + + if (this.matchesInferredType(value, 'longText')) { + candidates = ['longText']; + break; + } + + candidates = candidates.filter((type) => this.matchesInferredType(value, type)); + } + + return candidates[0] ?? 'singleLineText'; + }); + } + + private matchesInferredType(value: string, type: InferredCsvFieldType): boolean { + switch (type) { + case 'checkbox': + return value.toLowerCase() === 'true' || value.toLowerCase() === 'false'; + case 'number': + return value.trim() !== '' && !Number.isNaN(Number(value)); + case 'date': + return this.isValidImportDate(value); + case 'longText': + return /\n/.test(value); + case 'singleLineText': + return true; + } + } + + private isValidImportDate(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed || !dateFormatPatterns.some((pattern) => pattern.test(trimmed))) { + return false; + } + + const date = new Date(value); + if (date.toString() === 'Invalid Date') { + return false; + } + + const year = date.getFullYear(); + return year >= reasonableYearMin && year <= reasonableYearMax; + } + + private createFieldBuilder( + builder: ReturnType, + type: InferredCsvFieldType + ) { + switch (type) { + case 'checkbox': + return builder.field().checkbox(); + case 'number': + return builder.field().number(); + case 'date': + return builder.field().date(); + case 'longText': + return builder.field().longText(); + case 'singleLineText': + return builder.field().singleLineText(); + } + } + + private resolveFieldType(type: string | undefined): InferredCsvFieldType | undefined { + switch (type) { + case 'checkbox': + case 'number': + case 'date': + case 'longText': + case 'singleLineText': + return type; + default: + return undefined; + } + } + + private getUniqueFieldName(name: string, seenNames: ReadonlyArray): string { + const trimmed = name.trim() || 'Field'; + if (!seenNames.includes(trimmed)) { + return trimmed; + } + + let index = 2; + let candidate = `${trimmed} ${index}`; + while (seenNames.includes(candidate)) { + index += 1; + candidate = `${trimmed} ${index}`; + } + return candidate; + } + /** * 构建字段 ID 映射（CSV 列名 → 字段 ID） */ - private buildFieldIdMap(table: Table, headers: ReadonlyArray): Map { + private buildFieldIdMap( + table: Table, + headers: ReadonlyArray, + columns: ReadonlyArray + ): Map { const fields = table.getFields(); const map = new Map(); - // 按顺序匹配（假设字段顺序与 headers 顺序一致） - for (let i = 0; i < headers.length && i < fields.length; i++) { - const header = headers[i]; + // 按导入列定义匹配，保留 sourceColumnIndex 指向的源列。 + for (let i = 0; i < columns.length && i < fields.length; i++) { + const header = headers[columns[i].sourceColumnIndex]; const field = fields[i]; map.set(header, field.id().toString()); } @@ -362,6 +585,9 @@ export class ImportCsvHandler implements ICommandHandler, + options: ChunkPluginOptions + ): void { + const eventRecords = this.toEventRecords(records); + if (eventRecords.length === 0) { + return; + } + + options.events.push( + RecordsBatchCreated.create({ + tableId: options.table.id(), + baseId: options.table.baseId(), + records: eventRecords, + }) + ); + } + + private toEventRecords(records: ReadonlyArray): ReadonlyArray { + return records.map((record) => ({ + recordId: record.id().toString(), + fields: record + .fields() + .entries() + .map((entry) => ({ + fieldId: entry.fieldId.toString(), + value: entry.value.toValue(), + })), + })); + } + private async prepareChunkPluginExecution( transactionContext: ExecutionContextPort.IExecutionContext, previousExecution: RecordWritePluginExecution, @@ -507,18 +766,77 @@ export class ImportCsvHandler implements ICommandHandler>, + sampleSize: number + ): Promise<{ + sampleRows: ReadonlyArray>; + rowsAsync: AsyncIterable>; + }> { + const iterator = rowsAsync[Symbol.asyncIterator](); + const sampleRows: Record[] = []; + + while (sampleRows.length < sampleSize) { + const next = await iterator.next(); + if (next.done) { + break; + } + sampleRows.push(next.value); + } + + return { + sampleRows, + rowsAsync: this.prependRows(sampleRows, iterator), + }; + } + + private async *prependRows( + rows: ReadonlyArray>, + iterator: AsyncIterator> + ): AsyncIterable> { + for (const row of rows) { + yield row; + } + + while (true) { + const next = await iterator.next(); + if (next.done) { + return; + } + yield next.value; + } + } + /** * 将 CSV 行异步迭代器转换为记录字段值的 AsyncIterable */ private async *createRecordsIterableAsync( rows: AsyncIterable>, - fieldIdMap: Map + fieldIdMap: Map, + maxRowCount?: number ): AsyncIterable> { + let rowCount = 0; for await (const row of rows) { + rowCount += 1; + if (maxRowCount !== undefined && rowCount > maxRowCount) { + throw domainError.validation({ + code: 'validation.limit.rows_per_table_max', + message: `Exceed max row limit: ${maxRowCount}`, + details: { + max: maxRowCount, + maxRowCount, + rowCount, + }, + }); + } + const fieldValues = new Map(); for (const [csvColumn, value] of Object.entries(row)) { const fieldId = fieldIdMap.get(csvColumn); + if (value === '') { + continue; + } if (fieldId) { fieldValues.set(fieldId, value); } diff --git a/packages/v2/core/src/commands/ImportRecordsHandler.spec.ts b/packages/v2/core/src/commands/ImportRecordsHandler.spec.ts index 585e3ed904..2cf66e75d4 100644 --- a/packages/v2/core/src/commands/ImportRecordsHandler.spec.ts +++ b/packages/v2/core/src/commands/ImportRecordsHandler.spec.ts @@ -168,6 +168,8 @@ class FakeTableRepository implements ITableRepository { class FakeTableRecordRepository implements ITableRecordRepository { inserted: TableRecord[] = []; + insertedBatches: TableRecord[][] = []; + insertManyStreamOptions: InsertManyStreamOptions | undefined; async insert( _: IExecutionContext, @@ -191,11 +193,13 @@ class FakeTableRecordRepository implements ITableRecordRepository { batches: Iterable> | AsyncIterable>, options?: InsertManyStreamOptions ): Promise> { + this.insertManyStreamOptions = options; let totalInserted = 0; let batchIndex = 0; if (isAsyncIterable(batches)) { for await (const batch of batches) { + this.insertedBatches.push([...batch]); this.inserted.push(...batch); totalInserted += batch.length; options?.onBatchInserted?.({ batchIndex, insertedCount: batch.length, totalInserted }); @@ -203,6 +207,7 @@ class FakeTableRecordRepository implements ITableRecordRepository { } } else { for (const batch of batches) { + this.insertedBatches.push([...batch]); this.inserted.push(...batch); totalInserted += batch.length; options?.onBatchInserted?.({ batchIndex, insertedCount: batch.length, totalInserted }); @@ -425,7 +430,15 @@ describe('ImportRecordsHandler', () => { expect(result._unsafeUnwrap().totalImported).toBe(1); expect(tableRecordRepository.inserted).toHaveLength(1); expect(adapter.parseCalls).toHaveLength(1); - expectRecordWritePluginToBeSkipped(calls, RecordWriteOperationKind.importAppend); + expect(calls.supports).toEqual([ + RecordWriteOperationKind.importAppend, + RecordWriteOperationKind.importAppend, + ]); + expect(calls.prepare).toHaveLength(0); + expect(calls.prepareStates).toHaveLength(0); + expect(calls.guard).toHaveLength(0); + expect(calls.beforePersist).toHaveLength(0); + expect(calls.afterCommit).toHaveLength(0); }); it('returns validation.limit.rows_per_table_max for async sources that exceed maxRowCount', async () => { @@ -479,6 +492,83 @@ describe('ImportRecordsHandler', () => { expect(calls.guard).toHaveLength(0); }); + it('checks importAppend plugins per append chunk instead of the whole imported row count', async () => { + const { table, textFieldId } = buildTable(); + const adapter = new FakeImportSourceAdapter( + ok({ + headers: ['Title'], + rows: [['row 1'], ['row 2']], + }) + ); + const tableRecordRepository = new FakeTableRecordRepository(); + const seenScopes: Array<{ + scope?: 'operation' | 'chunk'; + recordCount: number; + recordsLength: number; + chunkIndex?: number; + }> = []; + + const handler = new ImportRecordsHandler( + new FakeImportSourceRegistry(adapter), + new FakeTableRepository([table]), + tableRecordRepository, + { + needsResolution: () => ok(false), + resolveAndReplaceMany: async () => ok([]), + } as unknown as RecordMutationSpecResolverService, + createRecordWritePluginRunner([ + { + name: 'chunk-record-limit', + supports: (operation) => operation === RecordWriteOperationKind.importAppend, + async guard(context) { + seenScopes.push({ + scope: context.orchestration?.scope, + recordCount: context.payload.recordCount, + recordsLength: context.payload.recordsFieldValues.length, + chunkIndex: context.orchestration?.chunkIndex, + }); + + if (context.orchestration?.scope === 'chunk' && context.payload.recordCount > 1) { + return err( + domainError.validation({ + code: 'validation.limit.records_per_mutation_max', + message: 'too many rows in one mutation', + }) + ); + } + + return ok(undefined); + }, + }, + ]), + { + execute: () => ok({ table, updateResult: undefined }), + } as unknown as RecordWriteSideEffectService, + { + execute: async () => ok({ table, events: [] }), + } as unknown as TableUpdateFlow, + new FakeEventBus(), + new FakeUnitOfWork() + ); + + const result = await handler.handle( + createContext(), + createCommand(table.id().toString(), textFieldId.toString(), undefined, { + batchSize: 1, + typecast: true, + }) + ); + + expect(result.isOk()).toBe(true); + expect(result._unsafeUnwrap().totalImported).toBe(2); + expect(tableRecordRepository.insertedBatches.map((batch) => batch.length)).toEqual([1, 1]); + expect(seenScopes).toEqual([ + { scope: 'operation', recordCount: 2, recordsLength: 0, chunkIndex: undefined }, + { scope: 'chunk', recordCount: 1, recordsLength: 1, chunkIndex: 0 }, + { scope: 'chunk', recordCount: 1, recordsLength: 1, chunkIndex: 1 }, + ]); + }); + it('stops before insert when plugin beforePersist rejects', async () => { const { table, textFieldId } = buildTable(); const adapter = new FakeImportSourceAdapter( @@ -591,6 +681,10 @@ describe('ImportRecordsHandler', () => { expect(published[0]).toBe(event); expect(isRecordsBatchCreatedEvent(published[1])).toBe(true); expect(tableRecordRepository.inserted).toHaveLength(1); + expect(tableRecordRepository.insertManyStreamOptions).toMatchObject({ + deferComputedUpdates: true, + enqueueDeferredComputedUpdates: true, + }); }); it('resolves mutate specs before yielding imported records', async () => { diff --git a/packages/v2/core/src/commands/ImportRecordsHandler.ts b/packages/v2/core/src/commands/ImportRecordsHandler.ts index dedaddb507..d5aa6f1d7e 100644 --- a/packages/v2/core/src/commands/ImportRecordsHandler.ts +++ b/packages/v2/core/src/commands/ImportRecordsHandler.ts @@ -3,10 +3,11 @@ import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; import { RecordMutationSpecResolverService } from '../application/services/RecordMutationSpecResolverService'; +import type { RecordWritePluginExecution } from '../application/services/RecordWritePluginRunner'; import { RecordWritePluginRunner } from '../application/services/RecordWritePluginRunner'; import { RecordWriteSideEffectService } from '../application/services/RecordWriteSideEffectService'; import { TableUpdateFlow } from '../application/services/TableUpdateFlow'; -import { domainError, type DomainError } from '../domain/shared/DomainError'; +import { domainError, isDomainError, type DomainError } from '../domain/shared/DomainError'; import type { IDomainEvent } from '../domain/shared/DomainEvent'; import { RecordsBatchCreated } from '../domain/table/events/RecordsBatchCreated'; import type { RecordValuesDTO } from '../domain/table/events/RecordFieldValuesDTO'; @@ -16,7 +17,12 @@ import { TableByIdSpec } from '../domain/table/specs/TableByIdSpec'; import type { Table } from '../domain/table/Table'; import * as EventBusPort from '../ports/EventBus'; import type { IExecutionContext } from '../ports/ExecutionContext'; -import { RecordWriteOperationKind, type RecordWriteFieldValues } from '../ports/RecordWritePlugin'; +import { + RecordWriteOperationKind, + type RecordWriteFieldValues, + type RecordWriteImportAppendPayload, + type RecordWritePluginOrchestration, +} from '../ports/RecordWritePlugin'; import type { IImportParseResult, IImportProgress, @@ -52,6 +58,13 @@ interface ImportStreamState { table: Table; events: IDomainEvent[]; currentBatch: number; + operationId: string; + totalRecordCount: number; + totalChunkCount: number; + sourceType: string; + sourceColumnMap: SourceColumnMap; + maxRowCount?: number; + previousPluginExecution: RecordWritePluginExecution; } /** @@ -148,28 +161,42 @@ export class ImportRecordsHandler } throw error; } - const pluginExecution = yield* await handler.recordWritePluginRunner.prepare({ - kind: RecordWriteOperationKind.importAppend, - executionContext: context, + const operationId = `import-records:${tableId.toString()}`; + const totalChunkCount = collectedBatches.batches.length; + const operationPluginExecution = yield* await handler.preparePluginExecution( + context, table, - payload: { + { sourceType: source.type, sourceColumnMap, - recordsFieldValues: collectedBatches.batches.flat(), + recordsFieldValues: [], batchSize, typecast, recordCount: collectedBatches.totalCount, maxRowCount, }, - isTransactionBound: false, - }); - yield* await pluginExecution.guard(); + { + mode: 'stream', + scope: 'operation', + operationId, + totalRecordCount: collectedBatches.totalCount, + totalChunkCount, + }, + false + ); // 5. Create streaming state const state: ImportStreamState = { table, events: [], currentBatch: 0, + operationId, + totalRecordCount: collectedBatches.totalCount, + totalChunkCount, + sourceType: source.type, + sourceColumnMap, + maxRowCount, + previousPluginExecution: operationPluginExecution, }; // 6. Stream insert via insertManyStream @@ -178,32 +205,40 @@ export class ImportRecordsHandler insertResult = yield* await handler.unitOfWork.withTransaction( context, async (transactionContext) => { - const recordBatches = handler.createRecordBatchesStream( - transactionContext, - state, - collectedBatches.batches, - typecast, - onProgress - ); - const beforePersistResult = await pluginExecution.beforePersist(transactionContext); - if (beforePersistResult.isErr()) { - return err(beforePersistResult.error); - } - return handler.tableRecordRepository.insertManyStream( - transactionContext, - state.table, - recordBatches, - { - deferComputedUpdates: true, - onBatchInserted: (progress) => { - onProgress?.({ - phase: 'inserting', - processedRows: progress.totalInserted, - currentBatch: state.currentBatch, - }); - }, + try { + const recordBatches = handler.createRecordBatchesStream( + transactionContext, + state, + collectedBatches.batches, + typecast, + onProgress + ); + return await handler.tableRecordRepository.insertManyStream( + transactionContext, + state.table, + recordBatches, + { + deferComputedUpdates: true, + enqueueDeferredComputedUpdates: true, + onBatchInserted: (progress) => { + onProgress?.({ + phase: 'inserting', + processedRows: progress.totalInserted, + currentBatch: state.currentBatch, + }); + }, + } + ); + } catch (error) { + if (isDomainError(error)) { + return err(error); } - ); + return err( + domainError.fromUnknown(error, { + code: 'import.insert_stream_failed', + }) + ); + } } ); @@ -217,12 +252,43 @@ export class ImportRecordsHandler processedRows: insertResult.totalInserted, currentBatch: state.currentBatch, }); - await pluginExecution.afterCommit(); + await state.previousPluginExecution.afterCommit(); return ok(ImportRecordsResult.create(insertResult.totalInserted, state.events)); }); } + private async preparePluginExecution( + context: IExecutionContext, + table: Table, + payload: RecordWriteImportAppendPayload, + orchestration: RecordWritePluginOrchestration, + isTransactionBound: boolean, + previousExecution?: RecordWritePluginExecution + ): Promise> { + const pluginExecutionResult = await this.recordWritePluginRunner.prepare( + { + kind: RecordWriteOperationKind.importAppend, + executionContext: context, + table, + payload, + orchestration, + isTransactionBound, + }, + previousExecution ? { previousExecution } : undefined + ); + if (pluginExecutionResult.isErr()) { + return err(pluginExecutionResult.error); + } + + const guardResult = await pluginExecutionResult.value.guard(); + if (guardResult.isErr()) { + return err(guardResult.error); + } + + return ok(pluginExecutionResult.value); + } + /** * Create async generator that yields processed record batches. * Each batch goes through: side effects → create records → resolve links. @@ -235,11 +301,42 @@ export class ImportRecordsHandler onProgress?: (progress: IImportProgress) => void ): AsyncGenerator> { for (const batchFieldValues of fieldValueBatches) { + const chunkIndex = state.currentBatch; state.currentBatch++; - onProgress?.({ phase: 'inserting', processedRows: 0, currentBatch: state.currentBatch }); + const currentBatch = state.currentBatch; + onProgress?.({ phase: 'inserting', processedRows: 0, currentBatch }); if (batchFieldValues.length === 0) continue; + const pluginExecutionResult = await this.preparePluginExecution( + context, + state.table, + { + sourceType: state.sourceType, + sourceColumnMap: state.sourceColumnMap, + recordsFieldValues: batchFieldValues, + batchSize: batchFieldValues.length, + typecast, + recordCount: batchFieldValues.length, + maxRowCount: state.maxRowCount, + }, + { + mode: 'stream', + scope: 'chunk', + operationId: state.operationId, + totalRecordCount: state.totalRecordCount, + totalChunkCount: state.totalChunkCount, + chunkIndex, + }, + true, + state.previousPluginExecution + ); + if (pluginExecutionResult.isErr()) { + throw pluginExecutionResult.error; + } + const pluginExecution = pluginExecutionResult.value; + state.previousPluginExecution = pluginExecution; + // Handle side effects for this batch (discover new select options) if (typecast) { const sideEffectResult = this.recordWriteSideEffectService.execute( @@ -292,10 +389,22 @@ export class ImportRecordsHandler tableId: state.table.id(), baseId: state.table.baseId(), records: eventRecords, + orchestration: { + operationId: state.operationId, + totalRecordCount: state.totalRecordCount, + totalChunkCount: state.totalChunkCount, + chunkIndex, + scope: 'chunk', + }, }) ); } + const beforePersistResult = await pluginExecution.beforePersist(context); + if (beforePersistResult.isErr()) { + throw beforePersistResult.error; + } + // Yield processed batch for insertion yield records; } diff --git a/packages/v2/core/src/commands/PasteHandler.spec.ts b/packages/v2/core/src/commands/PasteHandler.spec.ts index b67f456423..af4035d868 100644 --- a/packages/v2/core/src/commands/PasteHandler.spec.ts +++ b/packages/v2/core/src/commands/PasteHandler.spec.ts @@ -479,7 +479,7 @@ class FakeTableRecordRepository implements ITableRecordRepository { this.updateStreamContexts.push(context); this.onUpdateManyStream?.(table); let totalUpdated = 0; - const updatedRecords: NonNullable = []; + const updatedRecords: Array[number]> = []; const normalizeBatch = ( batch: UpdateManyStreamBatchInput ): ReadonlyArray => @@ -865,7 +865,7 @@ describe('PasteHandler', () => { noopRecordWriteUndoRedoPlanService, createRecordWritePluginRunner(), eventBus, - undoRedoService as unknown as UndoRedoService, + undoRedoService as unknown as UndoRedoStackService, unitOfWork ); @@ -925,7 +925,7 @@ describe('PasteHandler', () => { noopRecordWriteUndoRedoPlanService, createRecordWritePluginRunner(), eventBus, - undoRedoService as unknown as UndoRedoService, + undoRedoService as unknown as UndoRedoStackService, unitOfWork ); diff --git a/packages/v2/core/src/commands/PasteHandler.ts b/packages/v2/core/src/commands/PasteHandler.ts index fa96e4fbf9..9beff3b4fe 100644 --- a/packages/v2/core/src/commands/PasteHandler.ts +++ b/packages/v2/core/src/commands/PasteHandler.ts @@ -79,6 +79,7 @@ import type { RecordFilter } from '../queries/RecordFilterDto'; import { buildRecordConditionSpec, buildSanitizedRecordConditionSpec, + replaceCurrentUserTagInFilter, } from '../queries/RecordFilterMapper'; import type { RecordSearch } from '../queries/RecordSearch'; import { resolveVisibleRowSearch } from '../queries/RecordSearch'; @@ -353,7 +354,15 @@ export class PasteHandler implements ICommandHandler // 3. Build filter spec from effective view filter. Search-aware visible rows are handled // by the query repository so field-type-specific search semantics stay centralized. - const filterSpec = yield* buildSanitizedRecordConditionSpec(persistedTable, effectiveFilter); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + persistedTable, + effectiveFilter, + context.actorId.toString() + ); + const filterSpec = yield* buildSanitizedRecordConditionSpec( + persistedTable, + actorResolvedFilter + ); const visibleRowSearch = resolveVisibleRowSearch(command.search, orderedFieldIds); // 4. Get total row count for columns/rows type normalization @@ -421,8 +430,7 @@ export class PasteHandler implements ICommandHandler return ok({ updatedCount: 0, createdCount: 0, createdRecordIds: [] }); } - // 10. Build orderBy from group + sort for correct row mapping - // If none provided, fall back to view row order column (__row_{viewId}) + // 10. Build orderBy from group + sort to match the visible list row order. const effectiveGroup = command.ignoreViewQuery ? command.groupBy ?? undefined : mergedDefaults.group(); @@ -2464,9 +2472,14 @@ export class PasteStreamApplicationService extends PasteHandler { ? command.sort ?? undefined : mergedDefaults.sort(); + const actorResolvedFilter = replaceCurrentUserTagInFilter( + persistedTable, + effectiveFilter, + context.actorId.toString() + ); const filterSpecResult = await buildSanitizedRecordConditionSpec( persistedTable, - effectiveFilter + actorResolvedFilter ); if (filterSpecResult.isErr()) { return err(filterSpecResult.error); diff --git a/packages/v2/core/src/commands/TableFieldSpecs.spec.ts b/packages/v2/core/src/commands/TableFieldSpecs.spec.ts index 4d55b3eb8c..07044b4e74 100644 --- a/packages/v2/core/src/commands/TableFieldSpecs.spec.ts +++ b/packages/v2/core/src/commands/TableFieldSpecs.spec.ts @@ -22,8 +22,10 @@ type FieldInput = Parameters[0][number]; const createContextWithSelectOptionLimit = (maxChoicesPerField: number): IExecutionContext => ({ actorId: ActorId.create('system')._unsafeUnwrap(), config: { - selectFieldOptions: { - maxChoicesPerField, + tableLimits: { + fieldOptions: { + maxSelectChoices: maxChoicesPerField, + }, }, }, }); diff --git a/packages/v2/core/src/commands/TableFieldSpecs.ts b/packages/v2/core/src/commands/TableFieldSpecs.ts index fb4f535cea..8cc6a4d80f 100644 --- a/packages/v2/core/src/commands/TableFieldSpecs.ts +++ b/packages/v2/core/src/commands/TableFieldSpecs.ts @@ -1498,7 +1498,8 @@ class CreateSingleSelectFieldSpec implements ICreateTableFieldSpec { private readonly defaultValue: SelectDefaultValue | undefined, private readonly preventAutoNewOptions: SelectAutoNewOptions | undefined, private readonly notNull: FieldNotNull, - private readonly unique: FieldUnique + private readonly unique: FieldUnique, + private readonly domainContext: IDomainContext | undefined ) {} static create( @@ -1522,7 +1523,8 @@ class CreateSingleSelectFieldSpec implements ICreateTableFieldSpec { meta.defaultValue, meta.preventAutoNewOptions, meta.notNull, - meta.unique + meta.unique, + meta.domainContext ).withPrimary(meta.isPrimary) ); } @@ -1554,6 +1556,7 @@ class CreateSingleSelectFieldSpec implements ICreateTableFieldSpec { options: this.options, defaultValue: this.defaultValue, preventAutoNewOptions: this.preventAutoNewOptions, + domainContext: this.domainContext, notNull: this.notNull, unique: this.unique, }) @@ -1580,7 +1583,8 @@ class CreateMultipleSelectFieldSpec implements ICreateTableFieldSpec { private readonly defaultValue: SelectDefaultValue | undefined, private readonly preventAutoNewOptions: SelectAutoNewOptions | undefined, private readonly notNull: FieldNotNull, - private readonly unique: FieldUnique + private readonly unique: FieldUnique, + private readonly domainContext: IDomainContext | undefined ) {} static create( @@ -1604,7 +1608,8 @@ class CreateMultipleSelectFieldSpec implements ICreateTableFieldSpec { meta.defaultValue, meta.preventAutoNewOptions, meta.notNull, - meta.unique + meta.unique, + meta.domainContext ).withPrimary(meta.isPrimary) ); } @@ -1636,6 +1641,7 @@ class CreateMultipleSelectFieldSpec implements ICreateTableFieldSpec { options: this.options, defaultValue: this.defaultValue, preventAutoNewOptions: this.preventAutoNewOptions, + domainContext: this.domainContext, notNull: this.notNull, unique: this.unique, }) @@ -2254,11 +2260,14 @@ class CreateButtonFieldSpec implements ICreateTableFieldSpec { const sequence = ( values: ReadonlyArray> -): Result, DomainError> => - values.reduce, DomainError>>( - (acc, next) => acc.andThen((arr) => next.map((v) => [...arr, v])), - ok([]) - ); +): Result, DomainError> => { + const result: T[] = []; + for (const value of values) { + if (value.isErr()) return err, DomainError>(value.error); + result.push(value.value); + } + return ok(result); +}; const optional = ( raw: unknown, @@ -2318,17 +2327,28 @@ type ParsedSelectOptions = { preventAutoNewOptions?: SelectAutoNewOptions; }; +const parseSelectOptionList = ( + values: ReadonlyArray, + parser: (value: unknown, index: number) => Result +): Result, DomainError> => { + const options: SelectOption[] = []; + for (let index = 0; index < values.length; index += 1) { + const optionResult = parser(values[index], index); + if (optionResult.isErr()) return err(optionResult.error); + options.push(optionResult.value); + } + return ok(options); +}; + const parseSelectOptions = (raw: unknown): Result => { if (raw == null) return ok({ options: [] }); if (Array.isArray(raw)) { - const optionsResult = sequence( - raw.map((name, index) => - SelectOption.create({ - name, - color: fieldColorValues[index % fieldColorValues.length], - }) - ) + const optionsResult = parseSelectOptionList(raw, (name, index) => + SelectOption.create({ + name, + color: fieldColorValues[index % fieldColorValues.length], + }) ); return optionsResult.map((options) => ({ options })); } @@ -2340,20 +2360,18 @@ const parseSelectOptions = (raw: unknown): Result { - if (choice && typeof choice === 'object' && !Array.isArray(choice)) { - const rawChoice = choice as Record; - if (rawChoice.color == null) { - return SelectOption.create({ - ...rawChoice, - color: fieldColorValues[index % fieldColorValues.length], - }); - } + return parseSelectOptionList(rawChoices, (choice, index) => { + if (choice && typeof choice === 'object' && !Array.isArray(choice)) { + const rawChoice = choice as Record; + if (rawChoice.color == null) { + return SelectOption.create({ + ...rawChoice, + color: fieldColorValues[index % fieldColorValues.length], + }); } - return SelectOption.create(choice); - }) - ).andThen((options) => + } + return SelectOption.create(choice); + }).andThen((options) => optional(rawOptions.defaultValue, SelectDefaultValue.create).andThen((defaultValue) => optional(rawOptions.preventAutoNewOptions, SelectAutoNewOptions.create).map( (preventAutoNewOptions) => ({ diff --git a/packages/v2/core/src/commands/TableFieldUpdateSpecs.same-type.spec.ts b/packages/v2/core/src/commands/TableFieldUpdateSpecs.same-type.spec.ts index 9f3339bd38..5304d53d02 100644 --- a/packages/v2/core/src/commands/TableFieldUpdateSpecs.same-type.spec.ts +++ b/packages/v2/core/src/commands/TableFieldUpdateSpecs.same-type.spec.ts @@ -14,6 +14,7 @@ import { CellValueType } from '../domain/table/fields/types/CellValueType'; import { CheckboxDefaultValue } from '../domain/table/fields/types/CheckboxDefaultValue'; import { DateTimeFormatting } from '../domain/table/fields/types/DateTimeFormatting'; import { FieldColor } from '../domain/table/fields/types/FieldColor'; +import { FieldHasError } from '../domain/table/fields/types/FieldHasError'; import { FormulaExpression } from '../domain/table/fields/types/FormulaExpression'; import { NumberDefaultValue } from '../domain/table/fields/types/NumberDefaultValue'; import { NumberFormatting } from '../domain/table/fields/types/NumberFormatting'; @@ -34,6 +35,7 @@ import { UpdateSingleSelectOptionsSpec } from '../domain/table/specs/field-updat import { TableUpdateFieldAiConfigSpec } from '../domain/table/specs/TableUpdateFieldAiConfigSpec'; import { TableUpdateFieldDbFieldNameSpec } from '../domain/table/specs/TableUpdateFieldDbFieldNameSpec'; import { TableUpdateFieldDescriptionSpec } from '../domain/table/specs/TableUpdateFieldDescriptionSpec'; +import { TableUpdateFieldHasErrorSpec } from '../domain/table/specs/TableUpdateFieldHasErrorSpec'; import { Table } from '../domain/table/Table'; import { TableId } from '../domain/table/TableId'; import { TableName } from '../domain/table/TableName'; @@ -808,6 +810,43 @@ describe('TableFieldUpdateSpecs same-type updates', () => { ); }); + it('clears formula hasError when replacing the broken expression', () => { + const { currentField } = buildHarness('x', 'x', (builder, fieldId, primaryFieldId) => { + builder + .field() + .formula() + .withId(fieldId) + .withName(FieldName.create('Calc')._unsafeUnwrap()) + .withExpression(FormulaExpression.create(`{${primaryFieldId.toString()}}`)._unsafeUnwrap()) + .withResultType({ + cellValueType: CellValueType.string(), + isMultipleCellValue: CellValueMultiplicity.single(), + }) + .done(); + }); + currentField.setHasError(FieldHasError.error()); + + const specsResult = buildUpdateFieldSpecs(currentField, { + type: 'formula', + options: { + expression: '"repaired"', + }, + }); + + expect(specsResult.isOk()).toBe(true); + if (specsResult.isErr()) { + return; + } + + const specNames = specsResult.value.map((spec) => spec.constructor.name); + expect(specNames).toContain('UpdateFormulaExpressionSpec'); + const hasErrorSpec = specsResult.value.find( + (spec): spec is TableUpdateFieldHasErrorSpec => spec instanceof TableUpdateFieldHasErrorSpec + ); + expect(hasErrorSpec).toBeDefined(); + expect(hasErrorSpec?.isSettingError()).toBe(false); + }); + it.each([ { name: 'lastModifiedTime', diff --git a/packages/v2/core/src/commands/TableFieldUpdateSpecs.ts b/packages/v2/core/src/commands/TableFieldUpdateSpecs.ts index b239b203ef..df24559071 100644 --- a/packages/v2/core/src/commands/TableFieldUpdateSpecs.ts +++ b/packages/v2/core/src/commands/TableFieldUpdateSpecs.ts @@ -121,6 +121,7 @@ import { TableUpdateFieldAiConfigSpec } from '../domain/table/specs/TableUpdateF import { TableUpdateFieldConstraintsSpec } from '../domain/table/specs/TableUpdateFieldConstraintsSpec'; import { TableUpdateFieldDbFieldNameSpec } from '../domain/table/specs/TableUpdateFieldDbFieldNameSpec'; import { TableUpdateFieldDescriptionSpec } from '../domain/table/specs/TableUpdateFieldDescriptionSpec'; +import { TableUpdateFieldHasErrorSpec } from '../domain/table/specs/TableUpdateFieldHasErrorSpec'; import { TableUpdateFieldNameSpec } from '../domain/table/specs/TableUpdateFieldNameSpec'; import { TableUpdateFieldTypeSpec } from '../domain/table/specs/TableUpdateFieldTypeSpec'; import type { Table } from '../domain/table/Table'; @@ -190,11 +191,14 @@ const parseRequiredFormulaShowAs = (raw: unknown): Result( values: ReadonlyArray> -): Result, DomainError> => - values.reduce, DomainError>>( - (acc, next) => acc.andThen((arr) => next.map((v) => [...arr, v])), - ok([]) - ); +): Result, DomainError> => { + const result: T[] = []; + for (const value of values) { + if (value.isErr()) return err, DomainError>(value.error); + result.push(value.value); + } + return ok(result); +}; const parseTrackedFieldIds = ( raw: unknown @@ -2228,6 +2232,11 @@ class UpdateFormulaFieldSpec implements IUpdateTableFieldSpec { this.expressionValue ) ); + if (currentField.hasError().isError()) { + specs.push( + TableUpdateFieldHasErrorSpec.clearError(currentField.id(), currentField.hasError()) + ); + } } } diff --git a/packages/v2/core/src/commands/UpdateRecordHandler.spec.ts b/packages/v2/core/src/commands/UpdateRecordHandler.spec.ts index 68e665b185..8df93efda3 100644 --- a/packages/v2/core/src/commands/UpdateRecordHandler.spec.ts +++ b/packages/v2/core/src/commands/UpdateRecordHandler.spec.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest'; import type { RecordMutationSpecResolverService } from '../application/services/RecordMutationSpecResolverService'; import { RecordWriteSideEffectService } from '../application/services/RecordWriteSideEffectService'; import type { RecordWriteUndoRedoPlanService } from '../application/services/RecordWriteUndoRedoPlanService'; +import { ForeignTableLoaderService } from '../application/services/ForeignTableLoaderService'; import { TableQueryService } from '../application/services/TableQueryService'; import { TableUpdateFlow } from '../application/services/TableUpdateFlow'; import type { @@ -52,6 +53,7 @@ import type { ITableRecordRepository, RecordMutationResult, BatchRecordMutationResult, + UpdateOptions, } from '../ports/TableRecordRepository'; import type { ITableRepository } from '../ports/TableRepository'; import type { ITableSchemaRepository } from '../ports/TableSchemaRepository'; @@ -183,7 +185,10 @@ const buildTableWithLink = () => { return { table: builder.build()._unsafeUnwrap(), + baseId, tableId, + foreignTableId, + foreignPrimaryFieldId, textFieldId, linkFieldId, }; @@ -270,6 +275,7 @@ class FakeTableRecordRepository implements ITableRecordRepository { lastContext: IExecutionContext | undefined; lastRecordId: RecordId | undefined; lastMutateSpec: ICellValueSpec | undefined; + lastUpdateOptions: UpdateOptions | undefined; omitUpdateSnapshot = false; mutationApplied: boolean | undefined = true; changedFields: ReadonlyMap | undefined; @@ -302,11 +308,13 @@ class FakeTableRecordRepository implements ITableRecordRepository { context: IExecutionContext, _: Table, recordId: RecordId, - mutateSpec: ICellValueSpec + mutateSpec: ICellValueSpec, + options?: UpdateOptions ): Promise> { this.lastContext = context; this.lastRecordId = recordId; this.lastMutateSpec = mutateSpec; + this.lastUpdateOptions = options; return ok( this.omitUpdateSnapshot ? { @@ -606,6 +614,90 @@ describe('UpdateRecordHandler', () => { ).toEqual(persistedLinkValue); }); + it('fills missing persisted link titles for id-only link updates without typecast', async () => { + const { + table, + baseId, + tableId, + foreignTableId, + foreignPrimaryFieldId, + textFieldId, + linkFieldId, + } = buildTableWithLink(); + const targetRecordId = `rec${'r'.repeat(16)}`; + const recordResult = table + .createRecord(new Map([[textFieldId.toString(), 'Old Title']])) + ._unsafeUnwrap(); + const foreignTable = Table.builder() + .withId(foreignTableId) + .withBaseId(baseId) + .withName(TableName.create('Foreign Records')._unsafeUnwrap()) + .field() + .singleLineText() + .withId(foreignPrimaryFieldId) + .withName(FieldName.create('Name')._unsafeUnwrap()) + .primary() + .done() + .view() + .defaultGrid() + .done() + .build() + ._unsafeUnwrap(); + + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table, foreignTable); + const tableQueryService = new TableQueryService(tableRepository); + + const recordRepository = new FakeTableRecordRepository(); + const recordQueryRepository = new FakeTableRecordQueryRepository(); + recordQueryRepository.record = { + id: recordResult.record.id().toString(), + fields: { + [textFieldId.toString()]: 'Old Title', + [linkFieldId.toString()]: null, + }, + version: 1, + }; + + const foreignTableLoader = new ForeignTableLoaderService(tableRepository); + const eventBus = new FakeEventBus(); + const unitOfWork = new FakeUnitOfWork(); + + const handler = new UpdateRecordHandler( + tableQueryService, + recordRepository, + recordQueryRepository, + new FakeRecordOrderCalculator(), + new FakeRecordMutationSpecResolverService() as unknown as RecordMutationSpecResolverService, + noopRecordChangedValueDecoratorService, + createRecordWritePluginRunner(), + new RecordWriteSideEffectService(), + noopRecordWriteUndoRedoPlanService, + createTableUpdateFlow(tableRepository, eventBus, unitOfWork), + eventBus, + new FakeUndoRedoService() as unknown as UndoRedoStackService, + unitOfWork, + foreignTableLoader + ); + + const command = UpdateRecordCommand.create({ + tableId: tableId.toString(), + recordId: recordResult.record.id().toString(), + fields: { [linkFieldId.toString()]: [{ id: targetRecordId }] }, + })._unsafeUnwrap(); + + const result = await handler.handle(createContext(), command); + + expect(result.isOk()).toBe(true); + expect(recordRepository.lastUpdateOptions?.fillLinkTitles).toBe(true); + expect( + recordRepository.lastUpdateOptions?.fillLinkTitleForeignTables?.get(table.id().toString()) + ).toBeUndefined(); + expect( + recordRepository.lastUpdateOptions?.fillLinkTitleForeignTables?.get(foreignTableId.toString()) + ).toBe(foreignTable); + }); + it('skips plugins that do not support updateOne', async () => { const { table, tableId, textFieldId } = buildTable(); const recordResult = table diff --git a/packages/v2/core/src/commands/UpdateRecordHandler.ts b/packages/v2/core/src/commands/UpdateRecordHandler.ts index f4f19d708a..59b4dac27b 100644 --- a/packages/v2/core/src/commands/UpdateRecordHandler.ts +++ b/packages/v2/core/src/commands/UpdateRecordHandler.ts @@ -258,20 +258,20 @@ export class UpdateRecordHandler tableEvents = tableFlowResult.events; } yield* await pluginExecution.beforePersist(transactionContext); - const fillLinkTitleForeignTables = command.typecast - ? yield* await handler.foreignTableLoaderService.loadForLinkTitleFill( - transactionContext, - [recordUpdateResult.mutateSpec ?? null] - ) - : new Map(); + const fillLinkTitleForeignTables = + yield* await handler.foreignTableLoaderService.loadForLinkTitleFill( + transactionContext, + [mutateSpec] + ); const mutation = yield* await handler.tableRecordRepository.updateOne( transactionContext, tableForUpdate, command.recordId, mutateSpec, { - ...(command.typecast ? { fillLinkTitles: true } : {}), - ...(fillLinkTitleForeignTables.size > 0 ? { fillLinkTitleForeignTables } : {}), + ...(fillLinkTitleForeignTables.size > 0 + ? { fillLinkTitles: true, fillLinkTitleForeignTables } + : {}), } ); diff --git a/packages/v2/core/src/commands/UpdateRecordsHandler.spec.ts b/packages/v2/core/src/commands/UpdateRecordsHandler.spec.ts index beb5fb0f67..f7a9d654cd 100644 --- a/packages/v2/core/src/commands/UpdateRecordsHandler.spec.ts +++ b/packages/v2/core/src/commands/UpdateRecordsHandler.spec.ts @@ -43,6 +43,7 @@ import type { IRecordOrderCalculator } from '../ports/RecordOrderCalculator'; import { RecordWriteOperationKind } from '../ports/RecordWritePlugin'; import type { IFindOptions } from '../ports/RepositoryQuery'; import type { ITableRecordQueryRepository } from '../ports/TableRecordQueryRepository'; +import type { ISpan, ITracer, SpanAttributes } from '../ports/Tracer'; import type { ITableRecordRepository, RecordMutationResult, @@ -67,6 +68,56 @@ const createContext = (): IExecutionContext => { return { actorId, windowId: 'test-window' }; }; +class FakeSpan implements ISpan { + ended = false; + readonly errors: string[] = []; + + constructor( + readonly name: string, + readonly attributes: Record = {} + ) {} + + setAttribute(key: string, value: string | number | boolean): void { + this.attributes[key] = value; + } + + setAttributes(attributes: SpanAttributes): void { + Object.assign(this.attributes, attributes); + } + + recordError(message: string): void { + this.errors.push(message); + } + + end(): void { + this.ended = true; + } +} + +class FakeTracer implements ITracer { + readonly spans: FakeSpan[] = []; + private readonly activeSpans: FakeSpan[] = []; + + startSpan(name: string, attributes?: SpanAttributes): ISpan { + const span = new FakeSpan(name, attributes ? { ...attributes } : {}); + this.spans.push(span); + return span; + } + + async withSpan(span: ISpan, callback: () => Promise): Promise { + this.activeSpans.push(span as FakeSpan); + try { + return await callback(); + } finally { + this.activeSpans.pop(); + } + } + + getActiveSpan(): ISpan | undefined { + return this.activeSpans[this.activeSpans.length - 1]; + } +} + const createTableUpdateFlow = ( tableRepository: FakeTableRepository, eventBus: FakeEventBus, @@ -420,13 +471,16 @@ class FakeRecordOrderCalculator implements IRecordOrderCalculator { class FakeEventBus implements IEventBus { published: IDomainEvent[] = []; + calls?: string[]; async publish(_: IExecutionContext, event: IDomainEvent) { + this.calls?.push('publish'); this.published.push(event); return ok(undefined); } async publishMany(_: IExecutionContext, events: ReadonlyArray) { + this.calls?.push('publishMany'); this.published.push(...events); return ok(undefined); } @@ -448,13 +502,16 @@ class FakeUnitOfWork implements IUnitOfWork { class FakeUndoRedoService { entries: unknown[] = []; + calls?: string[]; async recordEntry(_context: IExecutionContext, _tableId: TableId, entry: unknown) { + this.calls?.push('undoRedo.append'); this.entries.push(entry); return ok(undefined); } async appendEntry(_context: IExecutionContext, _tableId: TableId, entry: unknown) { + this.calls?.push('undoRedo.append'); this.entries.push(entry); return ok(undefined); } @@ -857,6 +914,113 @@ describe('UpdateRecordsHandler', () => { ]); }); + it('traces explicit bulk-update preparation phases', async () => { + const { table, tableId, textFieldId, numberFieldId } = buildTable(); + const recordIdA = `rec${'m'.repeat(16)}`; + const recordIdB = `rec${'n'.repeat(16)}`; + const tracer = new FakeTracer(); + + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table); + const recordRepository = new FakeTableRecordRepository(); + const queryRepository = new FakeTableRecordQueryRepository(); + queryRepository.records = [ + { + id: recordIdA, + version: 2, + fields: { + [numberFieldId.toString()]: 1, + [textFieldId.toString()]: 'before-a', + }, + }, + { + id: recordIdB, + version: 5, + fields: { + [numberFieldId.toString()]: 2, + [textFieldId.toString()]: 'before-b', + }, + }, + ]; + + const eventBus = new FakeEventBus(); + const unitOfWork = new FakeUnitOfWork(); + const undoRedoService = new FakeUndoRedoService(); + const handler = createHandler( + tableRepository, + recordRepository, + eventBus, + unitOfWork, + undoRedoService, + { + queryRepository, + } + ); + + const result = await handler.handle( + { ...createContext(), tracer }, + UpdateRecordsCommand.create({ + tableId: tableId.toString(), + fieldKeyType: 'id', + records: [ + { + id: recordIdA, + fields: { [numberFieldId.toString()]: 99 }, + }, + { + id: recordIdB, + fields: { [textFieldId.toString()]: 'after-b' }, + }, + ], + })._unsafeUnwrap() + ); + + expect(result.isOk()).toBe(true); + expect(tracer.spans.map((span) => span.name)).toEqual( + expect.arrayContaining([ + 'teable.RecordBulkUpdateService.resolveExplicitUpdates', + 'teable.RecordBulkUpdateService.pluginPrepare', + 'teable.RecordBulkUpdateService.loadExplicitCurrentRecords', + 'teable.RecordBulkUpdateService.prepareTableForWrite', + 'teable.RecordBulkUpdateService.createUpdateRecordsStream', + 'teable.RecordBulkUpdateService.generateUpdateBatch', + 'teable.RecordBulkUpdateService.generateUpdateBatch.updateRecord', + 'teable.RecordBulkUpdateService.generateUpdateBatch.updateRecord.resolveFieldKeys', + 'teable.RecordBulkUpdateService.generateUpdateBatch.updateRecord.configureMutationSpec', + 'teable.RecordBulkUpdateService.generateUpdateBatch.updateRecord.buildMutationSpec', + 'teable.RecordBulkUpdateService.generateUpdateBatch.updateRecord.mutateRecord', + 'teable.RecordBulkUpdateService.generateUpdateBatch.yieldFinalBatch', + 'teable.RecordBulkUpdateService.resolveUpdateBatch', + 'teable.RecordBulkUpdateService.materializeResolvedBatch', + 'teable.RecordBulkUpdateService.updateManyStream', + 'teable.RecordBulkUpdateService.buildPersistedEventData', + 'teable.RecordBulkUpdateService.materializeResultRecords', + ]) + ); + const updateManyStreamSpan = tracer.spans.find( + (span) => span.name === 'teable.RecordBulkUpdateService.updateManyStream' + ); + expect(updateManyStreamSpan?.attributes).toMatchObject({ + 'record.update.batchCount': 1, + 'record.update.maxBatchSize': 2, + 'record.update.authorizedRecordCount': 2, + }); + const generateBatchSpan = tracer.spans.find( + (span) => span.name === 'teable.RecordBulkUpdateService.generateUpdateBatch' + ); + expect(generateBatchSpan?.attributes).toMatchObject({ + 'record.update.generateBatchTotalRecords': 2, + 'record.update.generateBatchTotalFieldAssignments': 2, + 'record.update.generateBatchMaxFieldsPerRecord': 1, + }); + expect( + Object.keys(generateBatchSpan?.attributes ?? {}).some((key) => + key.startsWith('record.update.generateBatch.record.resolveFieldKeys.ms') + ) + ).toBe(true); + expect(tracer.spans.every((span) => span.ended)).toBe(true); + }); + it('uses repository-returned versions for explicit bulk-update events', async () => { const { table, tableId, numberFieldId } = buildTable(); const recordId = `rec${'w'.repeat(16)}`; @@ -906,6 +1070,55 @@ describe('UpdateRecordsHandler', () => { }); }); + it('records explicit bulk-update undo entry before publishing asynchronous events', async () => { + const { table, tableId, numberFieldId } = buildTable(); + const recordId = `rec${'o'.repeat(16)}`; + const calls: string[] = []; + + const tableRepository = new FakeTableRepository(); + tableRepository.tables.push(table); + const recordRepository = new FakeTableRecordRepository(); + const queryRepository = new FakeTableRecordQueryRepository(); + queryRepository.records = [ + { + id: recordId, + version: 3, + fields: { [numberFieldId.toString()]: 1 }, + }, + ]; + + const eventBus = new FakeEventBus(); + eventBus.calls = calls; + const unitOfWork = new FakeUnitOfWork(); + const undoRedoService = new FakeUndoRedoService(); + undoRedoService.calls = calls; + const handler = createHandler( + tableRepository, + recordRepository, + eventBus, + unitOfWork, + undoRedoService, + { + queryRepository, + } + ); + + const result = await handler.handle( + createContext(), + UpdateRecordsCommand.create({ + tableId: tableId.toString(), + fieldKeyType: 'id', + records: [{ id: recordId, fields: { [numberFieldId.toString()]: 99 } }], + })._unsafeUnwrap() + ); + + result._unsafeUnwrap(); + + expect(calls).toEqual(['undoRedo.append', 'publishMany']); + expect(eventBus.published.some(isRecordsBatchUpdatedEvent)).toBe(true); + expect(undoRedoService.entries).toHaveLength(1); + }); + it('does not publish explicit update events for rows skipped by storage as no-ops', async () => { const { table, tableId, numberFieldId } = buildTable(); const recordId = `rec${'x'.repeat(16)}`; diff --git a/packages/v2/core/src/commands/shared/orderBy.spec.ts b/packages/v2/core/src/commands/shared/orderBy.spec.ts index 23fb9a5ec5..2afa599d0f 100644 --- a/packages/v2/core/src/commands/shared/orderBy.spec.ts +++ b/packages/v2/core/src/commands/shared/orderBy.spec.ts @@ -72,7 +72,7 @@ describe('orderBy helpers', () => { `); }); - it('merges group + sort and appends auto number as the stable tie-breaker', () => { + it('merges group + sort and appends view row order + auto number as tie-breakers', () => { const groupBy = resolveGroupByToOrderBy([{ fieldId: fieldA, order: 'asc' }])._unsafeUnwrap(); const sortBy = resolveOrderBy([{ fieldId: fieldB, order: 'desc' }])._unsafeUnwrap(); const merged = mergeOrderBy(groupBy, sortBy, viewId); @@ -88,6 +88,11 @@ describe('orderBy helpers', () => { "fieldId": "fld0000000000000002", "type": "field", }, + { + "column": "__row_viw0000000000000001", + "direction": "asc", + "type": "column", + }, { "column": "__auto_number", "direction": "asc", @@ -97,7 +102,7 @@ describe('orderBy helpers', () => { `); }); - it('uses view row order only when no field/group sort is active', () => { + it('uses view row order + auto number for pure manual ordering', () => { const merged = mergeOrderBy(undefined, undefined, viewId); expect(serializeOrderBy(merged)).toMatchInlineSnapshot(` [ @@ -106,6 +111,11 @@ describe('orderBy helpers', () => { "direction": "asc", "type": "column", }, + { + "column": "__auto_number", + "direction": "asc", + "type": "column", + }, ] `); }); @@ -129,12 +139,14 @@ describe('orderBy helpers', () => { `); }); - it('deduplicates repeated fields and keeps auto number as the fallback when sorting exists', () => { + it('deduplicates repeated fields and tie-breaker columns', () => { const groupBy = resolveGroupByToOrderBy([{ fieldId: fieldA, order: 'asc' }])._unsafeUnwrap(); + // The signature only accepts field sorts; smuggle a column item past the + // type system to exercise tie-breaker dedup at runtime. const sortBy = [ ...resolveOrderBy([{ fieldId: fieldA, order: 'asc' }])._unsafeUnwrap()!, { column: `__row_${viewId}`, direction: 'asc' } as const, - ]; + ] as unknown as Parameters[1]; const merged = mergeOrderBy(groupBy, sortBy, viewId); expect(serializeOrderBy(merged)).toMatchInlineSnapshot(` [ @@ -157,7 +169,7 @@ describe('orderBy helpers', () => { `); }); - it('keeps view row order ahead of auto number for range-command tie breaking', () => { + it('uses the list-query tie breaker for range-command row offsets', () => { const sortBy = resolveOrderBy([{ fieldId: fieldB, order: 'desc' }])._unsafeUnwrap(); const merged = mergeOrderByWithViewRowTieBreaker(undefined, sortBy, viewId); expect(serializeOrderBy(merged)).toMatchInlineSnapshot(` diff --git a/packages/v2/core/src/commands/shared/orderBy.ts b/packages/v2/core/src/commands/shared/orderBy.ts index d11dda5a32..c50104743b 100644 --- a/packages/v2/core/src/commands/shared/orderBy.ts +++ b/packages/v2/core/src/commands/shared/orderBy.ts @@ -15,7 +15,7 @@ import type * as TableRecordQueryRepositoryPort from '../../ports/TableRecordQue * const groupBy = resolveGroupByToOrderBy(group).value; * const sortBy = resolveOrderBy(sort).value; * const orderBy = mergeOrderBy(groupBy, sortBy, 'viw0000000000000001'); - * // orderBy -> [{ fieldId: ... }, { fieldId: ... }, { column: '__row_viw...' }] + * // orderBy -> [{ fieldId: ... }, { fieldId: ... }, { column: '__row_viw...' }, { column: '__auto_number' }] * ``` */ export type SortLike = { @@ -59,9 +59,13 @@ export const resolveGroupByToOrderBy = ( /** * Merge groupBy + sort order into a single orderBy list. - * Always appends a stable tie-breaker for consistent pagination. - * Use view row order only when the caller did not request any field/group sorting; - * otherwise fall back to `__auto_number` to match v1's tie-breaking semantics. + * Always appends stable tie-breakers for consistent pagination. + * Tie-breakers must match the v1 read path that drives the grid's visible + * order (`buildFilterSortQuery` appends the view row order column after the + * requested sorts): field/group sorts first, then the view's manual row + * order, then auto number. Breaking ties by auto number alone makes + * offset-addressed range commands (paste/clear/delete) resolve different + * rows than the grid displays whenever the sort has duplicate values. */ export const mergeOrderBy = ( groupByOrderBy: ReadonlyArray | undefined, @@ -82,53 +86,26 @@ export const mergeOrderBy = ( groupByOrderBy?.forEach(pushUnique); sortOrderBy?.forEach(pushUnique); - const hasExplicitOrder = result.length > 0; - - // Use view row order only for pure manual row ordering. Once a field/group sort is active, - // v1 falls back to auto number within ties instead of reusing the view row column. - pushUnique({ - column: !hasExplicitOrder && viewId ? `__row_${viewId}` : '__auto_number', - direction: 'asc', - }); + // The repository falls back per-table to auto number when the view row + // column does not exist, so the view row tie-breaker is always safe to emit. + if (viewId) { + pushUnique({ column: `__row_${viewId}`, direction: 'asc' }); + } + pushUnique({ column: '__auto_number', direction: 'asc' }); return result.length > 0 ? result : undefined; }; /** * Merge groupBy + sort order for offset-targeted range commands. - * Range commands address visible rows, so ties must continue to respect the view row order - * before falling back to auto number for a final deterministic ordering. + * Range commands address the same visible rows returned by list queries, so they must use + * the same final tie-breaker as `mergeOrderBy`. Otherwise grouped views can display rows in + * one order while clear/paste/delete resolves the selected offset in another order. */ export const mergeOrderByWithViewRowTieBreaker = ( groupByOrderBy: ReadonlyArray | undefined, sortOrderBy: ReadonlyArray | undefined, viewId: string | undefined ): ReadonlyArray | undefined => { - const result: TableRecordQueryRepositoryPort.TableRecordOrderBy[] = []; - const seen = new Set(); - - const pushUnique = (item: TableRecordQueryRepositoryPort.TableRecordOrderBy) => { - const key = - 'fieldId' in item ? `field:${item.fieldId.toString()}` : `column:${String(item.column)}`; - if (seen.has(key)) return; - seen.add(key); - result.push(item); - }; - - groupByOrderBy?.forEach(pushUnique); - sortOrderBy?.forEach(pushUnique); - - if (viewId) { - pushUnique({ - column: `__row_${viewId}`, - direction: 'asc', - }); - } - - pushUnique({ - column: '__auto_number', - direction: 'asc', - }); - - return result.length > 0 ? result : undefined; + return mergeOrderBy(groupByOrderBy, sortOrderBy, viewId); }; diff --git a/packages/v2/core/src/commands/shared/recordWriteScope.spec.ts b/packages/v2/core/src/commands/shared/recordWriteScope.spec.ts index 042ab44791..c6cbdfb8b2 100644 --- a/packages/v2/core/src/commands/shared/recordWriteScope.spec.ts +++ b/packages/v2/core/src/commands/shared/recordWriteScope.spec.ts @@ -15,6 +15,7 @@ import { TableId } from '../../domain/table/TableId'; import { TableName } from '../../domain/table/TableName'; import type { TableRecordReadModel } from '../../ports/TableRecordReadModel'; import type { + ITableRecordQueryOptions, ITableRecordQueryRepository, ITableRecordQueryResult, } from '../../ports/TableRecordQueryRepository'; @@ -43,7 +44,7 @@ const buildTable = () => { }; class FakeTableRecordQueryRepository implements ITableRecordQueryRepository { - calls: Array<{ options?: { mode?: string; includeTotal?: boolean } }> = []; + calls: Array<{ options?: ITableRecordQueryOptions }> = []; constructor(private readonly result: Result) {} @@ -51,7 +52,7 @@ class FakeTableRecordQueryRepository implements ITableRecordQueryRepository { _context: IExecutionContext, _table: Table, _spec?: unknown, - options?: { mode?: string; includeTotal?: boolean } + options?: ITableRecordQueryOptions ): Promise> { this.calls.push({ options }); return this.result; @@ -121,7 +122,8 @@ describe('recordWriteScope', () => { requestedIds, RecordByIdsSpec.create(requestedIds), queryRepository, - 'deleteMany' + 'deleteMany', + { projectionFieldIds: [table.primaryFieldId()] } ); expect(result.isErr()).toBe(true); @@ -132,7 +134,15 @@ describe('recordWriteScope', () => { requestedRecordCount: 2, authorizedRecordCount: 1, }); - expect(queryRepository.calls).toEqual([{ options: { mode: 'stored', includeTotal: false } }]); + expect(queryRepository.calls).toEqual([ + { + options: { + mode: 'stored', + includeTotal: false, + projectionFieldIds: [table.primaryFieldId()], + }, + }, + ]); }); it('propagates query repository errors', async () => { diff --git a/packages/v2/core/src/commands/shared/recordWriteScope.ts b/packages/v2/core/src/commands/shared/recordWriteScope.ts index 59a67a1898..07e9b7a031 100644 --- a/packages/v2/core/src/commands/shared/recordWriteScope.ts +++ b/packages/v2/core/src/commands/shared/recordWriteScope.ts @@ -4,6 +4,7 @@ import type { Result } from 'neverthrow'; import { domainError, type DomainError } from '../../domain/shared/DomainError'; import { composeAndSpecsOrUndefined } from '../../domain/shared/specification/composeAndSpecs'; import type { ISpecification } from '../../domain/shared/specification/ISpecification'; +import type { FieldId } from '../../domain/table/fields/FieldId'; import type { RecordId } from '../../domain/table/records/RecordId'; import { RecordByIdsSpec } from '../../domain/table/records/specs/RecordByIdsSpec'; import type { ITableRecordConditionSpecVisitor } from '../../domain/table/records/specs/ITableRecordConditionSpecVisitor'; @@ -25,7 +26,10 @@ export const ensureRecordIdsWithinScope = async ( recordIds: ReadonlyArray, scopeSpec: RecordConditionSpec | undefined, queryRepository: ITableRecordQueryRepository, - operation: string + operation: string, + options?: { + projectionFieldIds?: ReadonlyArray; + } ): Promise> => { if (!scopeSpec || recordIds.length === 0) { return ok(undefined); @@ -39,6 +43,7 @@ export const ensureRecordIdsWithinScope = async ( const queryResult = await queryRepository.find(context, table, scopedSpec, { mode: 'stored', includeTotal: false, + projectionFieldIds: options?.projectionFieldIds, }); if (queryResult.isErr()) { return err(queryResult.error); diff --git a/packages/v2/core/src/di/registerCoreServices.ts b/packages/v2/core/src/di/registerCoreServices.ts index 541664b6e5..7b42af2471 100644 --- a/packages/v2/core/src/di/registerCoreServices.ts +++ b/packages/v2/core/src/di/registerCoreServices.ts @@ -50,6 +50,7 @@ import { UserValueResolverService } from '../application/services/UserValueResol import { ViewOperationPluginRunner } from '../application/services/ViewOperationPluginRunner'; import { PasteStreamApplicationService } from '../commands/PasteHandler'; import { NoopAttachmentUrlSignerService } from '../ports/defaults/NoopAttachmentUrlSignerService'; +import { NoopComputedFieldBackfillService } from '../ports/defaults/NoopComputedFieldBackfillService'; import { NoopRecordOrderCalculator } from '../ports/defaults/NoopRecordOrderCalculator'; import { NoopUndoRedoStore } from '../ports/defaults/NoopUndoRedoStore'; import type { IFieldOperationPlugin } from '../ports/FieldOperationPlugin'; @@ -257,6 +258,16 @@ export const registerV2CoreServices = ( }); } + if (!container.isRegistered(v2CoreTokens.computedFieldBackfillService)) { + container.register( + v2CoreTokens.computedFieldBackfillService, + NoopComputedFieldBackfillService, + { + lifecycle, + } + ); + } + // AttachmentValueResolverService - resolve attachment values if (!container.isRegistered(v2CoreTokens.attachmentValueResolverService)) { container.register( diff --git a/packages/v2/core/src/domain/table/Table.spec.ts b/packages/v2/core/src/domain/table/Table.spec.ts index 5103c53141..6743a4f96b 100644 --- a/packages/v2/core/src/domain/table/Table.spec.ts +++ b/packages/v2/core/src/domain/table/Table.spec.ts @@ -2104,11 +2104,17 @@ describe('Table.createRecordsStream', () => { (field) => field.id === duplicatedLookupFieldId ); expect(duplicatedLookupField?.isLookup).toBe(true); + // lookupOptions carries the parent (duplicated) link's stable physical join metadata. + const duplicatedSelfLinkOptions = + duplicatedSelfLinkField?.type === 'link' ? duplicatedSelfLinkField.options : undefined; expect(duplicatedLookupField?.lookupOptions).toEqual({ linkFieldId: duplicatedSelfLinkFieldId, foreignTableId: duplicatedTableId.toString(), lookupFieldId: duplicatedPrimaryFieldId, relationship: 'manyMany', + fkHostTableName: duplicatedSelfLinkOptions?.fkHostTableName, + selfKeyName: duplicatedSelfLinkOptions?.selfKeyName, + foreignKeyName: duplicatedSelfLinkOptions?.foreignKeyName, }); const duplicatedButtonField = duplicatedDto.fields.find( diff --git a/packages/v2/core/src/domain/table/Table.ts b/packages/v2/core/src/domain/table/Table.ts index 38692229ba..68c1f34799 100644 --- a/packages/v2/core/src/domain/table/Table.ts +++ b/packages/v2/core/src/domain/table/Table.ts @@ -61,6 +61,8 @@ import { type CreateRecordsMethodResult, type CreateRecordsStreamOptions, type UpdateRecordItem, + type UpdateRecordOptions, + type UpdateRecordsStreamOptions, } from './methods/records'; import { rename as renameMethod } from './methods/rename'; import { validateFormSubmission as validateFormSubmissionMethod } from './methods/validateFormSubmission'; @@ -537,7 +539,7 @@ export class Table extends AggregateRoot { updateRecord( recordId: RecordId, fieldValues: ReadonlyMap, - options?: { typecast?: boolean } + options?: UpdateRecordOptions ): Result { return updateRecordMethod.call(this, recordId, fieldValues, options); } @@ -555,6 +557,7 @@ export class Table extends AggregateRoot { * @param options - Optional configuration * @param options.typecast - If true, values are converted to the expected type * @param options.batchSize - Number of records per batch (default: 500) + * @param options.maxBatchSize - Dynamic batch-size cap when batchSize is not specified * @returns Generator yielding Result batches of RecordUpdateResult * * @example @@ -577,7 +580,7 @@ export class Table extends AggregateRoot { */ *updateRecordsStream( updates: Iterable, - options?: { typecast?: boolean; batchSize?: number } + options?: UpdateRecordsStreamOptions ): Generator, DomainError>> { yield* updateRecordsStreamMethod.call(this, updates, options); } @@ -967,6 +970,7 @@ export class Table extends AggregateRoot { options: mergedOptions, defaultValue: (field as SingleSelectField).defaultValue(), preventAutoNewOptions: (field as SingleSelectField).preventAutoNewOptions(), + domainContext, }) : MultipleSelectField.create({ id: field.id(), @@ -974,6 +978,7 @@ export class Table extends AggregateRoot { options: mergedOptions, defaultValue: (field as MultipleSelectField).defaultValue(), preventAutoNewOptions: (field as MultipleSelectField).preventAutoNewOptions(), + domainContext, }); if (nextFieldResult.isErr()) return err(nextFieldResult.error); const nextField = nextFieldResult.value; diff --git a/packages/v2/core/src/domain/table/fields/FieldFactory.ts b/packages/v2/core/src/domain/table/fields/FieldFactory.ts index 4ca744b878..3a14c098bb 100644 --- a/packages/v2/core/src/domain/table/fields/FieldFactory.ts +++ b/packages/v2/core/src/domain/table/fields/FieldFactory.ts @@ -2,6 +2,7 @@ import { ok } from 'neverthrow'; import type { Result } from 'neverthrow'; import type { BaseId } from '../../base/BaseId'; +import type { IDomainContext } from '../../shared/DomainContext'; import type { DomainError } from '../../shared/DomainError'; import type { DbTableName } from '../DbTableName'; import type { TableId } from '../TableId'; @@ -193,6 +194,7 @@ export const createSelectField = (params: { options: ReadonlyArray; defaultValue?: SelectDefaultValue; preventAutoNewOptions?: SelectAutoNewOptions; + domainContext?: IDomainContext; notNull?: FieldNotNull; unique?: FieldUnique; }): Result => @@ -206,6 +208,7 @@ export const createMultipleSelectField = (params: { options: ReadonlyArray; defaultValue?: SelectDefaultValue; preventAutoNewOptions?: SelectAutoNewOptions; + domainContext?: IDomainContext; notNull?: FieldNotNull; unique?: FieldUnique; }): Result => diff --git a/packages/v2/core/src/domain/table/fields/types/FieldColor.ts b/packages/v2/core/src/domain/table/fields/types/FieldColor.ts index c8e492d993..d30dc3588b 100644 --- a/packages/v2/core/src/domain/table/fields/types/FieldColor.ts +++ b/packages/v2/core/src/domain/table/fields/types/FieldColor.ts @@ -60,6 +60,7 @@ export const fieldColorValues = [ export const fieldColorSchema = z.enum(fieldColorValues); export type FieldColorValue = z.infer; +const fieldColorSet: ReadonlySet = new Set(fieldColorValues); export class FieldColor extends ValueObject { private constructor(private readonly value: FieldColorValue) { @@ -67,6 +68,10 @@ export class FieldColor extends ValueObject { } static create(raw: unknown): Result { + if (typeof raw === 'string' && fieldColorSet.has(raw)) { + return ok(new FieldColor(raw as FieldColorValue)); + } + const parsed = fieldColorSchema.safeParse(raw); if (!parsed.success) return err(domainError.validation({ message: 'Invalid FieldColor' })); return ok(new FieldColor(parsed.data)); diff --git a/packages/v2/core/src/domain/table/fields/types/FieldValueObjects.spec.ts b/packages/v2/core/src/domain/table/fields/types/FieldValueObjects.spec.ts index 9835c17e4e..79c87af147 100644 --- a/packages/v2/core/src/domain/table/fields/types/FieldValueObjects.spec.ts +++ b/packages/v2/core/src/domain/table/fields/types/FieldValueObjects.spec.ts @@ -65,6 +65,16 @@ describe('SelectOption', () => { expect(dto.color).toBe('blue'); }); + it('treats an empty option id as omitted', () => { + const result = SelectOption.create({ id: '', name: 'Todo', color: 'blue' }); + expect(result._unsafeUnwrap().toDto().id).toMatch(/^cho/); + }); + + it('rejects invalid option objects', () => { + SelectOption.create({ id: null, name: 'Todo', color: 'blue' })._unsafeUnwrapErr(); + SelectOption.create({ name: 'Todo', color: 'invalid' })._unsafeUnwrapErr(); + }); + it('compares options by value', () => { const one = SelectOption.create({ id: 'cho12345678', name: 'Todo', color: 'blue' }); const two = SelectOption.create({ id: 'cho12345678', name: 'Todo', color: 'blue' }); @@ -156,6 +166,11 @@ describe('FieldUnique', () => { }); describe('SelectOptions', () => { + const createSelectOptions = (count: number) => + Array.from({ length: count }, (_, index) => + SelectOption.create({ name: `Option ${index + 1}`, color: 'blue' })._unsafeUnwrap() + ); + it('validates uniqueness and default values', () => { const optionOne = SelectOption.create({ name: 'Todo', color: 'blue' }); const optionTwo = SelectOption.create({ name: 'Done', color: 'green' }); @@ -183,6 +198,34 @@ describe('SelectOptions', () => { ); invalidResult._unsafeUnwrapErr(); }); + + it('uses runtime table limits when validating option count and names', () => { + const options = createSelectOptions(1001); + + const withoutRuntimeLimit = validateSelectOptions(options, undefined, 'single', { + domainContext: { config: { tableLimits: { fieldOptions: { maxSelectChoices: 1000 } } } }, + }); + expect(withoutRuntimeLimit.isErr()).toBe(true); + expect(withoutRuntimeLimit._unsafeUnwrapErr().code).toBe( + 'validation.field.select_options_limit' + ); + + const withRuntimeLimit = validateSelectOptions(options, undefined, 'single', { + domainContext: { config: { tableLimits: { fieldOptions: { maxSelectChoices: 1001 } } } }, + }); + expect(withRuntimeLimit.isOk()).toBe(true); + + const longName = SelectOption.create({ name: 'Long', color: 'blue' })._unsafeUnwrap(); + const nameLimit = validateSelectOptions([longName], undefined, 'single', { + domainContext: { + config: { tableLimits: { fieldOptions: { maxSelectChoiceNameLength: 3 } } }, + }, + }); + expect(nameLimit.isErr()).toBe(true); + expect(nameLimit._unsafeUnwrapErr().code).toBe( + 'validation.limit.select_choice_name_max_length' + ); + }); }); describe('FieldColor', () => { diff --git a/packages/v2/core/src/domain/table/fields/types/FormulaField.ts b/packages/v2/core/src/domain/table/fields/types/FormulaField.ts index 22c4ea2d4e..2cc86ff993 100644 --- a/packages/v2/core/src/domain/table/fields/types/FormulaField.ts +++ b/packages/v2/core/src/domain/table/fields/types/FormulaField.ts @@ -10,6 +10,7 @@ import type { OnTeableFieldDeleted, } from '../../OnTeableFieldDeleted'; import { UpdateFormulaExpressionSpec } from '../../specs/field-updates/UpdateFormulaExpressionSpec'; +import { UpdateUserMultiplicitySpec } from '../../specs/field-updates/UpdateUserMultiplicitySpec'; import type { ITableSpecVisitor } from '../../specs/ITableSpecVisitor'; import { TableUpdateFieldHasErrorSpec } from '../../specs/TableUpdateFieldHasErrorSpec'; import { TableUpdateFieldTypeSpec } from '../../specs/TableUpdateFieldTypeSpec'; @@ -271,12 +272,21 @@ export class FormulaField extends Field implements OnTeableFieldUpdated, OnTeabl spec.isTypeConversion() ); + const dependencyMultiplicityChanged = updateSpecs.some( + (spec): spec is UpdateUserMultiplicitySpec => + spec instanceof UpdateUserMultiplicitySpec && spec.fieldId().equals(updatedField.id()) + ); + const dependencyFormulaExpressionChanged = updateSpecs.some( (spec): spec is UpdateFormulaExpressionSpec => spec instanceof UpdateFormulaExpressionSpec && spec.fieldId().equals(updatedField.id()) ); - if (!dependencyTypeChanged && !dependencyFormulaExpressionChanged) { + if ( + !dependencyTypeChanged && + !dependencyMultiplicityChanged && + !dependencyFormulaExpressionChanged + ) { return ok(undefined); } diff --git a/packages/v2/core/src/domain/table/fields/types/MultipleSelectField.ts b/packages/v2/core/src/domain/table/fields/types/MultipleSelectField.ts index 17894a0446..84fdb7d64b 100644 --- a/packages/v2/core/src/domain/table/fields/types/MultipleSelectField.ts +++ b/packages/v2/core/src/domain/table/fields/types/MultipleSelectField.ts @@ -1,5 +1,6 @@ import type { Result } from 'neverthrow'; +import type { IDomainContext } from '../../../shared/DomainContext'; import type { DomainError } from '../../../shared/DomainError'; import { Field } from '../Field'; import type { FieldDuplicateParams } from '../Field'; @@ -10,7 +11,7 @@ import type { IFieldVisitor } from '../visitors/IFieldVisitor'; import { SelectAutoNewOptions } from './SelectAutoNewOptions'; import type { SelectDefaultValue } from './SelectDefaultValue'; import type { SelectOption } from './SelectOption'; -import { validateSelectOptions } from './SelectOptions'; +import { validateSelectOptions, type SelectOptionsValidationContext } from './SelectOptions'; export class MultipleSelectField extends Field { private constructor( @@ -29,8 +30,17 @@ export class MultipleSelectField extends Field { options: ReadonlyArray; defaultValue?: SelectDefaultValue; preventAutoNewOptions?: SelectAutoNewOptions; + domainContext?: IDomainContext; }): Result { - return validateSelectOptions(params.options, params.defaultValue, 'multiple').map( + const validationContext: SelectOptionsValidationContext = { + domainContext: params.domainContext, + }; + return validateSelectOptions( + params.options, + params.defaultValue, + 'multiple', + validationContext + ).map( (options) => new MultipleSelectField( params.id, diff --git a/packages/v2/core/src/domain/table/fields/types/SelectOption.ts b/packages/v2/core/src/domain/table/fields/types/SelectOption.ts index c286b7e410..bffc062d69 100644 --- a/packages/v2/core/src/domain/table/fields/types/SelectOption.ts +++ b/packages/v2/core/src/domain/table/fields/types/SelectOption.ts @@ -24,6 +24,22 @@ export class SelectOption extends ValueObject { } static create(raw: unknown): Result { + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + const option = raw as { id?: unknown; name?: unknown; color?: unknown }; + if ( + (option.id === undefined || typeof option.id === 'string') && + typeof option.name === 'string' && + typeof option.color === 'string' + ) { + return SelectOptionName.create(option.name).andThen((name) => { + const idResult = option.id ? SelectOptionId.create(option.id) : SelectOptionId.generate(); + return idResult.andThen((id) => + FieldColor.create(option.color).map((color) => new SelectOption(id, name, color)) + ); + }); + } + } + const parsed = selectOptionSchema.safeParse(raw); if (!parsed.success) return err(domainError.validation({ message: 'Invalid SelectOption' })); diff --git a/packages/v2/core/src/domain/table/fields/types/SelectOptionId.ts b/packages/v2/core/src/domain/table/fields/types/SelectOptionId.ts index 21619e9225..66d4cd00d1 100644 --- a/packages/v2/core/src/domain/table/fields/types/SelectOptionId.ts +++ b/packages/v2/core/src/domain/table/fields/types/SelectOptionId.ts @@ -17,6 +17,10 @@ export class SelectOptionId extends ValueObject { } static create(raw: unknown): Result { + if (typeof raw === 'string' && raw.length > 0) { + return ok(new SelectOptionId(raw)); + } + const parsed = selectOptionIdSchema.safeParse(raw); if (!parsed.success) return err(domainError.validation({ message: 'Invalid SelectOptionId' })); return ok(new SelectOptionId(parsed.data)); diff --git a/packages/v2/core/src/domain/table/fields/types/SelectOptionName.ts b/packages/v2/core/src/domain/table/fields/types/SelectOptionName.ts index acd40f0e06..e48bd7c323 100644 --- a/packages/v2/core/src/domain/table/fields/types/SelectOptionName.ts +++ b/packages/v2/core/src/domain/table/fields/types/SelectOptionName.ts @@ -13,6 +13,11 @@ export class SelectOptionName extends ValueObject { } static create(raw: unknown): Result { + if (typeof raw === 'string') { + const value = raw.trim(); + if (value.length > 0) return ok(new SelectOptionName(value)); + } + const parsed = selectOptionNameSchema.safeParse(raw); if (!parsed.success) { const issues = parsed.error.issues.map((i) => i.message).join(', '); diff --git a/packages/v2/core/src/domain/table/fields/types/SelectOptions.ts b/packages/v2/core/src/domain/table/fields/types/SelectOptions.ts index 0bf66c0e52..eb16a69ae9 100644 --- a/packages/v2/core/src/domain/table/fields/types/SelectOptions.ts +++ b/packages/v2/core/src/domain/table/fields/types/SelectOptions.ts @@ -1,26 +1,47 @@ import { err, ok } from 'neverthrow'; import type { Result } from 'neverthrow'; +import type { IDomainContext } from '../../../shared/DomainContext'; import { domainError, type DomainError } from '../../../shared/DomainError'; import type { SelectDefaultValue } from './SelectDefaultValue'; +import { + ensureSelectFieldOptionCountWithinLimit, + ensureSelectFieldOptionNameWithinLimit, +} from './SelectFieldOptionWriteConfig'; import type { SelectOption } from './SelectOption'; -const isUniqueByStringValue = (values: ReadonlyArray<{ toString(): string }>): boolean => { - const seen = new Set(); - for (const value of values) { - const key = value.toString(); - if (seen.has(key)) return false; - seen.add(key); - } - return true; +export type SelectOptionsValidationContext = { + domainContext?: IDomainContext; }; export const validateSelectOptions = ( options: ReadonlyArray, defaultValue?: SelectDefaultValue, - mode: 'single' | 'multiple' = 'single' + mode: 'single' | 'multiple' = 'single', + context?: SelectOptionsValidationContext ): Result, DomainError> => { - if (!isUniqueByStringValue(options.map((option) => option.name()))) + const domainContext = context?.domainContext; + if (domainContext) { + const countResult = ensureSelectFieldOptionCountWithinLimit(options.length, domainContext); + if (countResult.isErr()) return err(countResult.error); + } + + const names = new Set(); + let hasDuplicateName = false; + for (const option of options) { + const name = option.name().toString(); + if (domainContext) { + const nameLimitResult = ensureSelectFieldOptionNameWithinLimit(name.length, domainContext); + if (nameLimitResult.isErr()) return err(nameLimitResult.error); + } + if (names.has(name)) { + hasDuplicateName = true; + } else { + names.add(name); + } + } + + if (hasDuplicateName) return err(domainError.conflict({ message: 'SelectField options must be unique' })); if (defaultValue) { @@ -29,7 +50,6 @@ export const validateSelectOptions = ( domainError.validation({ message: 'SelectField defaultValue must be a single option' }) ); - const names = new Set(options.map((option) => option.name().toString())); const defaults = defaultValue.toDto(); const values = Array.isArray(defaults) ? defaults : [defaults]; for (const value of values) { diff --git a/packages/v2/core/src/domain/table/fields/types/SingleSelectField.ts b/packages/v2/core/src/domain/table/fields/types/SingleSelectField.ts index 91111d9af4..21e9169b02 100644 --- a/packages/v2/core/src/domain/table/fields/types/SingleSelectField.ts +++ b/packages/v2/core/src/domain/table/fields/types/SingleSelectField.ts @@ -1,5 +1,6 @@ import type { Result } from 'neverthrow'; +import type { IDomainContext } from '../../../shared/DomainContext'; import type { DomainError } from '../../../shared/DomainError'; import { Field } from '../Field'; import type { FieldDuplicateParams } from '../Field'; @@ -10,7 +11,7 @@ import type { IFieldVisitor } from '../visitors/IFieldVisitor'; import { SelectAutoNewOptions } from './SelectAutoNewOptions'; import type { SelectDefaultValue } from './SelectDefaultValue'; import type { SelectOption } from './SelectOption'; -import { validateSelectOptions } from './SelectOptions'; +import { validateSelectOptions, type SelectOptionsValidationContext } from './SelectOptions'; export class SingleSelectField extends Field { private constructor( @@ -29,8 +30,17 @@ export class SingleSelectField extends Field { options: ReadonlyArray; defaultValue?: SelectDefaultValue; preventAutoNewOptions?: SelectAutoNewOptions; + domainContext?: IDomainContext; }): Result { - return validateSelectOptions(params.options, params.defaultValue, 'single').map( + const validationContext: SelectOptionsValidationContext = { + domainContext: params.domainContext, + }; + return validateSelectOptions( + params.options, + params.defaultValue, + 'single', + validationContext + ).map( (options) => new SingleSelectField( params.id, diff --git a/packages/v2/core/src/domain/table/fields/visitors/FieldCellValueSchemaVisitor.ts b/packages/v2/core/src/domain/table/fields/visitors/FieldCellValueSchemaVisitor.ts index d5b42e624a..b073d71620 100644 --- a/packages/v2/core/src/domain/table/fields/visitors/FieldCellValueSchemaVisitor.ts +++ b/packages/v2/core/src/domain/table/fields/visitors/FieldCellValueSchemaVisitor.ts @@ -29,7 +29,8 @@ import { AbstractFieldVisitor } from './AbstractFieldVisitor'; import { parseDateValue } from './dateValueParser'; // Zod schema types -type ZodSchema = z.ZodTypeAny; +export type FieldCellValueSchema = z.ZodTypeAny; +type ZodSchema = FieldCellValueSchema; // Attachment item schema const attachmentItemSchema = z.object({ diff --git a/packages/v2/core/src/domain/table/fields/visitors/SetFieldValueSpecFactoryVisitor.ts b/packages/v2/core/src/domain/table/fields/visitors/SetFieldValueSpecFactoryVisitor.ts index f8c6794f3a..c83f84e7c0 100644 --- a/packages/v2/core/src/domain/table/fields/visitors/SetFieldValueSpecFactoryVisitor.ts +++ b/packages/v2/core/src/domain/table/fields/visitors/SetFieldValueSpecFactoryVisitor.ts @@ -177,7 +177,7 @@ export class SetFieldValueSpecFactoryVisitor extends AbstractFieldVisitor { const cellValue = CellValue.fromValidated(this.value as LinkItem[] | null); - return ok(new SetLinkValueSpec(field.id(), cellValue)); + return ok(new SetLinkValueSpec(field.id(), cellValue, field.foreignTableId())); } visitConditionalRollupField(_field: ConditionalRollupField): Result { diff --git a/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.spec.ts b/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.spec.ts new file mode 100644 index 0000000000..cb30177515 --- /dev/null +++ b/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.spec.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; + +import { calculateBatchSize } from './calculateBatchSize'; + +describe('calculateBatchSize', () => { + it('keeps the default stream cap at 500 rows', () => { + expect(calculateBatchSize(20)).toBe(500); + }); + + it('allows 1k-row batches for narrow mixed-field explicit updates', () => { + expect(calculateBatchSize(20, { maxBatchSize: 1000 })).toBe(1000); + }); + + it('scales down wide-table batches with per-row SQL overhead', () => { + expect(calculateBatchSize(100, { maxBatchSize: 1000 })).toBe(555); + expect(calculateBatchSize(200)).toBe(288); + }); + + it('clamps explicit batch sizes to the supported range', () => { + expect(calculateBatchSize(20, { userBatchSize: 10_000, maxBatchSize: 1000 })).toBe(1000); + expect(calculateBatchSize(20, 1)).toBe(100); + }); +}); diff --git a/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.ts b/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.ts index 3212ea2973..15c9b7352b 100644 --- a/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.ts +++ b/packages/v2/core/src/domain/table/methods/records/calculateBatchSize.ts @@ -10,9 +10,20 @@ const PG_PARAM_SAFE_LIMIT = 60000; const MIN_BATCH_SIZE = 100; /** - * Maximum batch size as a reasonable upper bound. + * Maximum batch size as a reasonable default upper bound. */ -const MAX_BATCH_SIZE = 500; +const DEFAULT_MAX_BATCH_SIZE = 500; + +/** + * Batch UPDATE SQL carries per-row overhead beyond user fields: record id, + * presence markers, and system metadata columns. + */ +const PER_ROW_SQL_OVERHEAD = 8; + +export interface CalculateBatchSizeOptions { + readonly userBatchSize?: number; + readonly maxBatchSize?: number; +} /** * Calculates an optimal batch size based on the number of fields. @@ -22,7 +33,7 @@ const MAX_BATCH_SIZE = 500; * 2. Maintain reasonable memory usage * 3. Balance between too many round trips and too large transactions * - * Formula: min(500, max(100, floor(60000 / fieldCount))) + * Formula: min(maxBatchSize, max(100, floor(60000 / (fieldCount + overhead)))) * * @param fieldCount - The number of fields in the table * @param userBatchSize - Optional user-specified batch size (takes precedence if provided) @@ -30,33 +41,45 @@ const MAX_BATCH_SIZE = 500; * * @example * ```typescript - * // Table with 10 fields → batch size 500 (capped at max) + * // Table with 10 fields -> batch size 500 (capped at default max) * calculateBatchSize(10) // returns 500 * - * // Table with 100 fields → batch size 500 (60000/100 = 600, capped at 500) + * // Narrow explicit bulk update can opt into a larger cap + * calculateBatchSize(10, { maxBatchSize: 1000 }) // returns 1000 + * + * // Table with 100 fields -> batch size 500 (60000/(100+8), capped at default max) * calculateBatchSize(100) // returns 500 * - * // Table with 200 fields → batch size 300 (60000/200 = 300) - * calculateBatchSize(200) // returns 300 + * // Table with 200 fields -> batch size 288 (60000/(200+8)) + * calculateBatchSize(200) // returns 288 * * // Table with 1000 fields → batch size 100 (capped at min) * calculateBatchSize(1000) // returns 100 * - * // User-specified batch size takes precedence + * // User-specified batch size takes precedence within the configured max * calculateBatchSize(10, 250) // returns 250 * ``` */ -export function calculateBatchSize(fieldCount: number, userBatchSize?: number): number { +export function calculateBatchSize( + fieldCount: number, + userBatchSizeOrOptions?: number | CalculateBatchSizeOptions +): number { + const options = + typeof userBatchSizeOrOptions === 'number' + ? { userBatchSize: userBatchSizeOrOptions } + : userBatchSizeOrOptions ?? {}; + const maxBatchSize = options.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE; + // If user explicitly specified a batch size, use it (but respect min/max bounds) - if (userBatchSize !== undefined) { - return Math.max(MIN_BATCH_SIZE, Math.min(MAX_BATCH_SIZE, userBatchSize)); + if (options.userBatchSize !== undefined) { + return Math.max(MIN_BATCH_SIZE, Math.min(maxBatchSize, options.userBatchSize)); } // Avoid division by zero if (fieldCount <= 0) { - return MAX_BATCH_SIZE; + return maxBatchSize; } - const dynamicSize = Math.floor(PG_PARAM_SAFE_LIMIT / fieldCount); - return Math.max(MIN_BATCH_SIZE, Math.min(MAX_BATCH_SIZE, dynamicSize)); + const dynamicSize = Math.floor(PG_PARAM_SAFE_LIMIT / (fieldCount + PER_ROW_SQL_OVERHEAD)); + return Math.max(MIN_BATCH_SIZE, Math.min(maxBatchSize, dynamicSize)); } diff --git a/packages/v2/core/src/domain/table/methods/records/index.ts b/packages/v2/core/src/domain/table/methods/records/index.ts index a374740c87..489c9aaf9e 100644 --- a/packages/v2/core/src/domain/table/methods/records/index.ts +++ b/packages/v2/core/src/domain/table/methods/records/index.ts @@ -2,5 +2,20 @@ export { createRecord } from './createRecord'; export { createRecords, type CreateRecordsMethodResult } from './createRecords'; export { createRecordsStream, type CreateRecordsStreamOptions } from './createRecordsStream'; export { createRecordsStreamAsync } from './createRecordsStreamAsync'; -export { updateRecord } from './updateRecord'; -export { updateRecordsStream, type UpdateRecordItem } from './updateRecordsStream'; +export { + createUpdateRecordBuildContext, + updateRecord, + type UpdateRecordBuildContext, + type UpdateRecordOptions, + type UpdateRecordTraceEvent, + type UpdateRecordTraceHook, + type UpdateRecordTracePhase, +} from './updateRecord'; +export { + updateRecordsStream, + type UpdateRecordItem, + type UpdateRecordsStreamOptions, + type UpdateRecordsStreamTraceEvent, + type UpdateRecordsStreamTraceHook, + type UpdateRecordsStreamTracePhase, +} from './updateRecordsStream'; diff --git a/packages/v2/core/src/domain/table/methods/records/updateRecord.ts b/packages/v2/core/src/domain/table/methods/records/updateRecord.ts index 2f89f1bb36..8fb542d9b1 100644 --- a/packages/v2/core/src/domain/table/methods/records/updateRecord.ts +++ b/packages/v2/core/src/domain/table/methods/records/updateRecord.ts @@ -1,7 +1,12 @@ import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; import { domainError, type DomainError } from '../../../shared/DomainError'; +import type { Field } from '../../fields/Field'; import { FieldByKeySpec } from '../../fields/specs/FieldByKeySpec'; +import { + FieldCellValueSchemaVisitor, + type FieldCellValueSchema, +} from '../../fields/visitors/FieldCellValueSchemaVisitor'; import type { FieldKeyMapping } from '../../records/RecordCreateResult'; import type { RecordId } from '../../records/RecordId'; import { RecordMutationSpecBuilder } from '../../records/RecordMutationSpecBuilder'; @@ -10,64 +15,187 @@ import { TableRecord } from '../../records/TableRecord'; import { TableRecordFields } from '../../records/TableRecordFields'; import type { Table } from '../../Table'; +export type UpdateRecordTracePhase = + | 'resolveFieldKeys' + | 'configureMutationSpec' + | 'buildMutationSpec' + | 'createEmptyRecord' + | 'mutateRecord' + | 'createUpdateResult'; + +export interface UpdateRecordTraceEvent { + readonly phase: UpdateRecordTracePhase; + readonly recordIndex?: number; + readonly fieldCount: number; + readonly editableFieldCount?: number; +} + +export type UpdateRecordTraceHook = ( + event: UpdateRecordTraceEvent, + callback: () => Result +) => Result; + +export interface UpdateRecordOptions { + readonly typecast?: boolean; + readonly trace?: UpdateRecordTraceHook; + readonly recordIndex?: number; + readonly buildContext?: UpdateRecordBuildContext; +} + +export interface UpdateRecordBuildContext { + readonly fieldByKey: ReadonlyMap; + readonly editableFieldById: ReadonlyMap; + readonly editableFieldSchemaById: ReadonlyMap; + readonly editableFieldCount: number; +} + +export function createUpdateRecordBuildContext(table: Table): UpdateRecordBuildContext { + const fieldByKey = new Map(); + const editableFieldById = new Map(); + const editableFieldSchemaById = new Map(); + const schemaVisitor = FieldCellValueSchemaVisitor.create(); + + for (const field of table.getFields()) { + const fieldId = field.id().toString(); + const fieldName = field.name().toString(); + if (!fieldByKey.has(fieldId)) { + fieldByKey.set(fieldId, field); + } + if (!fieldByKey.has(fieldName)) { + fieldByKey.set(fieldName, field); + } + } + + for (const field of table.getEditableFields()) { + const fieldId = field.id().toString(); + editableFieldById.set(fieldId, field); + const schemaResult = field.accept(schemaVisitor); + if (schemaResult.isOk()) { + editableFieldSchemaById.set(fieldId, schemaResult.value); + } + } + + return { + fieldByKey, + editableFieldById, + editableFieldSchemaById, + editableFieldCount: editableFieldById.size, + }; +} + export function updateRecord( this: Table, recordId: RecordId, fieldValues: ReadonlyMap, - options?: { typecast?: boolean } + options?: UpdateRecordOptions ): Result { const table = this; - const { typecast = false } = options ?? {}; + const { typecast = false, trace } = options ?? {}; + const recordIndex = options?.recordIndex; + const buildContext = options?.buildContext; + const runTrace = ( + phase: UpdateRecordTracePhase, + callback: () => Result, + extra?: Pick + ): Result => { + const event: UpdateRecordTraceEvent = { + phase, + fieldCount: fieldValues.size, + ...(recordIndex != null ? { recordIndex } : {}), + ...extra, + }; + return trace ? trace(event, callback) : callback(); + }; + return safeTry(function* () { // Resolve field keys to actual fields and build fieldKeyMapping - const fieldKeyMapping: FieldKeyMapping = new Map(); - const resolvedFieldValues = new Map(); - - for (const [key, value] of fieldValues.entries()) { - const spec = FieldByKeySpec.create(key); - const fieldResult = table.getField(spec); - - if (fieldResult.isErr()) { - return err( - domainError.notFound({ - code: 'field.not_found', - message: `Field not found: ${key}`, - }) - ); - } - - const field = fieldResult.value; - const fieldIdStr = field.id().toString(); - resolvedFieldValues.set(fieldIdStr, value); - fieldKeyMapping.set(fieldIdStr, key); - } + const resolvedFields = yield* runTrace('resolveFieldKeys', () => + safeTry<{ + fieldKeyMapping: FieldKeyMapping; + resolvedFieldValues: Map; + }, DomainError>(function* () { + const fieldKeyMapping: FieldKeyMapping = new Map(); + const resolvedFieldValues = new Map(); + + for (const [key, value] of fieldValues.entries()) { + const cachedField = buildContext?.fieldByKey.get(key); + const fieldResult = cachedField + ? ok(cachedField) + : table.getField(FieldByKeySpec.create(key)); + + if (fieldResult.isErr()) { + return err( + domainError.notFound({ + code: 'field.not_found', + message: `Field not found: ${key}`, + }) + ); + } + + const field = fieldResult.value; + const fieldIdStr = field.id().toString(); + resolvedFieldValues.set(fieldIdStr, value); + fieldKeyMapping.set(fieldIdStr, key); + } + + return ok({ fieldKeyMapping, resolvedFieldValues }); + }) + ); + const { fieldKeyMapping, resolvedFieldValues } = resolvedFields; const builder = RecordMutationSpecBuilder.create().withTypecast(typecast); - const fields = table.getEditableFields(); - - for (const field of fields) { - const fieldIdStr = field.id().toString(); - if (!resolvedFieldValues.has(fieldIdStr)) continue; - const providedValue = resolvedFieldValues.get(fieldIdStr); - if (providedValue === undefined) continue; - builder.set(field, providedValue); - } + const editableFields = buildContext ? undefined : table.getEditableFields(); + + yield* runTrace( + 'configureMutationSpec', + () => { + if (buildContext) { + for (const [fieldIdStr, providedValue] of resolvedFieldValues.entries()) { + const field = buildContext.editableFieldById.get(fieldIdStr); + if (!field) continue; + if (providedValue === undefined) continue; + builder.setWithSchema( + field, + providedValue, + buildContext.editableFieldSchemaById.get(fieldIdStr) + ); + } + return ok(undefined); + } + + for (const field of editableFields ?? []) { + const fieldIdStr = field.id().toString(); + if (!resolvedFieldValues.has(fieldIdStr)) continue; + const providedValue = resolvedFieldValues.get(fieldIdStr); + if (providedValue === undefined) continue; + builder.set(field, providedValue); + } + return ok(undefined); + }, + { editableFieldCount: buildContext?.editableFieldCount ?? editableFields?.length } + ); if (builder.hasErrors()) { return err(builder.getErrors()[0]!); } - const mutateSpec = yield* builder.build(); + const mutateSpec = yield* runTrace('buildMutationSpec', () => builder.build()); - const emptyFields = yield* TableRecordFields.create([]); - const emptyRecord = yield* TableRecord.create({ - id: recordId, - tableId: table.id(), - fieldValues: emptyFields.entries(), - }); + const emptyRecord = yield* runTrace('createEmptyRecord', () => + TableRecordFields.create([]).andThen((emptyFields) => + TableRecord.create({ + id: recordId, + tableId: table.id(), + fieldValues: emptyFields.entries(), + }) + ) + ); - const record = yield* mutateSpec.mutate(emptyRecord); + const record = yield* runTrace('mutateRecord', () => mutateSpec.mutate(emptyRecord)); - return ok(RecordUpdateResult.create(record, mutateSpec, fieldKeyMapping)); + const updateResult = yield* runTrace('createUpdateResult', () => + ok(RecordUpdateResult.create(record, mutateSpec, fieldKeyMapping)) + ); + return ok(updateResult); }); } diff --git a/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.spec.ts b/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.spec.ts new file mode 100644 index 0000000000..eaa873975a --- /dev/null +++ b/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.spec.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; + +import { BaseId } from '../../../base/BaseId'; +import { FieldId } from '../../fields/FieldId'; +import { FieldName } from '../../fields/FieldName'; +import { RecordId } from '../../records/RecordId'; +import { Table } from '../../Table'; +import { TableId } from '../../TableId'; +import { TableName } from '../../TableName'; + +const buildTable = (fieldCount: number) => { + const builder = Table.builder() + .withId(TableId.create(`tbl${'a'.repeat(16)}`)._unsafeUnwrap()) + .withBaseId(BaseId.create(`bse${'b'.repeat(16)}`)._unsafeUnwrap()) + .withName(TableName.create('Batch Update Table')._unsafeUnwrap()); + + for (let index = 0; index < fieldCount; index++) { + const fieldId = FieldId.create( + `fld${String.fromCharCode(97 + index).repeat(16)}` + )._unsafeUnwrap(); + const fieldBuilder = builder + .field() + .singleLineText() + .withId(fieldId) + .withName(FieldName.create(`Field ${index}`)._unsafeUnwrap()); + if (index === 0) { + fieldBuilder.primary(); + } + fieldBuilder.done(); + } + builder.view().defaultGrid().done(); + + return builder.build()._unsafeUnwrap(); +}; + +const buildUpdates = (count: number, fieldId: string) => + Array.from({ length: count }, (_, index) => ({ + recordId: RecordId.create(`rec${String(index).padStart(16, '0')}`)._unsafeUnwrap(), + fieldValues: new Map([[fieldId, `value-${index}`]]), + })); + +describe('updateRecordsStream', () => { + it('keeps the default 500-row stream batch cap', () => { + const table = buildTable(20); + const fieldId = table.getFields()[0]!.id().toString(); + const updates = buildUpdates(1000, fieldId); + + const batchSizes = [...table.updateRecordsStream(updates)].map((result) => { + if (result.isErr()) { + throw new Error(result.error.message); + } + return result._unsafeUnwrap().length; + }); + + expect(batchSizes).toEqual([500, 500]); + }); + + it('supports a 1k-row cap for narrow explicit bulk updates', () => { + const table = buildTable(20); + const fieldId = table.getFields()[0]!.id().toString(); + const updates = buildUpdates(1000, fieldId); + + const batchSizes = [...table.updateRecordsStream(updates, { maxBatchSize: 1000 })].map( + (result) => { + if (result.isErr()) { + throw new Error(result.error.message); + } + return result._unsafeUnwrap().length; + } + ); + + expect(batchSizes).toEqual([1000]); + }); + + it('supports mixing field ids and field names across a stream', () => { + const table = buildTable(2); + const [titleField, notesField] = table.getFields(); + const titleFieldId = titleField!.id().toString(); + const notesFieldId = notesField!.id().toString(); + const updates = [ + { + recordId: RecordId.create(`rec${'a'.repeat(16)}`)._unsafeUnwrap(), + fieldValues: new Map([[titleFieldId, 'title-by-id']]), + }, + { + recordId: RecordId.create(`rec${'b'.repeat(16)}`)._unsafeUnwrap(), + fieldValues: new Map([[notesField!.name().toString(), 'notes-by-name']]), + }, + ]; + + const [batchResult] = [...table.updateRecordsStream(updates)]; + const batch = batchResult!._unsafeUnwrap(); + + expect(batch[0]!.record.fields().get(titleField!.id())?.toValue()).toBe('title-by-id'); + expect(batch[0]!.fieldKeyMapping.get(titleFieldId)).toBe(titleFieldId); + expect(batch[1]!.record.fields().get(notesField!.id())?.toValue()).toBe('notes-by-name'); + expect(batch[1]!.fieldKeyMapping.get(notesFieldId)).toBe(notesField!.name().toString()); + }); +}); diff --git a/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.ts b/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.ts index 1703c8c24d..095b1376fa 100644 --- a/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.ts +++ b/packages/v2/core/src/domain/table/methods/records/updateRecordsStream.ts @@ -5,13 +5,46 @@ import type { RecordId } from '../../records/RecordId'; import type { RecordUpdateResult } from '../../records/RecordUpdateResult'; import type { Table } from '../../Table'; import { calculateBatchSize } from './calculateBatchSize'; -import { updateRecord } from './updateRecord'; +import { + createUpdateRecordBuildContext, + updateRecord, + type UpdateRecordBuildContext, + type UpdateRecordTraceHook, +} from './updateRecord'; export interface UpdateRecordItem { readonly recordId: RecordId; readonly fieldValues: ReadonlyMap; } +export type UpdateRecordsStreamTracePhase = + | 'updateRecord' + | 'yieldBatch' + | 'yieldFinalBatch'; + +export interface UpdateRecordsStreamTraceEvent { + readonly phase: UpdateRecordsStreamTracePhase; + readonly recordIndex?: number; + readonly batchIndex: number; + readonly batchSize: number; + readonly fieldCount?: number; + readonly targetBatchSize: number; +} + +export type UpdateRecordsStreamTraceHook = ( + event: UpdateRecordsStreamTraceEvent, + callback: () => T +) => T; + +export interface UpdateRecordsStreamOptions { + readonly typecast?: boolean; + readonly batchSize?: number; + readonly maxBatchSize?: number; + readonly trace?: UpdateRecordsStreamTraceHook; + readonly traceRecord?: UpdateRecordTraceHook; + readonly buildContext?: UpdateRecordBuildContext; +} + /** * Generates batched RecordUpdateResults from an iterable of update items. * @@ -27,29 +60,72 @@ export interface UpdateRecordItem { export function* updateRecordsStream( this: Table, updates: Iterable, - options?: { typecast?: boolean; batchSize?: number } + options?: UpdateRecordsStreamOptions ): Generator, DomainError>> { - const { typecast = false } = options ?? {}; - const batchSize = calculateBatchSize(this.getFields().length, options?.batchSize); + const { typecast = false, trace, traceRecord } = options ?? {}; + const buildContext = options?.buildContext ?? createUpdateRecordBuildContext(this); + const batchSize = calculateBatchSize(this.getFields().length, { + userBatchSize: options?.batchSize, + maxBatchSize: options?.maxBatchSize, + }); let batch: RecordUpdateResult[] = []; + let batchIndex = 0; + let recordIndex = 0; + + const runTrace = (event: UpdateRecordsStreamTraceEvent, callback: () => T): T => + trace ? trace(event, callback) : callback(); for (const { recordId, fieldValues } of updates) { // Use existing updateRecord method for each item - const updateResult = updateRecord.call(this, recordId, fieldValues, { typecast }); + const updateResult = runTrace( + { + phase: 'updateRecord', + recordIndex, + batchIndex, + batchSize: batch.length, + fieldCount: fieldValues.size, + targetBatchSize: batchSize, + }, + () => + updateRecord.call(this, recordId, fieldValues, { + typecast, + trace: traceRecord, + recordIndex, + buildContext, + }) + ); if (updateResult.isErr()) { yield err(updateResult.error); return; } batch.push(updateResult.value); + recordIndex += 1; if (batch.length >= batchSize) { - yield ok(batch); + yield runTrace( + { + phase: 'yieldBatch', + batchIndex, + batchSize: batch.length, + targetBatchSize: batchSize, + }, + () => ok(batch) + ); batch = []; // Reset for next batch, allows GC to collect previous batch + batchIndex += 1; } } // Yield remaining records if any if (batch.length > 0) { - yield ok(batch); + yield runTrace( + { + phase: 'yieldFinalBatch', + batchIndex, + batchSize: batch.length, + targetBatchSize: batchSize, + }, + () => ok(batch) + ); } } diff --git a/packages/v2/core/src/domain/table/methods/validateFormSubmission.spec.ts b/packages/v2/core/src/domain/table/methods/validateFormSubmission.spec.ts index e85c10d0ed..af6f0780d5 100644 --- a/packages/v2/core/src/domain/table/methods/validateFormSubmission.spec.ts +++ b/packages/v2/core/src/domain/table/methods/validateFormSubmission.spec.ts @@ -16,6 +16,7 @@ const createTestTable = () => { const tableName = TableName.create('Form Validation Table')._unsafeUnwrap(); const textFieldId = FieldId.create(`fld${'c'.repeat(16)}`)._unsafeUnwrap(); const hiddenFieldId = FieldId.create(`fld${'d'.repeat(16)}`)._unsafeUnwrap(); + const computedFieldId = FieldId.create(`fld${'e'.repeat(16)}`)._unsafeUnwrap(); const builder = Table.builder().withBaseId(baseId).withId(tableId).withName(tableName); builder @@ -31,6 +32,12 @@ const createTestTable = () => { .withId(hiddenFieldId) .withName(FieldName.create('Hidden Button')._unsafeUnwrap()) .done(); + builder + .field() + .createdBy() + .withId(computedFieldId) + .withName(FieldName.create('Creator')._unsafeUnwrap()) + .done(); builder.view().defaultGrid().done(); builder.view().form().defaultName().done(); @@ -46,6 +53,7 @@ const createTestTable = () => { table, textFieldId: textFieldId.toString(), hiddenFieldId: hiddenFieldId.toString(), + computedFieldId: computedFieldId.toString(), gridViewId: gridView.id().toString(), formViewId: formView.id().toString(), }; @@ -143,4 +151,15 @@ describe('Table.validateFormSubmission', () => { expect(result.isOk()).toBe(true); }); + + it('passes when a stale required form field is computed', () => { + const { table, computedFieldId, formViewId } = createTestTable(); + const tableWithRequiredComputedField = withFormRequiredFields(table, formViewId, [ + computedFieldId, + ]); + + const result = tableWithRequiredComputedField.validateFormSubmission(formViewId, new Map()); + + expect(result.isOk()).toBe(true); + }); }); diff --git a/packages/v2/core/src/domain/table/methods/validateFormSubmission.ts b/packages/v2/core/src/domain/table/methods/validateFormSubmission.ts index cb9fd7ed4e..94822d550d 100644 --- a/packages/v2/core/src/domain/table/methods/validateFormSubmission.ts +++ b/packages/v2/core/src/domain/table/methods/validateFormSubmission.ts @@ -45,10 +45,14 @@ export function validateFormSubmission( const columnMetaResult = formView.columnMeta(); if (columnMetaResult.isErr()) return err(columnMetaResult.error); const columnMeta = columnMetaResult.value.toDto(); + const fieldById = new Map(this.getFields().map((field) => [field.id().toString(), field])); const missingRequiredFieldIds: string[] = []; for (const visibleFieldId of visibleFieldIds) { const fieldId = visibleFieldId.toString(); + const field = fieldById.get(fieldId); + if (field?.computed().toBoolean()) continue; + const requiredByForm = columnMeta[fieldId]?.required === true; if (!requiredByForm) continue; diff --git a/packages/v2/core/src/domain/table/records/RecordMutationSpecBuilder.ts b/packages/v2/core/src/domain/table/records/RecordMutationSpecBuilder.ts index bee1f2c601..eddb7caa6b 100644 --- a/packages/v2/core/src/domain/table/records/RecordMutationSpecBuilder.ts +++ b/packages/v2/core/src/domain/table/records/RecordMutationSpecBuilder.ts @@ -5,6 +5,7 @@ import { domainError, type DomainError } from '../../shared/DomainError'; import { AndSpec } from '../../shared/specification/AndSpec'; import type { Field } from '../fields/Field'; import { FieldToSpecVisitor } from '../fields/visitors/FieldToSpecVisitor'; +import type { FieldCellValueSchema } from '../fields/visitors/FieldCellValueSchemaVisitor'; import type { ICellValueSpec, ICellValueSpecVisitor } from './specs/values/ICellValueSpecVisitor'; import { SetFieldValueSpecFactory } from './specs/values/SetFieldValueSpecFactory'; import type { TableRecord } from './TableRecord'; @@ -101,6 +102,19 @@ export class RecordMutationSpecBuilder { return this; } + setWithSchema(field: Field, value: unknown, schema: FieldCellValueSchema | undefined): this { + if (this.typecastMode || !schema) { + return this.set(field, value); + } + + const result = SetFieldValueSpecFactory.create(field, value, schema); + result.match( + (spec) => this.specs.push(spec), + (error) => this.errors.push(error) + ); + return this; + } + /** * Add a set operation with strict validation (ignores typecast mode). * diff --git a/packages/v2/core/src/domain/table/records/specs/values/SetFieldValueSpecFactory.ts b/packages/v2/core/src/domain/table/records/specs/values/SetFieldValueSpecFactory.ts index a29076d041..18fcd2a893 100644 --- a/packages/v2/core/src/domain/table/records/specs/values/SetFieldValueSpecFactory.ts +++ b/packages/v2/core/src/domain/table/records/specs/values/SetFieldValueSpecFactory.ts @@ -5,7 +5,10 @@ import z from 'zod'; import { domainError, type DomainError } from '../../../../shared/DomainError'; import type { Field } from '../../../fields/Field'; import { FieldType } from '../../../fields/FieldType'; -import { FieldCellValueSchemaVisitor } from '../../../fields/visitors/FieldCellValueSchemaVisitor'; +import { + FieldCellValueSchemaVisitor, + type FieldCellValueSchema, +} from '../../../fields/visitors/FieldCellValueSchemaVisitor'; import { SetFieldValueSpecFactoryVisitor } from '../../../fields/visitors/SetFieldValueSpecFactoryVisitor'; import type { ICellValueSpec } from './ICellValueSpecVisitor'; import { NoopCellValueSpec } from './NoopCellValueSpec'; @@ -32,7 +35,11 @@ export class SetFieldValueSpecFactory { * @param value - The raw value to validate and wrap * @returns A Result containing either the spec or a validation error */ - static create(field: Field, value: unknown): Result { + static create( + field: Field, + value: unknown, + schema?: FieldCellValueSchema + ): Result { if (field.type().equals(FieldType.button())) { return ok(NoopCellValueSpec.create()); } @@ -53,9 +60,10 @@ export class SetFieldValueSpecFactory { } return ok(ClearFieldValueSpec.create(field)); } - // Step 1: Get the validation schema for this field type - const schemaVisitor = FieldCellValueSchemaVisitor.create(); - return field.accept(schemaVisitor).andThen((schema) => { + const schemaResult = schema + ? ok(schema) + : field.accept(FieldCellValueSchemaVisitor.create()); + return schemaResult.andThen((schema) => { // Step 2: Validate the value against the schema const parseResult = schema.safeParse(value); if (!parseResult.success) { diff --git a/packages/v2/core/src/domain/table/records/specs/values/SetLinkValueSpec.ts b/packages/v2/core/src/domain/table/records/specs/values/SetLinkValueSpec.ts index e304da9d72..c32162677e 100644 --- a/packages/v2/core/src/domain/table/records/specs/values/SetLinkValueSpec.ts +++ b/packages/v2/core/src/domain/table/records/specs/values/SetLinkValueSpec.ts @@ -30,8 +30,13 @@ export class SetLinkValueSpec extends MutateOnlySpec item.id && !item.title); } diff --git a/packages/v2/core/src/domain/table/specs/field-updates/UpdateMultipleSelectOptionsSpec.ts b/packages/v2/core/src/domain/table/specs/field-updates/UpdateMultipleSelectOptionsSpec.ts index abf2318793..f3684267e3 100644 --- a/packages/v2/core/src/domain/table/specs/field-updates/UpdateMultipleSelectOptionsSpec.ts +++ b/packages/v2/core/src/domain/table/specs/field-updates/UpdateMultipleSelectOptionsSpec.ts @@ -139,6 +139,7 @@ export class UpdateMultipleSelectOptionsSpec< options: this.nextOptionsValue, defaultValue: field.defaultValue(), preventAutoNewOptions: field.preventAutoNewOptions(), + domainContext: this.domainContextValue, }); if (updatedFieldResult.isErr()) return err(updatedFieldResult.error); diff --git a/packages/v2/core/src/domain/table/specs/field-updates/UpdateSingleSelectOptionsSpec.ts b/packages/v2/core/src/domain/table/specs/field-updates/UpdateSingleSelectOptionsSpec.ts index ed3c838312..4f2c788adf 100644 --- a/packages/v2/core/src/domain/table/specs/field-updates/UpdateSingleSelectOptionsSpec.ts +++ b/packages/v2/core/src/domain/table/specs/field-updates/UpdateSingleSelectOptionsSpec.ts @@ -141,6 +141,7 @@ export class UpdateSingleSelectOptionsSpec< options: this.nextOptionsValue, defaultValue: field.defaultValue(), preventAutoNewOptions: field.preventAutoNewOptions(), + domainContext: this.domainContextValue, }); if (updatedFieldResult.isErr()) return err(updatedFieldResult.error); diff --git a/packages/v2/core/src/index.ts b/packages/v2/core/src/index.ts index d1afa6a8cd..34c0a8f492 100644 --- a/packages/v2/core/src/index.ts +++ b/packages/v2/core/src/index.ts @@ -44,6 +44,7 @@ export * from './application/services/DotTeaExportFieldNormalizer'; export * from './application/services/ForeignTableLoaderService'; export * from './application/services/LinkFieldUpdateSideEffectService'; export * from './application/services/LinkTitleResolverService'; +export * from './application/services/PreloadedTableContext'; export * from './application/services/TableDeletionSideEffectService'; export * from './application/services/AttachmentValueDecoratorService'; export * from './application/services/AttachmentValueResolverService'; @@ -156,6 +157,7 @@ export * from './commands/PublicCommand'; export * from './ports/DotTeaParser'; export * from './ports/AttachmentLookupService'; export * from './ports/AttachmentUrlSignerService'; +export * from './ports/ComputedFieldBackfillService'; export * from './ports/ComputedUpdateDrainService'; export * from './ports/FieldOperationPlugin'; export * from './ports/TableDataSafetyLimitPlugin'; diff --git a/packages/v2/core/src/ports/ComputedFieldBackfillService.ts b/packages/v2/core/src/ports/ComputedFieldBackfillService.ts new file mode 100644 index 0000000000..478d564a33 --- /dev/null +++ b/packages/v2/core/src/ports/ComputedFieldBackfillService.ts @@ -0,0 +1,23 @@ +import type { Result } from 'neverthrow'; + +import type { DomainError } from '../domain/shared/DomainError'; +import type { Field } from '../domain/table/fields/Field'; +import type { Table } from '../domain/table/Table'; +import type { IExecutionContext } from './ExecutionContext'; + +/** The computed fields that were actually backfilled (after filtering). */ +export type ComputedFieldBackfillManyResult = { + readonly fields: ReadonlyArray; +}; + +export interface IComputedFieldBackfillService { + executeSyncMany( + context: IExecutionContext, + input: { + table: Table; + fields: ReadonlyArray; + skipDistinctFilter?: boolean; + includeOneManyTwoWay?: boolean; + } + ): Promise>; +} diff --git a/packages/v2/core/src/ports/ExecutionContext.ts b/packages/v2/core/src/ports/ExecutionContext.ts index 4ebe457f46..3808920ac6 100644 --- a/packages/v2/core/src/ports/ExecutionContext.ts +++ b/packages/v2/core/src/ports/ExecutionContext.ts @@ -8,12 +8,18 @@ import type { ITracer } from './Tracer'; export interface IUnitOfWorkTransaction { readonly kind: 'unitOfWorkTransaction'; readonly scope?: UnitOfWorkScope; + readonly committed?: boolean; + readonly rolledBack?: boolean; afterCommit?(handler: UnitOfWorkAfterCommitHandler): void; afterRollback?(handler: UnitOfWorkAfterCommitHandler): void; } export type UnitOfWorkAfterCommitHandler = () => Promise | void; export type UnitOfWorkScope = 'meta' | 'data'; +export type ExecutionContextBackgroundTask = () => Promise | void; +export type ExecutionContextBackgroundTaskScheduler = ( + task: ExecutionContextBackgroundTask +) => void; export type IExecutionContextTransactions = Partial< Record @@ -35,6 +41,7 @@ export interface IExecutionContext { tracer?: ITracer; requestId?: string; windowId?: string; + scheduleBackgroundTask?: ExecutionContextBackgroundTaskScheduler; batchMutation?: IExecutionContextBatchMutation; undoRedo?: { mode: 'undo' | 'redo' | 'normal' }; duplicateTable?: { @@ -42,6 +49,9 @@ export interface IExecutionContext { duplicatedTableId?: string; includeRecords: boolean; }; + pendingTableDataInitialization?: { + tableIds: ReadonlyArray; + }; config?: { tableLimits?: TableDataSafetyLimitConfig; /** @deprecated Use `tableLimits.fieldOptions.maxSelectChoices`. */ @@ -122,6 +132,44 @@ export const getExecutionContextTranslator = ( context.$t?.(key as TableI18nKey, options) ?? key; }; +export const scheduleExecutionContextBackgroundTask = ( + context: IExecutionContext, + task: ExecutionContextBackgroundTask +): void => { + if (context.scheduleBackgroundTask) { + context.scheduleBackgroundTask(task); + return; + } + + const timeout = ( + globalThis as { + setTimeout?: (handler: () => void, timeout: number) => { unref?: () => void } | unknown; + } + ).setTimeout; + if (typeof timeout === 'function') { + const handle = timeout(() => void task(), 0) as { unref?: () => void } | undefined; + handle?.unref?.(); + return; + } + + const immediate = ( + globalThis as { setImmediate?: (handler: () => void) => { unref?: () => void } | unknown } + ).setImmediate; + if (typeof immediate === 'function') { + const handle = immediate(() => void task()) as { unref?: () => void } | undefined; + handle?.unref?.(); + return; + } + + const scheduler = (globalThis as { queueMicrotask?: (task: () => void) => void }).queueMicrotask; + if (typeof scheduler === 'function') { + scheduler(() => void task()); + return; + } + + void task(); +}; + export const getDomainContext = (context?: IExecutionContext): IDomainContext | undefined => { const tableLimits = context?.config?.tableLimits; const selectFieldOptions = context?.config?.selectFieldOptions; diff --git a/packages/v2/core/src/ports/TableOperationPlugin.ts b/packages/v2/core/src/ports/TableOperationPlugin.ts index 346e835bd5..75beceaf27 100644 --- a/packages/v2/core/src/ports/TableOperationPlugin.ts +++ b/packages/v2/core/src/ports/TableOperationPlugin.ts @@ -8,6 +8,7 @@ import type { IExecutionContext } from './ExecutionContext'; import type { PluginTraceContext } from './Tracer'; export const TableOperationKind = { + read: 'read', create: 'create', createMany: 'createMany', duplicate: 'duplicate', @@ -29,6 +30,11 @@ interface ITableOperationPluginContextBase; +export type ITableOperationReadContext = ITableOperationPluginContextBase< + 'read', + TableOperationReadPayload +>; export type ITableOperationCreateManyContext = ITableOperationPluginContextBase< 'createMany', TableOperationCreateManyPayload @@ -87,6 +97,7 @@ export type ITableOperationRenameContext = ITableOperationPluginContextBase< >; export type TableOperationPluginContextMap = { + read: ITableOperationReadContext; create: ITableOperationCreateContext; createMany: ITableOperationCreateManyContext; duplicate: ITableOperationDuplicateContext; diff --git a/packages/v2/core/src/ports/TableRecordRepository.ts b/packages/v2/core/src/ports/TableRecordRepository.ts index d0cf098d70..cf12625515 100644 --- a/packages/v2/core/src/ports/TableRecordRepository.ts +++ b/packages/v2/core/src/ports/TableRecordRepository.ts @@ -153,6 +153,13 @@ export interface InsertManyStreamOptions { * related tables and junction rows are present. */ skipComputedUpdates?: boolean; + + /** + * When true, adapters may skip returning per-field changed values for each + * inserted record. Restore-style callers that only need persistence/progress + * can avoid sending large inserted values back over the database connection. + */ + skipChangedFields?: boolean; } /** @@ -242,6 +249,13 @@ export interface UpdateManyStreamOptions { * Foreign tables referenced by missing-title link payloads. */ fillLinkTitleForeignTables?: ReadonlyMap; + + /** + * When true, link fields in the batch are known to have no existing persisted + * relation state, so adapters can skip loading old link ids while collecting + * relation impact. + */ + assumeEmptyLinkState?: boolean; } /** @@ -310,8 +324,8 @@ export interface RecordRestoreSystemValues { autoNumber?: number; createdTime?: string; createdBy?: string; - lastModifiedTime?: string; - lastModifiedBy?: string; + lastModifiedTime?: string | null; + lastModifiedBy?: string | null; /** Preserve per-view row-order snapshot during undo/redo restore. */ orders?: Readonly>; /** diff --git a/packages/v2/core/src/ports/TableSchemaRepository.ts b/packages/v2/core/src/ports/TableSchemaRepository.ts index f053e33194..009f6fc1de 100644 --- a/packages/v2/core/src/ports/TableSchemaRepository.ts +++ b/packages/v2/core/src/ports/TableSchemaRepository.ts @@ -7,15 +7,27 @@ import type { Table } from '../domain/table/Table'; import type { IExecutionContext } from './ExecutionContext'; import type { TableDeleteOptions } from './TableRepository'; +export type TableSchemaInsertManyOptions = { + knownTables?: ReadonlyArray
; + /** + * The caller is creating brand-new empty physical tables and will load records afterwards. + * Adapters may skip data repair/backfill SQL that is only needed for existing rows. + */ + optimizeForEmptyTables?: boolean; + /** + * The caller will perform system/restore writes that do not need undo snapshots while loading. + * Adapters may defer undo-capture trigger setup until the first normal record mutation. + */ + skipUndoCaptureSetup?: boolean; +}; + export interface ITableSchemaRepository { insert(context: IExecutionContext, table: Table): Promise>; ensureInserted?(context: IExecutionContext, table: Table): Promise>; insertMany( context: IExecutionContext, tables: ReadonlyArray
, - options?: { - knownTables?: ReadonlyArray
; - } + options?: TableSchemaInsertManyOptions ): Promise>; ensureInsertedMany?( context: IExecutionContext, diff --git a/packages/v2/core/src/ports/defaults/NoopComputedFieldBackfillService.ts b/packages/v2/core/src/ports/defaults/NoopComputedFieldBackfillService.ts new file mode 100644 index 0000000000..b025f3e1e5 --- /dev/null +++ b/packages/v2/core/src/ports/defaults/NoopComputedFieldBackfillService.ts @@ -0,0 +1,25 @@ +import { ok } from 'neverthrow'; +import type { Result } from 'neverthrow'; + +import type { DomainError } from '../../domain/shared/DomainError'; +import type { Field } from '../../domain/table/fields/Field'; +import type { Table } from '../../domain/table/Table'; +import type { + ComputedFieldBackfillManyResult, + IComputedFieldBackfillService, +} from '../ComputedFieldBackfillService'; +import type { IExecutionContext } from '../ExecutionContext'; + +export class NoopComputedFieldBackfillService implements IComputedFieldBackfillService { + async executeSyncMany( + _context: IExecutionContext, + _input: { + table: Table; + fields: ReadonlyArray; + skipDistinctFilter?: boolean; + includeOneManyTwoWay?: boolean; + } + ): Promise> { + return ok({ fields: [] }); + } +} diff --git a/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.spec.ts b/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.spec.ts index 698e472604..fcbd0d0352 100644 --- a/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.spec.ts +++ b/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.spec.ts @@ -22,8 +22,11 @@ import { DateTimeFormatting } from '../../../domain/table/fields/types/DateTimeF import { FieldColor } from '../../../domain/table/fields/types/FieldColor'; import { FormulaExpression } from '../../../domain/table/fields/types/FormulaExpression'; import { FormulaField } from '../../../domain/table/fields/types/FormulaField'; +import { LinkFieldConfig } from '../../../domain/table/fields/types/LinkFieldConfig'; +import { LinkRelationship } from '../../../domain/table/fields/types/LinkRelationship'; import { LongTextField } from '../../../domain/table/fields/types/LongTextField'; import { LookupField } from '../../../domain/table/fields/types/LookupField'; +import { LookupOptions } from '../../../domain/table/fields/types/LookupOptions'; import { MultipleSelectField } from '../../../domain/table/fields/types/MultipleSelectField'; import { NumberDefaultValue } from '../../../domain/table/fields/types/NumberDefaultValue'; import { NumberField } from '../../../domain/table/fields/types/NumberField'; @@ -56,6 +59,7 @@ import { ViewColumnMeta } from '../../../domain/table/views/ViewColumnMeta'; import { ViewId } from '../../../domain/table/views/ViewId'; import { ViewName } from '../../../domain/table/views/ViewName'; import { ViewQueryDefaults } from '../../../domain/table/views/ViewQueryDefaults'; +import type { ITableFieldPersistenceDTO } from '../TableMapper'; import { DefaultTableMapper } from './DefaultTableMapper'; const createFieldId = (seed: string) => FieldId.create(`fld${seed.repeat(16)}`); @@ -624,6 +628,108 @@ describe('DefaultTableMapper', () => { expect(lookupField?.dbFieldName).not.toBe('__auto_number'); }); + it('enriches lookup options with link fk metadata and resolves cellValueType', () => { + const baseId = BaseId.create(`bse${'r'.repeat(16)}`)._unsafeUnwrap(); + const tableId = TableId.create(`tbl${'r'.repeat(16)}`)._unsafeUnwrap(); + const primaryFieldId = createFieldId('s')._unsafeUnwrap(); + const linkFieldId = createFieldId('t')._unsafeUnwrap(); + const lookupFieldId = createFieldId('u')._unsafeUnwrap(); + const foreignTableId = TableId.create(`tbl${'v'.repeat(16)}`)._unsafeUnwrap(); + const foreignTargetFieldId = createFieldId('w')._unsafeUnwrap(); + + const linkConfig = LinkFieldConfig.create({ + relationship: LinkRelationship.manyOne().toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + // mutable link config that must NOT be copied into the lookup's options + visibleFieldIds: [foreignTargetFieldId.toString()], + })._unsafeUnwrap(); + + const builder = Table.builder() + .withId(tableId) + .withBaseId(baseId) + .withName(TableName.create('Lookup Enrichment')._unsafeUnwrap()); + builder + .field() + .singleLineText() + .withId(primaryFieldId) + .withName(FieldName.create('Title')._unsafeUnwrap()) + .primary() + .done(); + builder + .field() + .link() + .withId(linkFieldId) + .withName(FieldName.create('Link')._unsafeUnwrap()) + .withConfig(linkConfig) + .done(); + builder + .field() + .lookup() + .withId(lookupFieldId) + .withName(FieldName.create('Lookup Amount')._unsafeUnwrap()) + .withInnerField( + NumberField.create({ + id: createFieldId('x')._unsafeUnwrap(), + name: FieldName.create('Amount')._unsafeUnwrap(), + formatting: NumberFormatting.create({ type: 'decimal', precision: 2 })._unsafeUnwrap(), + })._unsafeUnwrap() + ) + .withLookupOptions( + LookupOptions.create({ + linkFieldId: linkFieldId.toString(), + foreignTableId: foreignTableId.toString(), + lookupFieldId: foreignTargetFieldId.toString(), + })._unsafeUnwrap() + ) + .withIsMultipleCellValue(false) + .done(); + builder.view().defaultGrid().done(); + + const table = builder.build()._unsafeUnwrap(); + const dto = new DefaultTableMapper().toDTO(table)._unsafeUnwrap(); + + const lookupDto = dto.fields.find((field) => field.id === lookupFieldId.toString()) as + | (ITableFieldPersistenceDTO & { + lookupOptions?: Record; + cellValueType?: string; + }) + | undefined; + + // The lookup must carry the parent link's stable physical join metadata so it passes + // fieldVoSchema on the client. + expect(lookupDto?.lookupOptions?.fkHostTableName).toBeTruthy(); + expect(lookupDto?.lookupOptions?.selfKeyName).toBeTruthy(); + expect(lookupDto?.lookupOptions?.foreignKeyName).toBeTruthy(); + expect(lookupDto?.lookupOptions?.linkFieldId).toBe(linkFieldId.toString()); + // cellValueType must be resolved (not null) so the shape-refresh op does not corrupt the field. + expect(lookupDto?.cellValueType).toBe('number'); + expect(lookupDto?.type).toBe('number'); + + // The lookup must NOT bake in the link's mutable record-scoping config (would go stale). + const linkDto = dto.fields.find((field) => field.id === linkFieldId.toString()) as + | (ITableFieldPersistenceDTO & { options?: Record }) + | undefined; + expect((linkDto?.options as Record)?.visibleFieldIds).toBeTruthy(); + const allowedLookupKeys = new Set([ + 'linkFieldId', + 'lookupFieldId', + 'foreignTableId', + 'relationship', + 'fkHostTableName', + 'selfKeyName', + 'foreignKeyName', + 'filter', + 'sort', + 'limit', + ]); + for (const key of Object.keys(lookupDto?.lookupOptions ?? {})) { + expect(allowedLookupKeys.has(key)).toBe(true); + } + expect(lookupDto?.lookupOptions).not.toHaveProperty('visibleFieldIds'); + expect(lookupDto?.lookupOptions).not.toHaveProperty('filterByViewId'); + }); + it('rehydrates conditional lookup inner formula when result type is provided', () => { const table = buildTable(); const mapper = new DefaultTableMapper(); diff --git a/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.ts b/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.ts index d731dacbcd..d96448286b 100644 --- a/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.ts +++ b/packages/v2/core/src/ports/mappers/defaults/DefaultTableMapper.ts @@ -12,8 +12,8 @@ import { FieldId } from '../../../domain/table/fields/FieldId'; import { FieldName } from '../../../domain/table/fields/FieldName'; import { AttachmentField } from '../../../domain/table/fields/types/AttachmentField'; import { AutoNumberField } from '../../../domain/table/fields/types/AutoNumberField'; -import { ButtonField } from '../../../domain/table/fields/types/ButtonField'; import { ButtonConfirm } from '../../../domain/table/fields/types/ButtonConfirm'; +import { ButtonField } from '../../../domain/table/fields/types/ButtonField'; import { ButtonLabel } from '../../../domain/table/fields/types/ButtonLabel'; import { ButtonMaxCount } from '../../../domain/table/fields/types/ButtonMaxCount'; import { ButtonResetCount } from '../../../domain/table/fields/types/ButtonResetCount'; @@ -126,11 +126,14 @@ import type { const sequenceResults = ( values: ReadonlyArray> -): Result, DomainError> => - values.reduce, DomainError>>( - (acc, next) => acc.andThen((arr) => next.map((v) => [...arr, v])), - ok([]) - ); +): Result, DomainError> => { + const result: T[] = []; + for (const value of values) { + if (value.isErr()) return err, DomainError>(value.error); + result.push(value.value); + } + return ok(result); +}; const optional = ( raw: unknown, @@ -262,7 +265,10 @@ class FieldToPersistenceVisitor implements IFieldVisitor ILinkFieldOptionsDTO['relationship'] | undefined + ) => ILinkFieldOptionsDTO['relationship'] | undefined, + private readonly resolveLinkFieldOptions?: ( + linkFieldId: string + ) => ILinkFieldOptionsDTO | undefined ) {} private baseField(field: Field): { @@ -631,6 +637,26 @@ class FieldToPersistenceVisitor implements IFieldVisitor> { + const result: Partial< + Pick + > = {}; + if (linkOptions.fkHostTableName != null) result.fkHostTableName = linkOptions.fkHostTableName; + if (linkOptions.selfKeyName != null) result.selfKeyName = linkOptions.selfKeyName; + if (linkOptions.foreignKeyName != null) result.foreignKeyName = linkOptions.foreignKeyName; + return result; + } + visitLinkField(field: LinkField): Result { const optionsResult = field.configDto().orElse(() => ok(this.mapLinkFieldOptions(field))); if (optionsResult.isErr()) { @@ -687,6 +713,17 @@ class FieldToPersistenceVisitor implements IFieldVisitor export class DefaultTableMapper implements ITableMapper { toDTO(table: Table): Result { const relationshipByLinkFieldId = new Map(); + const linkOptionsByLinkFieldId = new Map(); for (const field of table.getFields()) { if (!(field instanceof LinkField)) { continue; } relationshipByLinkFieldId.set(field.id().toString(), field.relationship().toString()); + const linkOptionsResult = field.configDto(); + if (linkOptionsResult.isOk()) { + linkOptionsByLinkFieldId.set(field.id().toString(), linkOptionsResult.value); + } } - const fieldVisitor = new FieldToPersistenceVisitor((linkFieldId) => - relationshipByLinkFieldId.get(linkFieldId) + const fieldVisitor = new FieldToPersistenceVisitor( + (linkFieldId) => relationshipByLinkFieldId.get(linkFieldId), + (linkFieldId) => linkOptionsByLinkFieldId.get(linkFieldId) ); const dbTableName = table .dbTableName() diff --git a/packages/v2/core/src/ports/memory/AsyncMemoryEventBus.ts b/packages/v2/core/src/ports/memory/AsyncMemoryEventBus.ts index c0e91d422c..04a1f516e3 100644 --- a/packages/v2/core/src/ports/memory/AsyncMemoryEventBus.ts +++ b/packages/v2/core/src/ports/memory/AsyncMemoryEventBus.ts @@ -42,19 +42,22 @@ const resolveErrorMessage = (error: unknown): string => { }; const defaultScheduler: AsyncEventBusScheduler = (task) => { - const immediate = (globalThis as { setImmediate?: (task: () => void) => void }).setImmediate; - if (typeof immediate === 'function') { - immediate(() => void task()); - return; - } - const timeout = ( globalThis as { setTimeout?: (handler: () => void, timeout: number) => void; } ).setTimeout; if (typeof timeout === 'function') { - timeout(() => void task(), 0); + // Let the HTTP response path finish before heavy fire-and-forget projections start. + timeout(() => { + timeout(() => void task(), 0); + }, 0); + return; + } + + const immediate = (globalThis as { setImmediate?: (task: () => void) => void }).setImmediate; + if (typeof immediate === 'function') { + immediate(() => void task()); return; } @@ -127,7 +130,7 @@ export class AsyncMemoryEventBus implements IEventBus { const wasDraining = this.draining; this.enrichWithRequestId(context, event); this.maybeRecordPublishedEvents([event]); - const targetSeq = this.enqueue(context, [event]); + const targetSeq = this.enqueue(context, [event], !shouldAwait); if (shouldAwait && !wasDraining) { await this.waitUntilProcessed(targetSeq); } @@ -147,7 +150,7 @@ export class AsyncMemoryEventBus implements IEventBus { this.enrichWithRequestId(context, event); } this.maybeRecordPublishedEvents(events); - const targetSeq = this.enqueue(context, events); + const targetSeq = this.enqueue(context, events, !shouldAwait); if (shouldAwait && !wasDraining) { await this.waitUntilProcessed(targetSeq); } @@ -168,7 +171,11 @@ export class AsyncMemoryEventBus implements IEventBus { this.publishedEvents.push(...events); } - private enqueue(context: IExecutionContext, events: ReadonlyArray): number { + private enqueue( + context: IExecutionContext, + events: ReadonlyArray, + preferContextBackgroundTask: boolean + ): number { const contextSnapshot = snapshotExecutionContext(context); let targetSeq = this.processedSeq; for (const event of events) { @@ -179,7 +186,7 @@ export class AsyncMemoryEventBus implements IEventBus { } if (!this.draining) { this.draining = true; - this.scheduleDrain(); + this.scheduleDrain(contextSnapshot, preferContextBackgroundTask); } return targetSeq; } @@ -206,7 +213,17 @@ export class AsyncMemoryEventBus implements IEventBus { } } - private scheduleDrain(): void { + private scheduleDrain( + context: IExecutionContext, + preferContextBackgroundTask: boolean = false + ): void { + if (preferContextBackgroundTask && context.scheduleBackgroundTask) { + context.scheduleBackgroundTask(async () => { + await this.drain(); + }); + return; + } + const schedule = this.options.schedule ?? defaultScheduler; schedule(async () => { await this.drain(); @@ -224,7 +241,7 @@ export class AsyncMemoryEventBus implements IEventBus { this.draining = false; if (this.queue.length > 0) { this.draining = true; - this.scheduleDrain(); + this.scheduleDrain(this.queue[0]!.context); } } @@ -368,6 +385,7 @@ export class AsyncMemoryEventBus implements IEventBus { const snapshotExecutionContext = (context: IExecutionContext): IExecutionContext => ({ ...context, + scheduleBackgroundTask: context.scheduleBackgroundTask, undoRedo: context.undoRedo ? { ...context.undoRedo } : undefined, duplicateTable: context.duplicateTable ? { ...context.duplicateTable } : undefined, config: context.config diff --git a/packages/v2/core/src/ports/memory/MemoryPorts.spec.ts b/packages/v2/core/src/ports/memory/MemoryPorts.spec.ts index 98598644cc..a54d01e909 100644 --- a/packages/v2/core/src/ports/memory/MemoryPorts.spec.ts +++ b/packages/v2/core/src/ports/memory/MemoryPorts.spec.ts @@ -326,6 +326,65 @@ describe('MemoryEventBus', () => { }); describe('AsyncMemoryEventBus', () => { + it('defers default fire-and-forget dispatch by two timer turns', async () => { + const globalWithSchedulers = globalThis as { + setTimeout?: typeof setTimeout; + setImmediate?: typeof setImmediate; + }; + const originalSetTimeout = globalWithSchedulers.setTimeout; + const originalSetImmediate = globalWithSchedulers.setImmediate; + const calls: string[] = []; + const timers: Array<() => void> = []; + + globalWithSchedulers.setTimeout = ((handler: () => void) => { + calls.push('timeout'); + timers.push(handler); + return 0 as unknown as ReturnType; + }) as typeof setTimeout; + globalWithSchedulers.setImmediate = ((handler: () => void) => { + calls.push('immediate'); + handler(); + return undefined as unknown as ReturnType; + }) as typeof setImmediate; + + try { + class TimerDispatchEvent implements IDomainEvent { + readonly name = DomainEventName.tableCreated(); + readonly occurredAt = OccurredAt.now(); + } + + let handled = false; + @ProjectionHandler(TimerDispatchEvent) + class TimerDispatchHandler implements IEventHandler { + async handle( + _context: IExecutionContext, + _event: TimerDispatchEvent + ): ReturnType['handle']> { + handled = true; + return ok(undefined); + } + } + expect(TimerDispatchHandler).toBeDefined(); + + const bus = new AsyncMemoryEventBus(new MapResolver()); + await bus.publish(createContext(), new TimerDispatchEvent()); + + expect(calls).toEqual(['timeout']); + expect(handled).toBe(false); + + timers.shift()?.(); + expect(calls).toEqual(['timeout', 'timeout']); + expect(handled).toBe(false); + + timers.shift()?.(); + await waitForPredicate(() => handled); + expect(handled).toBe(true); + } finally { + globalWithSchedulers.setTimeout = originalSetTimeout; + globalWithSchedulers.setImmediate = originalSetImmediate; + } + }); + it('publishes events without waiting for handlers', async () => { class PingEvent implements IDomainEvent { readonly name = DomainEventName.tableCreated(); @@ -365,6 +424,109 @@ describe('AsyncMemoryEventBus', () => { expect(handled).toBe(1); }); + it('uses execution-context background scheduling for non-awaitable events', async () => { + class BackgroundScheduledEvent implements IDomainEvent { + readonly name = DomainEventName.tableCreated(); + readonly occurredAt = OccurredAt.now(); + } + + let handled = false; + + @EventHandler(BackgroundScheduledEvent) + class BackgroundScheduledHandler implements IEventHandler { + async handle( + _context: IExecutionContext, + _event: BackgroundScheduledEvent + ): ReturnType['handle']> { + handled = true; + return ok(undefined); + } + } + expect(BackgroundScheduledHandler).toBeDefined(); + + const scheduledTasks: Array<() => Promise> = []; + const backgroundTasks: Array<() => Promise | void> = []; + const schedule: AsyncEventBusScheduler = (task) => { + scheduledTasks.push(task); + }; + + const resolver = new MapResolver(); + const bus = new AsyncMemoryEventBus(resolver, { schedule }); + const context: IExecutionContext = { + ...createContext(), + scheduleBackgroundTask: (task) => { + backgroundTasks.push(task); + }, + }; + + const publishResult = await bus.publish(context, new BackgroundScheduledEvent()); + publishResult._unsafeUnwrap(); + + expect(scheduledTasks).toHaveLength(0); + expect(backgroundTasks).toHaveLength(1); + expect(handled).toBe(false); + + await backgroundTasks.shift()?.(); + + expect(handled).toBe(true); + }); + + it('keeps awaitable events on the event-bus scheduler', async () => { + class AwaitableEvent implements IDomainEvent { + readonly name = DomainEventName.fieldCreated(); + readonly occurredAt = OccurredAt.now(); + } + + let handled = false; + + @EventHandler(AwaitableEvent) + class AwaitableHandler implements IEventHandler { + async handle( + _context: IExecutionContext, + _event: AwaitableEvent + ): ReturnType['handle']> { + handled = true; + return ok(undefined); + } + } + expect(AwaitableHandler).toBeDefined(); + + const scheduledTasks: Array<() => Promise> = []; + const backgroundTasks: Array<() => Promise | void> = []; + const schedule: AsyncEventBusScheduler = (task) => { + scheduledTasks.push(task); + }; + + const resolver = new MapResolver(); + const bus = new AsyncMemoryEventBus(resolver, { schedule }); + const context: IExecutionContext = { + ...createContext(), + scheduleBackgroundTask: (task) => { + backgroundTasks.push(task); + }, + }; + + let publishResolved = false; + const publishPromise = bus.publish(context, new AwaitableEvent()).then((result) => { + publishResolved = true; + return result; + }); + + await Promise.resolve(); + + expect(scheduledTasks).toHaveLength(1); + expect(backgroundTasks).toHaveLength(0); + expect(handled).toBe(false); + expect(publishResolved).toBe(false); + + await scheduledTasks.shift()?.(); + const publishResult = await publishPromise; + + publishResult._unsafeUnwrap(); + expect(handled).toBe(true); + expect(publishResolved).toBe(true); + }); + it('does not retain published events when recording is disabled', async () => { class PingEvent implements IDomainEvent { readonly name = DomainEventName.tableCreated(); diff --git a/packages/v2/core/src/ports/tokens.ts b/packages/v2/core/src/ports/tokens.ts index ab4fa76ac6..f53dde1f28 100644 --- a/packages/v2/core/src/ports/tokens.ts +++ b/packages/v2/core/src/ports/tokens.ts @@ -39,6 +39,7 @@ export const v2CoreTokens = { userLookupService: Symbol('v2.core.userLookupService'), userRenamePropagationService: Symbol('v2.core.userRenamePropagationService'), computedUpdateDrainService: Symbol('v2.core.computedUpdateDrainService'), + computedFieldBackfillService: Symbol('v2.core.computedFieldBackfillService'), schemaOperationRunnerService: Symbol('v2.core.schemaOperationRunnerService'), schemaOperationHandlers: Symbol('v2.core.schemaOperationHandlers'), recordWritePluginRunner: Symbol('v2.core.recordWritePluginRunner'), diff --git a/packages/v2/core/src/queries/GetTableByIdHandler.ts b/packages/v2/core/src/queries/GetTableByIdHandler.ts index db84d32fd5..5d5e5c12aa 100644 --- a/packages/v2/core/src/queries/GetTableByIdHandler.ts +++ b/packages/v2/core/src/queries/GetTableByIdHandler.ts @@ -5,9 +5,12 @@ import type { Result } from 'neverthrow'; import { domainError, isNotFoundError, type DomainError } from '../domain/shared/DomainError'; import type { Table } from '../domain/table/Table'; import { Table as TableAggregate } from '../domain/table/Table'; +import { TableOperationPluginRunner } from '../application/services/TableOperationPluginRunner'; import type { IExecutionContext } from '../ports/ExecutionContext'; +import { NoopLogger } from '../ports/defaults/NoopLogger'; import * as LoggerPort from '../ports/Logger'; import * as TableRepositoryPort from '../ports/TableRepository'; +import { TableOperationKind } from '../ports/TableOperationPlugin'; import { v2CoreTokens } from '../ports/tokens'; import { GetTableByIdQuery } from './GetTableByIdQuery'; import { QueryHandler, type IQueryHandler } from './QueryHandler'; @@ -27,7 +30,12 @@ export class GetTableByIdHandler implements IQueryHandler { updateOne: async (_context, _table, _spec) => err(domainError.notFound({ message: 'Not found' })), delete: async (_context, _table) => err(domainError.notFound({ message: 'Not found' })), + restore: async (_context, _table) => err(domainError.notFound({ message: 'Not found' })), }; const recordQueryRepo: ITableRecordQueryRepository = { @@ -543,6 +544,10 @@ describe('ListTableRecordsHandler', () => { column: `__row_${viewId}`, direction: 'asc', }, + { + column: '__auto_number', + direction: 'asc', + }, ]); }); @@ -651,12 +656,20 @@ describe('ListTableRecordsHandler', () => { { fieldId: titleField.id().toString(), direction: 'desc', + column: undefined, }, { fieldId: statusField.id().toString(), direction: 'asc', + column: undefined, + }, + { + fieldId: undefined, + column: `__row_${view.id().toString()}`, + direction: 'asc', }, { + fieldId: undefined, column: '__auto_number', direction: 'asc', }, diff --git a/packages/v2/core/src/queries/ListTableRecordsHandler.ts b/packages/v2/core/src/queries/ListTableRecordsHandler.ts index 9472419b72..a5f3cb0ed6 100644 --- a/packages/v2/core/src/queries/ListTableRecordsHandler.ts +++ b/packages/v2/core/src/queries/ListTableRecordsHandler.ts @@ -3,10 +3,13 @@ import { err, ok, safeTry } from 'neverthrow'; import type { Result } from 'neverthrow'; import { FieldKeyResolverService } from '../application/services/FieldKeyResolverService'; -import { mergeOrderBy, resolveOrderBy as resolveQueryOrderBy } from '../commands/shared/orderBy'; +import { + mergeOrderBy, + resolveGroupByToOrderBy, + resolveOrderBy as resolveQueryOrderBy, +} from '../commands/shared/orderBy'; import { domainError, isNotFoundError, type DomainError } from '../domain/shared/DomainError'; import { type ISpecification } from '../domain/shared/specification/ISpecification'; -import { FieldType } from '../domain/table/fields/FieldType'; import { FieldId } from '../domain/table/fields/FieldId'; import { FieldKeyType } from '../domain/table/fields/FieldKeyType'; import { FieldCondition } from '../domain/table/fields/types/FieldCondition'; @@ -37,13 +40,14 @@ import { type RecordFilter, type RecordFilterCondition, type RecordFilterNode, - type RecordFilterValue, } from './RecordFilterDto'; -import { buildRecordConditionSpec, sanitizeRecordFilter } from './RecordFilterMapper'; +import { + buildRecordConditionSpec, + replaceCurrentUserTagInFilter, + sanitizeRecordFilter, +} from './RecordFilterMapper'; import { RecordSearch, resolveVisibleRowSearch } from './RecordSearch'; -const currentUserFilterValue = 'Me'; - export class ListTableRecordsResult { private constructor( readonly records: ReadonlyArray, @@ -158,60 +162,6 @@ function resolveFilterNodeFieldKeys( return ok(node); } -function isUserLikeFieldType(type: FieldType): boolean { - return ( - type.equals(FieldType.user()) || - type.equals(FieldType.createdBy()) || - type.equals(FieldType.lastModifiedBy()) - ); -} - -function replaceCurrentUserTagInFilter( - table: Table, - filter: RecordFilter | null | undefined, - actorId: string -): RecordFilter | null | undefined { - if (!filter) { - return filter; - } - - const replaceNode = (node: RecordFilterNode): RecordFilterNode => { - if (isRecordFilterNot(node)) { - return { not: replaceNode(node.not) }; - } - - if (isRecordFilterGroup(node)) { - return { - ...node, - items: node.items.map((item) => replaceNode(item)), - }; - } - - if (!isRecordFilterCondition(node)) { - return node; - } - - const fieldResult = table.getField((field) => field.id().toString() === node.fieldId); - if (fieldResult.isErr() || !isUserLikeFieldType(fieldResult.value.type())) { - return node; - } - - const replaceValue = (value: RecordFilterValue): RecordFilterValue => { - if (Array.isArray(value)) { - return value.map((item) => (item === currentUserFilterValue ? actorId : item)); - } - return value === currentUserFilterValue ? actorId : value; - }; - - return { - ...node, - value: replaceValue(node.value), - }; - }; - - return replaceNode(filter); -} - type IRecordReadQuerySource = { enabledFieldIds?: ReadonlyArray; }; @@ -465,8 +415,11 @@ export class ListTableRecordsHandler effectiveQueryDefaults?.manualSort(), resolvedSort ); + const effectiveGroup = query.groupBy?.length + ? undefined + : effectiveQueryDefaults?.group(); const orderBy = mergeOrderBy( - undefined, + yield* resolveGroupByToOrderBy(effectiveGroup), yield* resolveQueryOrderBy(effectiveSort), query.viewId ); diff --git a/packages/v2/core/src/queries/RecordFilterMapper.spec.ts b/packages/v2/core/src/queries/RecordFilterMapper.spec.ts index 3de27e4815..7b6047ecad 100644 --- a/packages/v2/core/src/queries/RecordFilterMapper.spec.ts +++ b/packages/v2/core/src/queries/RecordFilterMapper.spec.ts @@ -10,7 +10,11 @@ import { Table } from '../domain/table/Table'; import { TableId } from '../domain/table/TableId'; import { TableName } from '../domain/table/TableName'; import type { RecordFilter } from './RecordFilterDto'; -import { buildRecordConditionSpec, sanitizeRecordFilter } from './RecordFilterMapper'; +import { + buildRecordConditionSpec, + replaceCurrentUserTagInFilter, + sanitizeRecordFilter, +} from './RecordFilterMapper'; const baseId = (seed: string) => BaseId.create(`bse${seed.repeat(16)}`)._unsafeUnwrap(); const recordId = (seed: string) => RecordId.create(`rec${seed.repeat(16)}`)._unsafeUnwrap(); @@ -32,6 +36,9 @@ const buildTable = () => { .withName(FieldName.create('Status')._unsafeUnwrap()) .withOptions([selectOption('Open')]) .done(); + builder.field().user().withName(FieldName.create('Owner')._unsafeUnwrap()).done(); + builder.field().createdBy().withName(FieldName.create('Creator')._unsafeUnwrap()).done(); + builder.field().lastModifiedBy().withName(FieldName.create('Modifier')._unsafeUnwrap()).done(); builder.view().defaultGrid().done(); return builder.build()._unsafeUnwrap(); @@ -251,4 +258,44 @@ describe('RecordFilterMapper', () => { ], }); }); + + it('replaces current-user tag only for user-like filter fields', () => { + const table = buildTable(); + const ownerField = table + .getField((field) => field.name().toString() === 'Owner') + ._unsafeUnwrap(); + const creatorField = table + .getField((field) => field.name().toString() === 'Creator') + ._unsafeUnwrap(); + const modifierField = table + .getField((field) => field.name().toString() === 'Modifier') + ._unsafeUnwrap(); + const titleField = table + .getField((field) => field.name().toString() === 'Title') + ._unsafeUnwrap(); + + const filter: RecordFilter = { + conjunction: 'and', + items: [ + { fieldId: ownerField.id().toString(), operator: 'is', value: 'Me' }, + { fieldId: creatorField.id().toString(), operator: 'isAnyOf', value: ['Me', 'usrOther'] }, + { fieldId: modifierField.id().toString(), operator: 'isNot', value: 'Me' }, + { fieldId: titleField.id().toString(), operator: 'is', value: 'Me' }, + ], + }; + + expect(replaceCurrentUserTagInFilter(table, filter, 'usrCurrent')).toEqual({ + conjunction: 'and', + items: [ + { fieldId: ownerField.id().toString(), operator: 'is', value: 'usrCurrent' }, + { + fieldId: creatorField.id().toString(), + operator: 'isAnyOf', + value: ['usrCurrent', 'usrOther'], + }, + { fieldId: modifierField.id().toString(), operator: 'isNot', value: 'usrCurrent' }, + { fieldId: titleField.id().toString(), operator: 'is', value: 'Me' }, + ], + }); + }); }); diff --git a/packages/v2/core/src/queries/RecordFilterMapper.ts b/packages/v2/core/src/queries/RecordFilterMapper.ts index ac43f18b5b..7062e9f41d 100644 --- a/packages/v2/core/src/queries/RecordFilterMapper.ts +++ b/packages/v2/core/src/queries/RecordFilterMapper.ts @@ -5,6 +5,7 @@ import { domainError, type DomainError } from '../domain/shared/DomainError'; import type { ISpecification } from '../domain/shared/specification/ISpecification'; import { notSpec } from '../domain/shared/specification/NotSpec'; import { FieldId } from '../domain/table/fields/FieldId'; +import { FieldType } from '../domain/table/fields/FieldType'; import type { ITableRecordConditionSpecVisitor } from '../domain/table/records/specs/ITableRecordConditionSpecVisitor'; import { RecordConditionSpecBuilder } from '../domain/table/records/specs/RecordConditionSpecBuilder'; import type { RecordConditionValue } from '../domain/table/records/specs/RecordConditionValues'; @@ -28,6 +29,8 @@ import { type RecordFilterValue, } from './RecordFilterDto'; +const currentUserFilterValue = 'Me'; + const resolveField = (table: Table, rawFieldId: string) => { return FieldId.create(rawFieldId).andThen((fieldId) => table @@ -150,6 +153,60 @@ const sanitizeNode = ( return err(domainError.validation({ message: 'Invalid record filter node' })); }; +function isUserLikeFieldType(type: FieldType): boolean { + return ( + type.equals(FieldType.user()) || + type.equals(FieldType.createdBy()) || + type.equals(FieldType.lastModifiedBy()) + ); +} + +export function replaceCurrentUserTagInFilter( + table: Table, + filter: RecordFilter | null | undefined, + actorId: string +): RecordFilter | null | undefined { + if (!filter) { + return filter; + } + + const replaceNode = (node: RecordFilterNode): RecordFilterNode => { + if (isRecordFilterNot(node)) { + return { not: replaceNode(node.not) }; + } + + if (isRecordFilterGroup(node)) { + return { + ...node, + items: node.items.map((item) => replaceNode(item)), + }; + } + + if (!isRecordFilterCondition(node)) { + return node; + } + + const fieldResult = table.getField((field) => field.id().toString() === node.fieldId); + if (fieldResult.isErr() || !isUserLikeFieldType(fieldResult.value.type())) { + return node; + } + + const replaceValue = (value: RecordFilterValue): RecordFilterValue => { + if (Array.isArray(value)) { + return value.map((item) => (item === currentUserFilterValue ? actorId : item)); + } + return value === currentUserFilterValue ? actorId : value; + }; + + return { + ...node, + value: replaceValue(node.value), + }; + }; + + return replaceNode(filter); +} + export const buildRecordConditionSpec = ( table: Table, filter: RecordFilter diff --git a/packages/v2/devtools/src/commands/data-db/index.ts b/packages/v2/devtools/src/commands/data-db/index.ts new file mode 100644 index 0000000000..25eb010a01 --- /dev/null +++ b/packages/v2/devtools/src/commands/data-db/index.ts @@ -0,0 +1 @@ +export { dataDbMigrationStatus } from './migration-status'; diff --git a/packages/v2/devtools/src/commands/data-db/migration-status.ts b/packages/v2/devtools/src/commands/data-db/migration-status.ts new file mode 100644 index 0000000000..6b879f93b5 --- /dev/null +++ b/packages/v2/devtools/src/commands/data-db/migration-status.ts @@ -0,0 +1,82 @@ +import { Command, Options } from '@effect/cli'; +import type { Option } from 'effect'; +import { Effect } from 'effect'; +import { DataDbMigrationInspector } from '../../services/DataDbMigrationInspector'; +import { Output } from '../../services/Output'; +import { connectionOption, limitOption, optionToUndefined, spaceIdOptionalOption } from '../shared'; + +const jobIdOption = Options.text('job-id').pipe( + Options.withDescription('Migration job ID (sdmj...)'), + Options.optional +); + +const includeHistoryOption = Options.boolean('include-history').pipe( + Options.withDefault(false), + Options.withDescription('Include succeeded, failed, canceled, and rolled_back migration jobs') +); + +const handler = (args: { + readonly connection: Option.Option; + readonly spaceId: Option.Option; + readonly jobId: Option.Option; + readonly includeHistory: boolean; + readonly limit: number; +}) => + Effect.gen(function* () { + const inspector = yield* DataDbMigrationInspector; + const output = yield* Output; + const spaceId = optionToUndefined(args.spaceId); + const jobId = optionToUndefined(args.jobId); + const input = { + connection: optionToUndefined(args.connection), + ...(spaceId ? { spaceId } : {}), + ...(jobId ? { jobId } : {}), + includeHistory: args.includeHistory, + limit: args.limit, + }; + + const result = yield* inspector + .getStatus({ + ...(spaceId ? { spaceId } : {}), + ...(jobId ? { jobId } : {}), + includeHistory: args.includeHistory, + limit: args.limit, + }) + .pipe( + Effect.catchAll((error) => + Effect.gen(function* () { + yield* output.error('data-db.migration-status', input, error); + return yield* Effect.fail(error); + }) + ) + ); + + if (result.total === 0) { + yield* output.empty( + 'data-db.migration-status', + input, + jobId + ? 'No migration job matched the requested job ID.' + : 'No active migration jobs matched the filters.' + ); + return; + } + + yield* output.success('data-db.migration-status', input, result); + }); + +export const dataDbMigrationStatus = Command.make( + 'migration-status', + { + connection: connectionOption, + spaceId: spaceIdOptionalOption, + jobId: jobIdOption, + includeHistory: includeHistoryOption, + limit: limitOption, + }, + handler +).pipe( + Command.withDescription( + 'Show per-space BYODB data database migration phase, progress, validation, and rollback status' + ) +); diff --git a/packages/v2/devtools/src/commands/index.ts b/packages/v2/devtools/src/commands/index.ts index 38ca9c0486..165ec9fe42 100644 --- a/packages/v2/devtools/src/commands/index.ts +++ b/packages/v2/devtools/src/commands/index.ts @@ -9,6 +9,7 @@ import { computedTask, computedTasks, } from './computed'; +import { dataDbMigrationStatus } from './data-db'; import { dottea } from './dottea'; import { explainCreate, @@ -77,6 +78,12 @@ export const computed = Command.make('computed').pipe( ]) ); +// data-db subcommand group +export const dataDb = Command.make('data-db').pipe( + Command.withDescription('Inspect data database migration and routing state'), + Command.withSubcommands([dataDbMigrationStatus]) +); + // records subcommand group (application layer queries and mutations) export const records = Command.make('records').pipe( Command.withDescription('Query and mutate records via application layer'), @@ -121,6 +128,7 @@ export const root = Command.make('teable-devtools').pipe( Command.withDescription('Teable v2 developer tools CLI'), Command.withSubcommands([ computed, + dataDb, explain, mock, records, @@ -133,4 +141,4 @@ export const root = Command.make('teable-devtools').pipe( ]) ); -export { dottea, relations }; +export { dataDbMigrationStatus, dottea, relations }; diff --git a/packages/v2/devtools/src/layers/AppLayer.ts b/packages/v2/devtools/src/layers/AppLayer.ts index 31eda948a1..81b84fc836 100644 --- a/packages/v2/devtools/src/layers/AppLayer.ts +++ b/packages/v2/devtools/src/layers/AppLayer.ts @@ -3,6 +3,7 @@ import type { CommandExplain } from '../services/CommandExplain'; import type { ComputedTaskControl } from '../services/ComputedTaskControl'; import type { ComputedTaskInspector } from '../services/ComputedTaskInspector'; import type { DatabaseConfig } from '../services/Database'; +import type { DataDbMigrationInspector } from '../services/DataDbMigrationInspector'; import type { DebugData } from '../services/DebugData'; import type { DotTeaImporter } from '../services/DotTeaImporter'; import type { MockRecords } from '../services/MockRecords'; @@ -16,6 +17,7 @@ import { CommandExplainLive } from './CommandExplainLive'; import { ComputedTaskControlLive } from './ComputedTaskControlLive'; import { ComputedTaskInspectorLive } from './ComputedTaskInspectorLive'; import { DatabaseLive, DatabaseConfigFromOption } from './DatabaseLive'; +import { DataDbMigrationInspectorLive } from './DataDbMigrationInspectorLive'; import { DebugDataLive } from './DebugDataLive'; import { DotTeaImporterLive } from './DotTeaImporterLive'; import { MockRecordsLive } from './MockRecordsLive'; @@ -68,6 +70,7 @@ export const FullLayer = (connectionString?: string) => { CommandExplainLive.pipe(Layer.provide(dbLayer)), ComputedTaskControlLive.pipe(Layer.provide(dbLayer)), ComputedTaskInspectorLive.pipe(Layer.provide(dbLayer)), + DataDbMigrationInspectorLive.pipe(Layer.provide(dbLayer)), MockRecordsLive.pipe(Layer.provide(dbLayer)), SchemaCheckerLive.pipe(Layer.provide(dbLayer)), SchemaOperationControlLive.pipe(Layer.provide(dbLayer)), @@ -87,6 +90,7 @@ export type AppLayerType = Layer.Layer< CommandExplain['Type'] & ComputedTaskControl['Type'] & ComputedTaskInspector['Type'] & + DataDbMigrationInspector['Type'] & MockRecords['Type'] & SchemaChecker['Type'] & SchemaOperationControl['Type'] & diff --git a/packages/v2/devtools/src/layers/DataDbMigrationInspectorLive.ts b/packages/v2/devtools/src/layers/DataDbMigrationInspectorLive.ts new file mode 100644 index 0000000000..8000ea55ab --- /dev/null +++ b/packages/v2/devtools/src/layers/DataDbMigrationInspectorLive.ts @@ -0,0 +1,156 @@ +import { v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; +import { Effect, Layer } from 'effect'; +import type { Kysely } from 'kysely'; +import { sql } from 'kysely'; +import { CliError } from '../errors/CliError'; +import { Database } from '../services/Database'; +import { + DataDbMigrationInspector, + type DataDbMigrationStatusInput, + type DataDbMigrationStatusOutput, + type DataDbMigrationStatusRawRow, + type DataDbMigrationStatusRow, + toDataDbMigrationStatusRow, +} from '../services/DataDbMigrationInspector'; + +const activeMigrationStates = [ + 'pending', + 'preflight', + 'freezing_writes', + 'copying', + 'validating', + 'switching', +]; + +const migrationStatusColumns: ReadonlyArray = [ + 'jobId', + 'spaceId', + 'state', + 'targetMode', + 'targetConnectionId', + 'targetHost', + 'targetDatabase', + 'targetInternalSchema', + 'phase', + 'percent', + 'estimatedTotalBytes', + 'completedEstimatedBytes', + 'estimatedTotalRows', + 'completedEstimatedRows', + 'etaMs', + 'validationPhase', + 'baseCheckCount', + 'sharedCheckCount', + 'mismatchCount', + 'rollbackEligible', + 'rollbackFindingCount', + 'rollbackSwitchedAt', + 'rollbackCheckedAt', + 'lastError', + 'startedAt', + 'completedAt', + 'createdTime', + 'lastModifiedTime', +]; + +const normalizeLimit = (value: number | undefined): number => { + const resolved = value ?? 20; + if (!Number.isFinite(resolved) || resolved <= 0) { + throw new CliError({ + message: 'limit must be a positive number.', + code: 'INVALID_LIMIT', + details: { limit: resolved }, + }); + } + return Math.min(Math.floor(resolved), 500); +}; + +const getStatus = ( + db: Kysely, + input: DataDbMigrationStatusInput +): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const limit = normalizeLimit(input.limit); + const conditions = []; + if (input.jobId) { + conditions.push(sql`j.id = ${input.jobId}`); + } else { + if (input.spaceId) { + conditions.push(sql`j.space_id = ${input.spaceId}`); + } + if (!input.includeHistory) { + conditions.push(sql`j.state::text = ANY(${activeMigrationStates})`); + } + } + + const whereSql = conditions.length ? sql`WHERE ${sql.join(conditions, sql` AND `)}` : sql``; + const result = await sql` + SELECT + j.id AS "jobId", + j.space_id AS "spaceId", + j.state::text AS "state", + j.target_mode AS "targetMode", + j.target_connection_id AS "targetConnectionId", + c.display_host AS "targetHost", + c.display_database AS "targetDatabase", + j.target_internal_schema AS "targetInternalSchema", + j.copy_stats AS "copyStats", + j.validation_stats AS "validationStats", + j.last_error AS "lastError", + j.started_at AS "startedAt", + j.completed_at AS "completedAt", + j.created_time AS "createdTime", + j.last_modified_time AS "lastModifiedTime" + FROM space_data_db_migration_job AS j + LEFT JOIN data_db_connection AS c ON c.id = j.target_connection_id + ${whereSql} + ORDER BY j.created_time DESC + LIMIT ${limit} + `.execute(db); + const rows = result.rows.map(toDataDbMigrationStatusRow); + + return { + snapshotAt: new Date().toISOString(), + filters: { + ...(input.spaceId ? { spaceId: input.spaceId } : {}), + ...(input.jobId ? { jobId: input.jobId } : {}), + includeHistory: Boolean(input.includeHistory), + limit, + }, + total: rows.length, + migrationTable: { + columns: migrationStatusColumns, + rows, + }, + notes: input.jobId + ? [] + : input.includeHistory + ? ['Historical migration jobs are included; output is ordered by created_time desc.'] + : [ + 'Only active migration states are shown. Use --include-history to include terminal jobs.', + ], + }; + }, + catch: (error) => + new CliError({ + message: error instanceof Error ? error.message : String(error), + code: 'DATA_DB_MIGRATION_STATUS_QUERY_FAILED', + details: { + hint: 'Confirm the meta database has the space_data_db_migration_job migration applied.', + }, + }), + }); + +export const DataDbMigrationInspectorLive = Layer.effect( + DataDbMigrationInspector, + Effect.gen(function* () { + const { container } = yield* Database; + const db = container.resolve(v2MetaDbTokens.db) as Kysely; + + return { + getStatus: (input: DataDbMigrationStatusInput) => getStatus(db, input), + }; + }) +); diff --git a/packages/v2/devtools/src/layers/SchemaCheckerLive.ts b/packages/v2/devtools/src/layers/SchemaCheckerLive.ts index 7060106aef..b79fb26e41 100644 --- a/packages/v2/devtools/src/layers/SchemaCheckerLive.ts +++ b/packages/v2/devtools/src/layers/SchemaCheckerLive.ts @@ -1,4 +1,4 @@ -import { v2DataDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { v2DataDbTokens, v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { createSchemaChecker, PostgresSchemaIntrospector, @@ -36,6 +36,7 @@ export const SchemaCheckerLive = Layer.effect( try: async () => { const tableRepo = container.resolve(v2CoreTokens.tableRepository) as ITableRepository; const db = container.resolve(v2DataDbTokens.db) as Kysely; + const metaDb = container.resolve(v2MetaDbTokens.db) as Kysely; const actorIdResult = ActorId.create('cli-schema-checker'); if (actorIdResult.isErr()) throw actorIdResult.error; const context = { actorId: actorIdResult.value }; @@ -54,6 +55,7 @@ export const SchemaCheckerLive = Layer.effect( const introspector = new PostgresSchemaIntrospector(db); const checker = createSchemaChecker({ db, + metaDb, introspector, schema, }); @@ -104,6 +106,7 @@ export const SchemaCheckerLive = Layer.effect( try: async () => { const tableRepo = container.resolve(v2CoreTokens.tableRepository) as ITableRepository; const db = container.resolve(v2DataDbTokens.db) as Kysely; + const metaDb = container.resolve(v2MetaDbTokens.db) as Kysely; const actorIdResult = ActorId.create('cli-schema-checker'); if (actorIdResult.isErr()) throw actorIdResult.error; const context = { actorId: actorIdResult.value }; @@ -122,6 +125,7 @@ export const SchemaCheckerLive = Layer.effect( const introspector = new PostgresSchemaIntrospector(db); const checker = createSchemaChecker({ db, + metaDb, introspector, schema, }); diff --git a/packages/v2/devtools/src/layers/SchemaRepairerLive.ts b/packages/v2/devtools/src/layers/SchemaRepairerLive.ts index c3ca5dedd1..cbb97581e1 100644 --- a/packages/v2/devtools/src/layers/SchemaRepairerLive.ts +++ b/packages/v2/devtools/src/layers/SchemaRepairerLive.ts @@ -1,4 +1,4 @@ -import { v2DataDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { v2DataDbTokens, v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { createSchemaRepairer, PostgresSchemaIntrospector, @@ -94,9 +94,11 @@ export const SchemaRepairerLive = Layer.effect( const table = await loadTable(tableId); const schema = table.baseId().toString(); const db = container.resolve(v2DataDbTokens.db) as Kysely; + const metaDb = container.resolve(v2MetaDbTokens.db) as Kysely; const introspector = new PostgresSchemaIntrospector(db); const repairer = createSchemaRepairer({ db, + metaDb, introspector, schema, }); diff --git a/packages/v2/devtools/src/layers/index.ts b/packages/v2/devtools/src/layers/index.ts index 6d3ee02e79..5abd93bfbe 100644 --- a/packages/v2/devtools/src/layers/index.ts +++ b/packages/v2/devtools/src/layers/index.ts @@ -3,6 +3,7 @@ export * from './DebugDataLive'; export * from './CommandExplainLive'; export * from './ComputedTaskControlLive'; export * from './ComputedTaskInspectorLive'; +export * from './DataDbMigrationInspectorLive'; export * from './MockRecordsLive'; export * from './NodeCryptoHasher'; export * from './OutputLive'; diff --git a/packages/v2/devtools/src/services/DataDbMigrationInspector.ts b/packages/v2/devtools/src/services/DataDbMigrationInspector.ts new file mode 100644 index 0000000000..7c6ec7ff50 --- /dev/null +++ b/packages/v2/devtools/src/services/DataDbMigrationInspector.ts @@ -0,0 +1,167 @@ +import type { Effect } from 'effect'; +import { Context } from 'effect'; +import type { CliError } from '../errors'; +import type { CliTable } from './ComputedTaskInspector'; + +export interface DataDbMigrationStatusInput { + readonly spaceId?: string; + readonly jobId?: string; + readonly includeHistory?: boolean; + readonly limit?: number; +} + +export interface DataDbMigrationStatusRawRow { + readonly jobId: string; + readonly spaceId: string; + readonly state: string; + readonly targetMode: string; + readonly targetConnectionId: string | null; + readonly targetHost: string | null; + readonly targetDatabase: string | null; + readonly targetInternalSchema: string; + readonly copyStats: unknown; + readonly validationStats: unknown; + readonly lastError: string | null; + readonly startedAt: Date | string | null; + readonly completedAt: Date | string | null; + readonly createdTime: Date | string; + readonly lastModifiedTime: Date | string | null; +} + +export interface DataDbMigrationStatusRow { + readonly jobId: string; + readonly spaceId: string; + readonly state: string; + readonly targetMode: string; + readonly targetConnectionId: string | null; + readonly targetHost: string | null; + readonly targetDatabase: string | null; + readonly targetInternalSchema: string; + readonly phase: string | null; + readonly percent: number | null; + readonly estimatedTotalBytes: number | null; + readonly completedEstimatedBytes: number | null; + readonly estimatedTotalRows: number | null; + readonly completedEstimatedRows: number | null; + readonly etaMs: number | null; + readonly validationPhase: string | null; + readonly baseCheckCount: number | null; + readonly sharedCheckCount: number | null; + readonly mismatchCount: number | null; + readonly rollbackEligible: boolean | null; + readonly rollbackFindingCount: number | null; + readonly rollbackSwitchedAt: string | null; + readonly rollbackCheckedAt: string | null; + readonly lastError: string | null; + readonly startedAt: string | null; + readonly completedAt: string | null; + readonly createdTime: string; + readonly lastModifiedTime: string | null; +} + +export interface DataDbMigrationStatusOutput { + readonly snapshotAt: string; + readonly filters: { + readonly spaceId?: string; + readonly jobId?: string; + readonly includeHistory: boolean; + readonly limit: number; + }; + readonly total: number; + readonly migrationTable: CliTable; + readonly notes: ReadonlyArray; +} + +export class DataDbMigrationInspector extends Context.Tag('DataDbMigrationInspector')< + DataDbMigrationInspector, + { + readonly getStatus: ( + input: DataDbMigrationStatusInput + ) => Effect.Effect; + } +>() {} + +const toIso = (value: Date | string | null | undefined): string | null => { + if (!value) { + return null; + } + if (value instanceof Date) { + return value.toISOString(); + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); +}; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +const parseJsonRecord = (value: unknown): Record | undefined => { + if (isRecord(value)) { + return value; + } + if (typeof value !== 'string') { + return undefined; + } + + try { + const parsed = JSON.parse(value) as unknown; + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +}; + +const toNumberOrNull = (value: unknown): number | null => + typeof value === 'number' && Number.isFinite(value) ? value : null; + +const getProgress = (stats: Record | undefined) => + parseJsonRecord(stats?.progress); + +export const toDataDbMigrationStatusRow = ( + row: DataDbMigrationStatusRawRow +): DataDbMigrationStatusRow => { + const copyStats = parseJsonRecord(row.copyStats); + const validationStats = parseJsonRecord(row.validationStats); + const progress = getProgress(copyStats) ?? getProgress(validationStats); + const rollback = parseJsonRecord(validationStats?.rollback); + + return { + jobId: row.jobId, + spaceId: row.spaceId, + state: row.state, + targetMode: row.targetMode, + targetConnectionId: row.targetConnectionId, + targetHost: row.targetHost, + targetDatabase: row.targetDatabase, + targetInternalSchema: row.targetInternalSchema, + phase: + (typeof progress?.phase === 'string' ? progress.phase : null) ?? + (typeof copyStats?.phase === 'string' ? copyStats.phase : null) ?? + (typeof validationStats?.phase === 'string' ? validationStats.phase : null), + percent: toNumberOrNull(progress?.percent), + estimatedTotalBytes: toNumberOrNull(progress?.estimatedTotalBytes), + completedEstimatedBytes: toNumberOrNull(progress?.completedEstimatedBytes), + estimatedTotalRows: toNumberOrNull(progress?.estimatedTotalRows), + completedEstimatedRows: toNumberOrNull(progress?.completedEstimatedRows), + etaMs: toNumberOrNull(progress?.etaMs), + validationPhase: typeof validationStats?.phase === 'string' ? validationStats.phase : null, + baseCheckCount: Array.isArray(validationStats?.baseSchemas) + ? validationStats.baseSchemas.length + : null, + sharedCheckCount: Array.isArray(validationStats?.sharedTables) + ? validationStats.sharedTables.length + : null, + mismatchCount: Array.isArray(validationStats?.mismatches) + ? validationStats.mismatches.length + : null, + rollbackEligible: typeof rollback?.eligible === 'boolean' ? rollback.eligible : null, + rollbackFindingCount: Array.isArray(rollback?.findings) ? rollback.findings.length : null, + rollbackSwitchedAt: typeof rollback?.switchedAt === 'string' ? rollback.switchedAt : null, + rollbackCheckedAt: typeof rollback?.checkedAt === 'string' ? rollback.checkedAt : null, + lastError: row.lastError, + startedAt: toIso(row.startedAt), + completedAt: toIso(row.completedAt), + createdTime: toIso(row.createdTime) ?? '', + lastModifiedTime: toIso(row.lastModifiedTime), + }; +}; diff --git a/packages/v2/devtools/src/services/index.ts b/packages/v2/devtools/src/services/index.ts index 1b30bacb2f..c25cee79fb 100644 --- a/packages/v2/devtools/src/services/index.ts +++ b/packages/v2/devtools/src/services/index.ts @@ -3,6 +3,7 @@ export * from './DebugData'; export * from './CommandExplain'; export * from './ComputedTaskControl'; export * from './ComputedTaskInspector'; +export * from './DataDbMigrationInspector'; export * from './MockRecords'; export * from './Output'; export * from './SchemaChecker'; diff --git a/packages/v2/dottea/src/index.ts b/packages/v2/dottea/src/index.ts index 6706f9941e..71f559f353 100644 --- a/packages/v2/dottea/src/index.ts +++ b/packages/v2/dottea/src/index.ts @@ -8,7 +8,7 @@ import { type IDotTeaParser, type NormalizedDotTeaStructure, } from '@teable/v2-core'; -import { normalizeField } from './normalizer'; +import { normalizeFields } from './normalizer'; import { injectable } from '@teable/v2-di'; import { err, ok } from 'neverthrow'; import type { Result } from 'neverthrow'; @@ -130,18 +130,12 @@ export class DotTeaParser implements IDotTeaParser { ); // Normalize all tables and their fields using table-local field IDs so - // legacy formulas that reference deleted fields can be downgraded safely. + // legacy formulas and relation dependencies can be downgraded safely. const normalizedTables = structure.tables.map((table) => { - const tableFieldTypesById = new Map( - table.fields.filter((field) => field.id).map((field) => [field.id!, field.type] as const) - ); - return { ...(table.id ? { id: table.id } : {}), name: table.name, - fields: table.fields.map((field) => - normalizeField(field, tableFieldTypesById, { availableTableIds, fieldIdsByTableId }) - ), + fields: normalizeFields(table.fields, { availableTableIds, fieldIdsByTableId }), views: table.views ?.filter((view) => (view.type ? allowedViewTypes.has(view.type) : true)) .map((view) => ({ diff --git a/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.spec.ts b/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.spec.ts index 92961a4502..dc0e1721c3 100644 --- a/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.spec.ts +++ b/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.spec.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest'; import { buildTableFromInput } from '../../../core/src/commands/TableInputParser'; import { FormulaField } from '../../../core/src/domain/table/fields/types/FormulaField'; import type { ITableFieldInput } from '../../../core/src/schemas/field'; -import { normalizeField } from './DotTeaFieldNormalizer'; +import { normalizeField, normalizeFields } from './DotTeaFieldNormalizer'; describe('DotTeaFieldNormalizer', () => { it('deduplicates select choices that only differ by surrounding whitespace', () => { @@ -301,6 +301,73 @@ describe('DotTeaFieldNormalizer', () => { }); }); + it('downgrades lookups that depend on links downgraded during the same normalization pass', () => { + const hostTableId = `tbl${'h'.repeat(16)}`; + const foreignTableId = `tbl${'f'.repeat(16)}`; + const linkFieldId = `fld${'l'.repeat(16)}`; + const dependentLookupFieldId = `fld${'u'.repeat(16)}`; + const lookupFieldId = `fld${'v'.repeat(16)}`; + const missingVisibleFieldId = `fld${'z'.repeat(16)}`; + + const normalized = normalizeFields( + [ + { + id: linkFieldId, + type: 'link', + name: 'Broken Link', + options: { + relationship: 'manyOne', + foreignTableId, + lookupFieldId, + visibleFieldIds: [missingVisibleFieldId], + }, + }, + { + id: dependentLookupFieldId, + type: 'singleLineText', + isLookup: true, + name: 'Lookup Through Broken Link', + lookupOptions: { + foreignTableId, + linkFieldId, + lookupFieldId, + }, + }, + ], + { + availableTableIds: new Set([hostTableId, foreignTableId]), + fieldIdsByTableId: new Map([[foreignTableId, new Set([lookupFieldId])]]), + } + ); + + expect(normalized).toEqual([ + expect.objectContaining({ id: linkFieldId, type: 'singleLineText' }), + expect.objectContaining({ id: dependentLookupFieldId, type: 'singleLineText' }), + ]); + + const result = buildTableFromInput({ + baseId: `bse${'b'.repeat(16)}`, + tableId: hostTableId, + name: 'Import Test', + fields: [ + { + id: `fld${'p'.repeat(16)}`, + type: 'singleLineText', + name: 'Name', + isPrimary: true, + }, + ...normalized.map((field) => ({ + id: field.id, + type: field.type, + name: field.name, + options: field.options, + })), + ] as ITableFieldInput[], + }); + + expect(result.isOk()).toBe(true); + }); + it('preserves required many-one link fields from dottea exports', () => { const hostTableId = `tbl${'h'.repeat(16)}`; const foreignTableId = `tbl${'f'.repeat(16)}`; diff --git a/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.ts b/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.ts index 905dc486d8..9b98b6edb9 100644 --- a/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.ts +++ b/packages/v2/dottea/src/normalizer/DotTeaFieldNormalizer.ts @@ -152,6 +152,7 @@ type NormalizedFieldOptions = { export type NormalizeFieldOptions = { readonly availableTableIds?: ReadonlySet; readonly fieldIdsByTableId?: ReadonlyMap>; + readonly hostFieldTypesById?: ReadonlyMap; }; const referencesMissingForeignTable = ( @@ -209,6 +210,22 @@ const referencesMissingHostLinkField = ( fieldTypesById: ReadonlyMap ): boolean => Boolean(linkFieldId && fieldTypesById.get(linkFieldId) !== 'link'); +const exportedFieldType = (field: DotTeaFieldInput): string => + field.isConditionalLookup ? 'conditionalLookup' : field.isLookup ? 'lookup' : field.type; + +const normalizedFieldType = (field: NormalizedDotTeaField): string => field.type; + +const areFieldTypeMapsEqual = ( + left: ReadonlyMap, + right: ReadonlyMap +): boolean => { + if (left.size !== right.size) return false; + for (const [key, value] of left) { + if (right.get(key) !== value) return false; + } + return true; +}; + /** * Normalize a field's options from v1 (dottea) format to v2 format. * This handles the conversion of link, lookup, formula, rollup, conditionalRollup, @@ -229,6 +246,7 @@ export const normalizeFieldOptions = ( ? normalizeSelectChoices(rawOptions) : rawOptions; const rawLookupOptions = asRecord(field.lookupOptions); + const hostFieldTypesById = options?.hostFieldTypesById ?? fieldTypesById; const lookupOptions = normalizeLookupOptions(rawLookupOptions); // v1 exports lookup fields using the looked-up value type plus isLookup=true. @@ -266,7 +284,10 @@ export const normalizeFieldOptions = ( } if ( referencesMissingForeignTable(readString(lookupOptions, 'foreignTableId'), options) || - referencesMissingHostLinkField(readString(lookupOptions, 'linkFieldId'), fieldTypesById) || + referencesMissingHostLinkField( + readString(lookupOptions, 'linkFieldId'), + hostFieldTypesById + ) || referencesMissingForeignField( readString(lookupOptions, 'foreignTableId'), readString(lookupOptions, 'lookupFieldId'), @@ -288,11 +309,7 @@ export const normalizeFieldOptions = ( readString(linkOptions, 'lookupFieldId'), options ) || - referencesMissingVisibleField( - readString(linkOptions, 'foreignTableId'), - linkOptions, - options - ) + referencesMissingVisibleField(readString(linkOptions, 'foreignTableId'), linkOptions, options) ) { return { type: 'singleLineText', options: normalizedSelectOptions }; } @@ -304,7 +321,10 @@ export const normalizeFieldOptions = ( return formulaOptions && lookupOptions && !referencesMissingForeignTable(readString(lookupOptions, 'foreignTableId'), options) && - !referencesMissingHostLinkField(readString(lookupOptions, 'linkFieldId'), fieldTypesById) && + !referencesMissingHostLinkField( + readString(lookupOptions, 'linkFieldId'), + hostFieldTypesById + ) && !referencesMissingForeignField( readString(lookupOptions, 'foreignTableId'), readString(lookupOptions, 'lookupFieldId'), @@ -403,3 +423,30 @@ export const normalizeField = ( return baseField; }; + +export const normalizeFields = ( + fields: ReadonlyArray, + options?: NormalizeFieldOptions +): ReadonlyArray => { + const rawFieldTypesById = new Map( + fields.filter((field) => field.id).map((field) => [field.id!, exportedFieldType(field)]) + ); + let hostFieldTypesById: ReadonlyMap = rawFieldTypesById; + + for (let i = 0; i <= fields.length; i++) { + const normalized = fields.map((field) => + normalizeField(field, rawFieldTypesById, { ...options, hostFieldTypesById }) + ); + const nextHostFieldTypesById = new Map( + normalized.filter((field) => field.id).map((field) => [field.id!, normalizedFieldType(field)]) + ); + if (areFieldTypeMapsEqual(hostFieldTypesById, nextHostFieldTypesById)) { + return normalized; + } + hostFieldTypesById = nextHostFieldTypesById; + } + + return fields.map((field) => + normalizeField(field, rawFieldTypesById, { ...options, hostFieldTypesById }) + ); +}; diff --git a/packages/v2/dottea/src/normalizer/index.ts b/packages/v2/dottea/src/normalizer/index.ts index 39e74f8c66..7268cdefaf 100644 --- a/packages/v2/dottea/src/normalizer/index.ts +++ b/packages/v2/dottea/src/normalizer/index.ts @@ -1,5 +1,6 @@ export { normalizeFieldOptions, normalizeField, + normalizeFields, extractFieldReferences, } from './DotTeaFieldNormalizer'; diff --git a/packages/v2/e2e/src/clear.e2e.spec.ts b/packages/v2/e2e/src/clear.e2e.spec.ts index 73895b3c43..e853220a78 100644 --- a/packages/v2/e2e/src/clear.e2e.spec.ts +++ b/packages/v2/e2e/src/clear.e2e.spec.ts @@ -5,7 +5,9 @@ import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { getSharedTestContext, type SharedTestContext } from './shared/globalTestContext'; import { setupGroupedLinkRangeFixture, + setupGroupedSingleSelectRangeFixture, type GroupedLinkRangeFixture, + type GroupedSingleSelectRangeFixture, } from './shared/groupedLinkRangeFixture'; /** @@ -271,6 +273,142 @@ describe('v2 http clear (e2e)', () => { }); }); + describe('clear with wide projection', () => { + it('should clear the populated row when a blank row is directly above it', async () => { + const table = await ctx.createTable({ + baseId: ctx.baseId, + name: `Wide Clear Projection ${Date.now()}`, + fields: [ + { name: 'Primary Label', type: 'singleLineText', isPrimary: true }, + { name: 'Context A', type: 'singleLineText' }, + { name: 'Context B', type: 'singleLineText' }, + { name: 'Row Key', type: 'singleLineText' }, + { name: 'Tag', type: 'singleLineText' }, + { name: 'Reference', type: 'singleLineText' }, + { name: 'Target Text', type: 'singleLineText' }, + { name: 'Target Number', type: 'number' }, + { name: 'Target Owner', type: 'singleLineText' }, + ], + views: [{ type: 'grid' }], + }); + + const viewId = table.views[0].id; + const fieldByName = new Map(table.fields.map((field) => [field.name, field.id])); + const primaryFieldId = fieldByName.get('Primary Label') ?? ''; + const contextAFieldId = fieldByName.get('Context A') ?? ''; + const contextBFieldId = fieldByName.get('Context B') ?? ''; + const rowKeyFieldId = fieldByName.get('Row Key') ?? ''; + const tagFieldId = fieldByName.get('Tag') ?? ''; + const referenceFieldId = fieldByName.get('Reference') ?? ''; + const targetTextFieldId = fieldByName.get('Target Text') ?? ''; + const targetNumberFieldId = fieldByName.get('Target Number') ?? ''; + const targetOwnerFieldId = fieldByName.get('Target Owner') ?? ''; + + const projection = [ + primaryFieldId, + contextAFieldId, + contextBFieldId, + rowKeyFieldId, + tagFieldId, + referenceFieldId, + targetTextFieldId, + targetNumberFieldId, + targetOwnerFieldId, + ]; + + await ctx.createRecords(table.id, [ + { + fields: { + [primaryFieldId]: 'row-1', + [contextAFieldId]: 'context-a-1', + [rowKeyFieldId]: 'key-1', + [tagFieldId]: 'tag-1', + [referenceFieldId]: 'ref-1', + [targetTextFieldId]: 'unrelated text', + [targetNumberFieldId]: 1, + [targetOwnerFieldId]: 'owner-1', + }, + }, + { + fields: { + [primaryFieldId]: 'row-2', + [rowKeyFieldId]: 'key-2', + }, + }, + { + fields: { + [primaryFieldId]: 'row-3', + [rowKeyFieldId]: 'key-3', + }, + }, + { + fields: { + [primaryFieldId]: 'row-before-target', + [contextAFieldId]: 'context-before-target', + [rowKeyFieldId]: 'key-before-target', + }, + }, + { + fields: { + [primaryFieldId]: 'target-row', + [contextAFieldId]: 'context-target', + [contextBFieldId]: 'context-b-target', + [rowKeyFieldId]: 'key-target', + [tagFieldId]: 'tag-target', + [referenceFieldId]: 'ref-target', + [targetTextFieldId]: 'target text', + [targetNumberFieldId]: 2, + [targetOwnerFieldId]: 'target owner', + }, + }, + { + fields: { + [primaryFieldId]: 'row-after-target', + [contextAFieldId]: 'context-after-target', + [contextBFieldId]: 'context-b-after-target', + [rowKeyFieldId]: 'key-after-target', + [tagFieldId]: 'tag-after-target', + [referenceFieldId]: 'ref-after-target', + [targetTextFieldId]: 'after target text', + [targetNumberFieldId]: 3, + [targetOwnerFieldId]: 'after target owner', + }, + }, + ]); + + const result = await ctx.clear({ + tableId: table.id, + viewId, + ranges: [ + [6, 4], + [8, 4], + ], + projection, + }); + + expect(result.updatedCount).toBe(1); + + const records = await ctx.listRecords(table.id); + const blankAboveTarget = records.find( + (record) => record.fields[rowKeyFieldId] === 'key-before-target' + ); + const target = records.find((record) => record.fields[rowKeyFieldId] === 'key-target'); + const nextRecord = records.find( + (record) => record.fields[rowKeyFieldId] === 'key-after-target' + ); + + expect(blankAboveTarget?.fields[targetTextFieldId]).toBeNull(); + expect(blankAboveTarget?.fields[targetNumberFieldId]).toBeNull(); + expect(blankAboveTarget?.fields[targetOwnerFieldId]).toBeNull(); + expect(target?.fields[targetTextFieldId]).toBeNull(); + expect(target?.fields[targetNumberFieldId]).toBeNull(); + expect(target?.fields[targetOwnerFieldId]).toBeNull(); + expect(nextRecord?.fields[targetTextFieldId]).toBe('after target text'); + expect(nextRecord?.fields[targetNumberFieldId]).toBe(3); + expect(nextRecord?.fields[targetOwnerFieldId]).toBe('after target owner'); + }); + }); + describe('clear with filter', () => { let tableId: string; let viewId: string; @@ -410,6 +548,59 @@ describe('v2 http clear (e2e)', () => { }); }); + describe('clear with grouped singleSelect view order parity', () => { + const expectOnlySecondVisibleRowCleared = async (fixture: GroupedSingleSelectRangeFixture) => { + const records = await ctx.listRecords(fixture.tableId); + const order2 = records.find((record) => record.id === fixture.recordIds.order2); + const order3 = records.find((record) => record.id === fixture.recordIds.order3); + const order4 = records.find((record) => record.id === fixture.recordIds.order4); + + // Visible order follows the view row order (reverse of creation), so the + // second visible row is 加单3. + expect(order4?.fields[fixture.nameFieldId]).toBe('加单4'); + expect(order3?.fields[fixture.nameFieldId]).toBeNull(); + expect(order2?.fields[fixture.nameFieldId]).toBe('加单2'); + }; + + it('should clear the visible row in a saved grouped view when row order differs', async () => { + const fixture = await setupGroupedSingleSelectRangeFixture(ctx, 'clear-saved-group', { + persistViewQuery: true, + }); + + const result = await ctx.clear({ + tableId: fixture.tableId, + viewId: fixture.viewId, + ranges: [ + [0, 1], + [0, 1], + ], + }); + + expect(result.updatedCount).toBe(1); + await expectOnlySecondVisibleRowCleared(fixture); + }); + + it('should clear the visible row in a personal grouped view request', async () => { + const fixture = await setupGroupedSingleSelectRangeFixture(ctx, 'clear-personal-group'); + + const result = await ctx.clear({ + tableId: fixture.tableId, + viewId: fixture.viewId, + ranges: [ + [0, 1], + [0, 1], + ], + ignoreViewQuery: true, + groupBy: fixture.groupByAsc, + sort: fixture.sortAsc, + projection: fixture.projection, + }); + + expect(result.updatedCount).toBe(1); + await expectOnlySecondVisibleRowCleared(fixture); + }); + }); + describe('clear with sort (view row order)', () => { /** * Critical test for ensuring clear operations target the correct rows @@ -1041,11 +1232,11 @@ describe('v2 http clear (e2e)', () => { it('should clear correct row at large offset when sort values tie', async () => { const targetOffset = 400; - const orderColumn = `__row_${tieViewId}`; + // All sort values tie, so the visible order follows the view row order column. const expected = await sql<{ __id: string }>` SELECT "__id" FROM ${sql.table(tieDbTableName)} - ORDER BY ${sql.ref(orderColumn)} ASC + ORDER BY ${sql.ref(`__row_${tieViewId}`)} ASC OFFSET ${targetOffset} LIMIT 1 `.execute(ctx.testContainer.db); diff --git a/packages/v2/e2e/src/computed-record-events-realtime.e2e.spec.ts b/packages/v2/e2e/src/computed-record-events-realtime.e2e.spec.ts index 3db89b849f..142557f993 100644 --- a/packages/v2/e2e/src/computed-record-events-realtime.e2e.spec.ts +++ b/packages/v2/e2e/src/computed-record-events-realtime.e2e.spec.ts @@ -18,6 +18,7 @@ import { NoopLogger } from '@teable/v2-core'; import type { ICreateTableCommandInput } from '@teable/v2-core'; import type { DependencyContainer } from '@teable/v2-di'; import express from 'express'; +import { sql } from 'kysely'; import ShareDb from 'sharedb'; import type { Doc } from 'sharedb/lib/client'; import { Connection } from 'sharedb/lib/client'; @@ -407,6 +408,105 @@ describe('v2 computed record events and realtime projection (e2e)', () => { }; }, }, + { + label: 'lookup multi-user', + setup: async () => { + const aliceEmail = 'alice+realtime-lookup@e2e.com'; + const bobEmail = 'bob+realtime-lookup@e2e.com'; + const aliceCell = { + id: 'usrRealtimeLookupAlice', + title: 'Alice', + email: aliceEmail, + avatarUrl: '/api/attachments/read/public/avatar/usrRealtimeLookupAlice', + }; + const bobCell = { + id: 'usrRealtimeLookupBob', + title: 'Bob', + email: bobEmail, + avatarUrl: '/api/attachments/read/public/avatar/usrRealtimeLookupBob', + }; + await sql` + insert into users (id, name, email) + values + (${aliceCell.id}, ${aliceCell.title}, ${aliceEmail}), + (${bobCell.id}, ${bobCell.title}, ${bobEmail}) + on conflict (id) do nothing + `.execute(testContainer.db); + + const foreignPrimaryFieldId = createFieldId(); + const foreignUserFieldId = createFieldId(); + const foreignTable = await createTable({ + baseId, + name: `Computed Event Lookup User Foreign ${fieldIdCounter}`, + fields: [ + { type: 'singleLineText', id: foreignPrimaryFieldId, name: 'Name', isPrimary: true }, + { + type: 'user', + id: foreignUserFieldId, + name: 'Members', + options: { isMultiple: true, shouldNotify: false }, + }, + ], + views: [{ type: 'grid' }], + }); + + const hostPrimaryFieldId = createFieldId(); + const linkFieldId = createFieldId(); + const lookupFieldId = createFieldId(); + const hostTable = await createTable({ + baseId, + name: `Computed Event Lookup User Host ${fieldIdCounter}`, + fields: [ + { type: 'singleLineText', id: hostPrimaryFieldId, name: 'Name', isPrimary: true }, + { + type: 'link', + id: linkFieldId, + name: 'Link', + options: { + relationship: 'manyOne', + foreignTableId: foreignTable.id, + lookupFieldId: foreignPrimaryFieldId, + isOneWay: true, + }, + }, + { + type: 'lookup', + id: lookupFieldId, + name: 'Lookup Members', + options: { + linkFieldId, + foreignTableId: foreignTable.id, + lookupFieldId: foreignUserFieldId, + }, + }, + ], + views: [{ type: 'grid' }], + }); + + const foreignRecord = await createRecord(foreignTable.id, { + [foreignPrimaryFieldId]: 'Source 1', + [foreignUserFieldId]: [aliceCell], + }); + const hostRecord = await createRecord(hostTable.id, { + [hostPrimaryFieldId]: 'Host 1', + [linkFieldId]: { id: foreignRecord.id }, + }); + await drainOutbox(testContainer); + + return { + hostTableId: hostTable.id, + hostRecordId: hostRecord.id, + computedFieldId: lookupFieldId, + initialValue: [aliceCell], + expectedValue: [aliceCell, bobCell], + trigger: async () => { + await updateRecord(foreignTable.id, foreignRecord.id, { + [foreignUserFieldId]: [aliceCell, bobCell], + }); + }, + }; + }, + }, { label: 'rollup', setup: async () => { diff --git a/packages/v2/e2e/src/create-field/conditionalRollup/conditionalRollup.spec.ts b/packages/v2/e2e/src/create-field/conditionalRollup/conditionalRollup.spec.ts index 9b609b01d4..b59d7f8f51 100644 --- a/packages/v2/e2e/src/create-field/conditionalRollup/conditionalRollup.spec.ts +++ b/packages/v2/e2e/src/create-field/conditionalRollup/conditionalRollup.spec.ts @@ -130,4 +130,189 @@ describe('create-field: conditionalRollup v1 parity', () => { if (tagsTableId) await ctx.deleteTable(tagsTableId).catch(() => undefined); } }); + + test('evaluates equality filter comparing lookup link titles to host text', async () => { + let unitsTableId: string | undefined; + let accountsTableId: string | undefined; + let refundsTableId: string | undefined; + let hostTableId: string | undefined; + + try { + const unitsTable = await ctx.createTable({ + baseId: ctx.baseId, + name: nextName('v2-create-reg-units'), + fields: [{ type: 'singleLineText', name: 'Name', isPrimary: true }], + }); + unitsTableId = unitsTable.id; + const unitNameId = unitsTable.fields.find((f) => f.isPrimary)?.id; + if (!unitNameId) throw new Error('Missing unit primary field'); + + const unitAId = (await ctx.createRecord(unitsTable.id, { [unitNameId]: 'UnitA' })).id; + const unitBId = (await ctx.createRecord(unitsTable.id, { [unitNameId]: 'UnitB' })).id; + + const accountsTable = await ctx.createTable({ + baseId: ctx.baseId, + name: nextName('v2-create-reg-accounts'), + fields: [{ type: 'singleLineText', name: 'Account', isPrimary: true }], + }); + accountsTableId = accountsTable.id; + const accountNameId = accountsTable.fields.find((f) => f.isPrimary)?.id; + if (!accountNameId) throw new Error('Missing account primary field'); + + const accountsWithUnitLink = await ctx.createField({ + baseId: ctx.baseId, + tableId: accountsTable.id, + field: { + type: 'link', + name: 'Unit', + options: { + relationship: 'manyOne', + foreignTableId: unitsTable.id, + lookupFieldId: unitNameId, + isOneWay: true, + }, + }, + }); + const accountUnitLinkId = accountsWithUnitLink.fields.find((f) => f.name === 'Unit')?.id; + if (!accountUnitLinkId) throw new Error('Missing account unit link field'); + + const accountAId = ( + await ctx.createRecord(accountsTable.id, { + [accountNameId]: 'AccountA', + [accountUnitLinkId]: { id: unitAId }, + }) + ).id; + const accountBId = ( + await ctx.createRecord(accountsTable.id, { + [accountNameId]: 'AccountB', + [accountUnitLinkId]: { id: unitBId }, + }) + ).id; + + const refundsTable = await ctx.createTable({ + baseId: ctx.baseId, + name: nextName('v2-create-reg-refunds'), + fields: [ + { type: 'singleLineText', name: 'Title', isPrimary: true }, + { type: 'number', name: 'Amount' }, + ], + }); + refundsTableId = refundsTable.id; + const refundAmountId = refundsTable.fields.find((f) => f.name === 'Amount')?.id; + if (!refundAmountId) throw new Error('Missing refund amount field'); + + const refundsWithAccountLink = await ctx.createField({ + baseId: ctx.baseId, + tableId: refundsTable.id, + field: { + type: 'link', + name: 'Account', + options: { + relationship: 'manyOne', + foreignTableId: accountsTable.id, + lookupFieldId: accountNameId, + isOneWay: true, + }, + }, + }); + const refundAccountLinkId = refundsWithAccountLink.fields.find( + (f) => f.name === 'Account' + )?.id; + if (!refundAccountLinkId) throw new Error('Missing refund account link field'); + + const refundsWithUnitLookup = await ctx.createField({ + baseId: ctx.baseId, + tableId: refundsTable.id, + field: { + type: 'lookup', + name: 'Unit Lookup', + options: { + linkFieldId: refundAccountLinkId, + foreignTableId: accountsTable.id, + lookupFieldId: accountUnitLinkId, + }, + }, + }); + const refundUnitLookupId = refundsWithUnitLookup.fields.find( + (f) => f.name === 'Unit Lookup' + )?.id; + if (!refundUnitLookupId) throw new Error('Missing refund unit lookup field'); + + await ctx.createRecord(refundsTable.id, { + Title: 'r1', + Amount: 10, + [refundAccountLinkId]: { id: accountAId }, + }); + await ctx.createRecord(refundsTable.id, { + Title: 'r2', + Amount: 20, + [refundAccountLinkId]: { id: accountBId }, + }); + await ctx.createRecord(refundsTable.id, { + Title: 'r3', + Amount: 5, + [refundAccountLinkId]: { id: accountAId }, + }); + + const hostTable = await ctx.createTable({ + baseId: ctx.baseId, + name: nextName('v2-create-reg-unit-host'), + fields: [{ type: 'singleLineText', name: 'UnitName', isPrimary: true }], + }); + hostTableId = hostTable.id; + const hostUnitNameId = hostTable.fields.find((f) => f.name === 'UnitName')?.id; + if (!hostUnitNameId) throw new Error('Missing host unit text field'); + + await ctx.createRecord(hostTable.id, { [hostUnitNameId]: 'UnitA' }); + await ctx.createRecord(hostTable.id, { [hostUnitNameId]: 'UnitB' }); + await ctx.createRecord(hostTable.id, { [hostUnitNameId]: 'UnitC' }); + + const tableWithRollup = await ctx.createField({ + baseId: ctx.baseId, + tableId: hostTable.id, + field: { + type: 'conditionalRollup', + name: 'Sum By Lookup Unit', + options: { expression: 'sum({values})' }, + config: { + foreignTableId: refundsTable.id, + lookupFieldId: refundAmountId, + condition: { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: refundUnitLookupId, + operator: 'is', + value: hostUnitNameId, + isSymbol: true, + }, + ], + }, + }, + }, + }, + }); + const rollupFieldId = tableWithRollup.fields.find((f) => f.name === 'Sum By Lookup Unit')?.id; + if (!rollupFieldId) throw new Error('Missing conditional rollup field'); + + await ctx.drainOutbox(); + + const hostRecords = await ctx.listRecords(hostTable.id, { limit: 10, offset: 0 }); + const byUnitName = new Map( + hostRecords.map((record) => [ + String(record.fields[hostUnitNameId] ?? ''), + record.fields[rollupFieldId], + ]) + ); + expect(byUnitName.get('UnitA')).toBe(15); + expect(byUnitName.get('UnitB')).toBe(20); + expect(byUnitName.get('UnitC')).toBe(0); + } finally { + if (hostTableId) await ctx.deleteTable(hostTableId).catch(() => undefined); + if (refundsTableId) await ctx.deleteTable(refundsTableId).catch(() => undefined); + if (accountsTableId) await ctx.deleteTable(accountsTableId).catch(() => undefined); + if (unitsTableId) await ctx.deleteTable(unitsTableId).catch(() => undefined); + } + }); }); diff --git a/packages/v2/e2e/src/createRecords.e2e.spec.ts b/packages/v2/e2e/src/createRecords.e2e.spec.ts index 975415298c..31730302a8 100644 --- a/packages/v2/e2e/src/createRecords.e2e.spec.ts +++ b/packages/v2/e2e/src/createRecords.e2e.spec.ts @@ -663,6 +663,80 @@ describe('v2 http createRecords (e2e)', () => { expect(result.rows[0].version).toBe(1); expect(result.rows[0].created_by).toBe(ctx.testUser.id); }); + + it('creates records when a dependent pending table has missing physical storage', async () => { + const pendingTable = await createTable({ + baseId: ctx.baseId, + name: 'Pending Table Outside Create Scope', + fields: [{ type: 'singleLineText', name: 'Pending Title', isPrimary: true }], + views: [{ type: 'grid' }], + }); + const tableWithLink = await ctx.createField({ + baseId: ctx.baseId, + tableId: pendingTable.id, + field: { + type: 'link', + name: 'Ready Table Link', + options: { + relationship: 'manyOne', + foreignTableId: tableId, + lookupFieldId: textFieldId, + isOneWay: true, + }, + }, + }); + const linkField = tableWithLink.fields.find((field) => field.name === 'Ready Table Link'); + if (!linkField) { + throw new Error('Missing link field on pending table'); + } + await ctx.createField({ + baseId: ctx.baseId, + tableId: pendingTable.id, + field: { + type: 'lookup', + name: 'Ready Table Value', + options: { + foreignTableId: tableId, + lookupFieldId: numberFieldId, + linkFieldId: linkField.id, + }, + }, + }); + const pendingTableMeta = await ctx.testContainer.metaDb + .selectFrom('table_meta') + .select('db_table_name') + .where('id', '=', pendingTable.id) + .executeTakeFirst(); + const pendingDbTableName = pendingTableMeta?.db_table_name; + if (!pendingDbTableName) { + throw new Error('Missing pending table db name'); + } + + try { + await sql` + UPDATE "table_meta" + SET "provision_state" = 'pending' + WHERE "id" = ${pendingTable.id} + `.execute(ctx.testContainer.metaDb); + await sql` + DROP TABLE ${sql.table(pendingDbTableName)} + `.execute(ctx.testContainer.db); + + const records = await createRecords(tableId, [ + { fields: { [textFieldId]: 'Ready table record', [numberFieldId]: 444 } }, + ]); + + expect(records).toHaveLength(1); + expect(records[0].fields[textFieldId]).toBe('Ready table record'); + } finally { + await sql` + UPDATE "table_meta" + SET "provision_state" = 'error', + "deleted_time" = COALESCE("deleted_time", now()) + WHERE "id" = ${pendingTable.id} + `.execute(ctx.testContainer.metaDb); + } + }); }); describe('error handling', () => { diff --git a/packages/v2/e2e/src/deleteByRange.e2e.spec.ts b/packages/v2/e2e/src/deleteByRange.e2e.spec.ts index 080b8a7b6d..faa7f7d99a 100644 --- a/packages/v2/e2e/src/deleteByRange.e2e.spec.ts +++ b/packages/v2/e2e/src/deleteByRange.e2e.spec.ts @@ -5,7 +5,9 @@ import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { getSharedTestContext, type SharedTestContext } from './shared/globalTestContext'; import { setupGroupedLinkRangeFixture, + setupGroupedSingleSelectRangeFixture, type GroupedLinkRangeFixture, + type GroupedSingleSelectRangeFixture, } from './shared/groupedLinkRangeFixture'; /** @@ -588,6 +590,106 @@ describe('v2 http deleteByRange (e2e)', () => { }); }); + describe('deleteByRange with grouped singleSelect view order parity', () => { + const expectOnlySecondVisibleRowDeleted = async (fixture: GroupedSingleSelectRangeFixture) => { + const records = await ctx.listRecords(fixture.tableId); + + // Visible order follows the view row order (reverse of creation), so the + // second visible row is 加单3. + expect(records.find((record) => record.id === fixture.recordIds.order2)).toBeDefined(); + expect(records.find((record) => record.id === fixture.recordIds.order3)).toBeUndefined(); + expect(records.find((record) => record.id === fixture.recordIds.order4)).toBeDefined(); + }; + + it('should delete the visible row in a saved grouped view when row order differs', async () => { + const fixture = await setupGroupedSingleSelectRangeFixture(ctx, 'delete-saved-group', { + persistViewQuery: true, + }); + + const result = await ctx.deleteByRange({ + tableId: fixture.tableId, + viewId: fixture.viewId, + ranges: [[1, 1]], + type: 'rows', + }); + + expect(result.deletedCount).toBe(1); + expect(result.deletedRecordIds).toEqual([fixture.recordIds.order3]); + await expectOnlySecondVisibleRowDeleted(fixture); + }); + + it('should list records using saved view group order so visible row offsets match range commands', async () => { + const table = await ctx.createTable({ + baseId: ctx.baseId, + name: `Saved Group List Order ${Date.now()}`, + fields: [ + { name: 'Name', type: 'singleLineText', isPrimary: true }, + { + name: 'Store', + type: 'singleSelect', + options: { + choices: [ + { name: 'A', color: 'blueLight1' }, + { name: 'B', color: 'greenLight1' }, + ], + }, + }, + ], + views: [{ type: 'grid' }], + }); + const viewId = table.views[0]?.id; + const nameFieldId = table.fields.find((field) => field.isPrimary)?.id; + const storeFieldId = table.fields.find((field) => field.name === 'Store')?.id; + if (!viewId || !nameFieldId || !storeFieldId) { + throw new Error('Failed to resolve saved group list fixture metadata'); + } + + const b1 = await ctx.createRecord(table.id, { [nameFieldId]: 'B1', [storeFieldId]: 'B' }); + const a1 = await ctx.createRecord(table.id, { [nameFieldId]: 'A1', [storeFieldId]: 'A' }); + const b2 = await ctx.createRecord(table.id, { [nameFieldId]: 'B2', [storeFieldId]: 'B' }); + const a2 = await ctx.createRecord(table.id, { [nameFieldId]: 'A2', [storeFieldId]: 'A' }); + await ctx.drainOutbox(); + + await ctx.testContainer.db + .updateTable('view') + .set({ + group: JSON.stringify([{ fieldId: storeFieldId, order: 'asc' }]), + }) + .where('id', '=', viewId) + .execute(); + + const visibleRecords = await ctx.listRecords(table.id, { viewId }); + expect(visibleRecords.map((record) => record.id)).toEqual([a1.id, a2.id, b1.id, b2.id]); + + const result = await ctx.deleteByRange({ + tableId: table.id, + viewId, + ranges: [[1, 1]], + type: 'rows', + }); + + expect(result.deletedRecordIds).toEqual([a2.id]); + }); + + it('should delete the visible row in a personal grouped view request', async () => { + const fixture = await setupGroupedSingleSelectRangeFixture(ctx, 'delete-personal-group'); + + const result = await ctx.deleteByRange({ + tableId: fixture.tableId, + viewId: fixture.viewId, + ranges: [[1, 1]], + type: 'rows', + ignoreViewQuery: true, + groupBy: fixture.groupByAsc, + sort: fixture.sortAsc, + }); + + expect(result.deletedCount).toBe(1); + expect(result.deletedRecordIds).toEqual([fixture.recordIds.order3]); + await expectOnlySecondVisibleRowDeleted(fixture); + }); + }); + describe('deleteByRange with self conditional rollups', () => { it('deletes a row without error when same-table conditionalRollup filters use field references', async () => { const table = await ctx.createTable({ @@ -1078,11 +1180,11 @@ describe('v2 http deleteByRange (e2e)', () => { it('should delete correct row at large offset when sort values tie', async () => { const targetOffset = 400; - const orderColumn = `__row_${tieViewId}`; + // All sort values tie, so the visible order follows the view row order column. const expected = await sql<{ __id: string }>` SELECT "__id" FROM ${sql.table(tieDbTableName)} - ORDER BY ${sql.ref(orderColumn)} ASC + ORDER BY ${sql.ref(`__row_${tieViewId}`)} ASC OFFSET ${targetOffset} LIMIT 1 `.execute(ctx.testContainer.db); diff --git a/packages/v2/e2e/src/deleteField.e2e.spec.ts b/packages/v2/e2e/src/deleteField.e2e.spec.ts index b5161f11c2..7e9620fd51 100644 --- a/packages/v2/e2e/src/deleteField.e2e.spec.ts +++ b/packages/v2/e2e/src/deleteField.e2e.spec.ts @@ -547,6 +547,7 @@ describe('v2 http deleteField (e2e)', () => { let tableId: string | undefined; try { const sourceFieldId = createFieldId(); + const fallbackFieldId = createFieldId(); const formulaFieldId = createFieldId(); const table = await ctx.createTable({ baseId: ctx.baseId, @@ -554,8 +555,17 @@ describe('v2 http deleteField (e2e)', () => { fields: [ { type: 'singleLineText', name: 'Name', isPrimary: true }, { type: 'singleLineText', id: sourceFieldId, name: 'Source' }, + { type: 'singleLineText', id: fallbackFieldId, name: 'Fallback' }, + ], + records: [ + { + fields: { + Name: 'r1', + [sourceFieldId]: 'Source 1', + [fallbackFieldId]: 'Fallback 1', + }, + }, ], - records: [{ fields: { Name: 'r1', [sourceFieldId]: 'Source 1' } }], }); tableId = table.id; @@ -591,6 +601,26 @@ describe('v2 http deleteField (e2e)', () => { expect(await getStoredCellValue(tableId, firstRecordId, formulaDbFieldName)).toBeNull(); const recordsAfter = await ctx.listRecordsWithoutDrain(tableId); expect(recordsAfter).toHaveLength(1); + + await ctx.updateField({ + tableId, + fieldId: formulaFieldId, + field: { + type: 'formula', + options: { expression: `LOWER({${fallbackFieldId}})` }, + }, + }); + await ctx.drainOutbox(); + + const tableAfterFormulaRepair = await ctx.getTableById(tableId); + const repairedFormulaField = tableAfterFormulaRepair.fields.find( + (field) => field.id === formulaFieldId + ); + expect(repairedFormulaField?.type).toBe('formula'); + expect(repairedFormulaField?.hasError).toBeFalsy(); + expect(await getStoredCellValue(tableId, firstRecordId, formulaDbFieldName)).toBe( + 'fallback 1' + ); } finally { await safeDeleteTable(tableId); } diff --git a/packages/v2/e2e/src/form-submit-required-computed.e2e.spec.ts b/packages/v2/e2e/src/form-submit-required-computed.e2e.spec.ts new file mode 100644 index 0000000000..c503d446f4 --- /dev/null +++ b/packages/v2/e2e/src/form-submit-required-computed.e2e.spec.ts @@ -0,0 +1,77 @@ +import { submitRecordOkResponseSchema } from '@teable/v2-contract-http'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { getSharedTestContext, type SharedTestContext } from './shared/globalTestContext'; + +let nextId = 0; + +const createFieldId = () => { + nextId += 1; + return `fld${nextId.toString(36).padStart(16, '0')}`; +}; + +describe('v2 http form submit required computed field regression (e2e)', () => { + let ctx: SharedTestContext; + + beforeAll(async () => { + ctx = await getSharedTestContext(); + }, 30000); + + it('submits when stale form metadata marks a computed field as required', async () => { + const titleFieldId = createFieldId(); + const createdByFieldId = createFieldId(); + + const table = await ctx.createTable({ + baseId: ctx.baseId, + name: 'form-required-computed', + fields: [ + { id: titleFieldId, type: 'singleLineText', name: 'Title', isPrimary: true }, + { id: createdByFieldId, type: 'createdBy', name: 'Creator' }, + ], + views: [ + { type: 'grid', name: 'Grid' }, + { type: 'form', name: 'Form' }, + ], + }); + + const formView = table.views.find((view) => view.type === 'form'); + if (!formView) { + throw new Error('Missing form view'); + } + + await ctx.testContainer.db + .updateTable('view') + .set({ + column_meta: JSON.stringify({ + [titleFieldId]: { order: 0, visible: true, required: true }, + [createdByFieldId]: { order: 1, visible: true, required: true }, + }), + }) + .where('id', '=', formView.id) + .execute(); + + const response = await fetch(`${ctx.baseUrl}/tables/submitRecord`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + tableId: table.id, + formId: formView.id, + fields: { + [titleFieldId]: 'submitted', + }, + }), + }); + + const rawBody = await response.json(); + expect(response.status).toBe(201); + + const parsed = submitRecordOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.record.fields[titleFieldId]).toBe('submitted'); + + const records = await ctx.listRecords(table.id, { limit: 1000 }); + expect(records).toHaveLength(1); + expect(records[0]?.fields[titleFieldId]).toBe('submitted'); + }); +}); diff --git a/packages/v2/e2e/src/formula.e2e.spec.ts b/packages/v2/e2e/src/formula.e2e.spec.ts index e0ddec5c51..dcd13a7f8e 100644 --- a/packages/v2/e2e/src/formula.e2e.spec.ts +++ b/packages/v2/e2e/src/formula.e2e.spec.ts @@ -16763,7 +16763,7 @@ describe('v2 http formula (e2e)', () => { /** * Scenario: Formula directly references createdBy field * Formula: {createdByField} - * Expect: returns the name string of the creator (via subquery from users table) + * Expect: returns the name string from the stored creator snapshot */ it('should extract name from createdBy field - {createdByField}', async () => { const createTableResponse = await fetch(`${ctx.baseUrl}/tables/create`, { @@ -16865,7 +16865,7 @@ describe('v2 http formula (e2e)', () => { /** * Scenario: Formula directly references lastModifiedBy field * Formula: {lastModifiedByField} - * Expect: returns the name string of the last modifier (via subquery from users table) + * Expect: returns the name string from the stored last modifier snapshot */ it('should extract name from lastModifiedBy field - {lastModifiedByField}', async () => { const createTableResponse = await fetch(`${ctx.baseUrl}/tables/create`, { diff --git a/packages/v2/e2e/src/importCsv.e2e.spec.ts b/packages/v2/e2e/src/importCsv.e2e.spec.ts index f2fc0e41d3..56de409f7c 100644 --- a/packages/v2/e2e/src/importCsv.e2e.spec.ts +++ b/packages/v2/e2e/src/importCsv.e2e.spec.ts @@ -1,8 +1,36 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; import { createV2HttpClient } from '@teable/v2-contract-http-client'; import { beforeAll, describe, expect, it } from 'vitest'; import { getSharedTestContext, type SharedTestContext } from './shared/globalTestContext'; +const startChunkedCsvServer = async ( + chunks: string[] +): Promise<{ url: string; close: () => Promise }> => { + const server = createServer(async (_req, res) => { + res.writeHead(200, { 'content-type': 'text/csv; charset=utf-8' }); + for (const chunk of chunks) { + res.write(chunk); + await new Promise((resolve) => setTimeout(resolve, 1)); + } + res.end(); + }); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}/import.csv`, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + }; +}; + describe('v2 http importCsv (e2e)', () => { let ctx: SharedTestContext; @@ -39,7 +67,11 @@ Jane,,555-1234 expect(result.table.baseId).toBe(ctx.baseId); expect(result.table.fields).toHaveLength(3); expect(result.table.fields.map((f) => f.name)).toEqual(['Name', 'Age', 'City']); - expect(result.table.fields.every((f) => f.type === 'singleLineText')).toBe(true); + expect(result.table.fields.map((f) => f.type)).toEqual([ + 'singleLineText', + 'number', + 'singleLineText', + ]); // 验证导入的记录数 expect(result.totalImported).toBe(3); @@ -141,6 +173,74 @@ Jane,,555-1234 expect(body.data.totalImported).toBe(1000); }); + it('imports chunked CSV URL data without treating later rows as headers', async () => { + const headers = + 'Title,Description,Status,Priority,Tags,Amount,Quantity,Start Date,Due Date,Active,Score,Owner Text,Notes,Category,Labels,External ID,Source,Percent,Approved,Comment'; + const statuses = ['T', 'D', 'N']; + const tagPairs = ['A, B', 'B, C', 'C, D', 'D, A']; + const rows = Array.from({ length: 1000 }, (_, i) => { + const rowNumber = i + 1; + const startDate = new Date(Date.UTC(2026, 0, rowNumber)).toISOString().slice(0, 10); + return [ + `R${rowNumber}`, + '', + statuses[i % statuses.length], + '', + `"${tagPairs[i % tagPairs.length]}"`, + String(rowNumber % 10), + '', + startDate, + '', + rowNumber % 2 ? 'true' : '', + String((i % 5) + 1), + '', + '', + '', + '', + '', + '', + '', + '', + '', + ].join(','); + }); + const chunks = [ + `${headers}\n${rows.slice(0, 363).join('\n')}\n`, + `${rows.slice(363, 700).join('\n')}\n`, + rows.slice(700).join('\n'), + ]; + const csvServer = await startChunkedCsvServer(chunks); + + try { + const body = await ctx.importCsv({ + baseId: ctx.baseId, + csvUrl: csvServer.url, + tableName: 'Chunked URL Import', + batchSize: 100, + }); + + expect(body.table.name).toBe('Chunked URL Import'); + expect(body.table.fields).toHaveLength(20); + expect(body.totalImported).toBe(1000); + + const titleFieldId = body.table.fields.find((field) => field.name === 'Title')!.id; + const tagsFieldId = body.table.fields.find((field) => field.name === 'Tags')!.id; + const amountFieldId = body.table.fields.find((field) => field.name === 'Amount')!.id; + const records = await ctx.listRecords(body.table.id, { limit: 1000 }); + const recordsByTitle = new Map( + records.map((record) => [record.fields[titleFieldId], record]) + ); + + expect(records).toHaveLength(1000); + expect(recordsByTitle.size).toBe(1000); + expect(recordsByTitle.get('R1')?.fields[amountFieldId]).toBe(1); + expect(recordsByTitle.get('R364')?.fields[tagsFieldId]).toBe('D, A'); + expect(recordsByTitle.get('R1000')?.fields[amountFieldId]).toBe(0); + } finally { + await csvServer.close(); + } + }); + it('returns 400 for empty CSV', async () => { const client = createV2HttpClient({ baseUrl: ctx.baseUrl }); diff --git a/packages/v2/e2e/src/importRecords.e2e.spec.ts b/packages/v2/e2e/src/importRecords.e2e.spec.ts index ff17148425..c5ce3a48b8 100644 --- a/packages/v2/e2e/src/importRecords.e2e.spec.ts +++ b/packages/v2/e2e/src/importRecords.e2e.spec.ts @@ -9,6 +9,20 @@ describe('v2 http importRecords (e2e)', () => { ctx = await getSharedTestContext(); }); + const expectSingleLookupValue = (value: unknown, expected: string) => { + if (Array.isArray(value)) { + expect(value).toEqual([expected]); + return; + } + expect(value).toBe(expected); + }; + + const expectLinkTitles = (value: unknown, expectedTitles: string[]) => { + expect(Array.isArray(value)).toBe(true); + const actualTitles = (value as Array<{ title?: unknown }>).map((item) => item.title).sort(); + expect(actualTitles).toEqual([...expectedTitles].sort()); + }; + describe('basic import', () => { it('should import CSV records into a simple text table', async () => { // Create table with text fields @@ -208,6 +222,106 @@ describe('v2 http importRecords (e2e)', () => { expect(result.totalImported).toBe(150); }); + + it('updates lookup values and reverse links when typecasting link titles across batches', async () => { + const sourceTable = await ctx.createTable({ + baseId: ctx.baseId, + name: 'ImportLinkLookupSource', + fields: [ + { type: 'singleLineText', name: 'Name', isPrimary: true }, + { type: 'singleLineText', name: 'Status' }, + ], + views: [{ type: 'grid' }], + }); + const sourceNameFieldId = sourceTable.fields.find((f) => f.name === 'Name')!.id; + const sourceStatusFieldId = sourceTable.fields.find((f) => f.name === 'Status')!.id; + + const sourceRecords = await ctx.createRecords(sourceTable.id, [ + { fields: { [sourceNameFieldId]: 'Source A', [sourceStatusFieldId]: 'Status A' } }, + { fields: { [sourceNameFieldId]: 'Source B', [sourceStatusFieldId]: 'Status B' } }, + ]); + const sourceRecordByName = new Map( + sourceRecords.map((record) => [record.fields[sourceNameFieldId], record]) + ); + + const targetTable = await ctx.createTable({ + baseId: ctx.baseId, + name: 'ImportLinkLookupTarget', + fields: [ + { type: 'singleLineText', name: 'Name', isPrimary: true }, + { + type: 'link', + name: 'Source', + options: { + relationship: 'manyMany', + foreignTableId: sourceTable.id, + lookupFieldId: sourceNameFieldId, + }, + }, + ], + views: [{ type: 'grid' }], + }); + const targetNameFieldId = targetTable.fields.find((f) => f.name === 'Name')!.id; + const targetLinkFieldId = targetTable.fields.find((f) => f.name === 'Source')!.id; + + const tableWithLookup = await ctx.createField({ + baseId: ctx.baseId, + tableId: targetTable.id, + field: { + type: 'lookup', + name: 'Source Status', + options: { + linkFieldId: targetLinkFieldId, + foreignTableId: sourceTable.id, + lookupFieldId: sourceStatusFieldId, + }, + }, + }); + const lookupFieldId = tableWithLookup.fields.find((f) => f.name === 'Source Status')!.id; + const sourceTableWithSymmetricLink = await ctx.getTableById(sourceTable.id); + const symmetricLinkFieldId = sourceTableWithSymmetricLink.fields.find( + (field) => + field.type === 'link' && + (field.options as { symmetricFieldId?: string }).symmetricFieldId === targetLinkFieldId + )?.id; + expect(symmetricLinkFieldId).toBeDefined(); + if (!symmetricLinkFieldId) { + throw new Error('Missing symmetric link field'); + } + + const result = await ctx.importRecords({ + tableId: targetTable.id, + fileType: 'csv', + sourceColumnMap: { + [targetNameFieldId]: 0, + [targetLinkFieldId]: 1, + }, + csvData: 'Name,Source\nImported 1,Source A\nImported 2,Source B', + options: { + typecast: true, + batchSize: 1, + }, + }); + + expect(result.totalImported).toBe(2); + await ctx.drainOutbox(); + + const targetRecords = await ctx.listRecordsWithoutDrain(targetTable.id); + const targetByName = new Map( + targetRecords.map((record) => [record.fields[targetNameFieldId], record]) + ); + expectSingleLookupValue(targetByName.get('Imported 1')?.fields[lookupFieldId], 'Status A'); + expectSingleLookupValue(targetByName.get('Imported 2')?.fields[lookupFieldId], 'Status B'); + + const sourceRecordsAfterImport = await ctx.listRecordsWithoutDrain(sourceTable.id); + const sourceByName = new Map( + sourceRecordsAfterImport.map((record) => [record.fields[sourceNameFieldId], record]) + ); + expectLinkTitles(sourceByName.get('Source A')?.fields[symmetricLinkFieldId], ['Imported 1']); + expectLinkTitles(sourceByName.get('Source B')?.fields[symmetricLinkFieldId], ['Imported 2']); + expect(sourceRecordByName.get('Source A')?.id).toBe(sourceByName.get('Source A')?.id); + expect(sourceRecordByName.get('Source B')?.id).toBe(sourceByName.get('Source B')?.id); + }); }); describe('partial column mapping', () => { diff --git a/packages/v2/e2e/src/lookup-no-extra-updates.e2e.spec.ts b/packages/v2/e2e/src/lookup-no-extra-updates.e2e.spec.ts index 60f43fd6b9..9e284c9c36 100644 --- a/packages/v2/e2e/src/lookup-no-extra-updates.e2e.spec.ts +++ b/packages/v2/e2e/src/lookup-no-extra-updates.e2e.spec.ts @@ -224,4 +224,102 @@ describe('lookup computed update scope regression', () => { await deleteTableSafe(ctx, sourceTableId); } }); + + test('refreshes an existing lookup when a linked multi-user source gains another user', async () => { + let sourceTableId: string | undefined; + let hostTableId: string | undefined; + + try { + const bobCell = { id: 'usrLookupMultiUserBob', title: 'Bob' }; + await sql` + insert into users (id, name, email) + values (${bobCell.id}, ${bobCell.title}, ${'bob+lookup-multi-user@e2e.com'}) + on conflict (id) do nothing + `.execute(ctx.testContainer.db); + + const aliceCell = { id: ctx.testUser.id, title: ctx.testUser.name }; + const sourceNameFieldId = createFieldId(); + const sourceUserFieldId = createFieldId(); + const hostNameFieldId = createFieldId(); + const hostLinkFieldId = createFieldId(); + const hostLookupFieldId = createFieldId(); + + const sourceTable = await ctx.createTable({ + baseId: ctx.baseId, + name: 'LookupMultiUserSource', + fields: [ + { type: 'singleLineText', id: sourceNameFieldId, name: 'Name', isPrimary: true }, + { + type: 'user', + id: sourceUserFieldId, + name: 'Members', + options: { isMultiple: true, shouldNotify: false }, + }, + ], + views: [{ type: 'grid' }], + }); + sourceTableId = sourceTable.id; + + const hostTable = await ctx.createTable({ + baseId: ctx.baseId, + name: 'LookupMultiUserHost', + fields: [ + { type: 'singleLineText', id: hostNameFieldId, name: 'Name', isPrimary: true }, + { + type: 'link', + id: hostLinkFieldId, + name: 'Source', + options: { + relationship: 'manyOne', + foreignTableId: sourceTable.id, + lookupFieldId: sourceNameFieldId, + }, + }, + { + type: 'lookup', + id: hostLookupFieldId, + name: 'Lookup Members', + options: { + linkFieldId: hostLinkFieldId, + foreignTableId: sourceTable.id, + lookupFieldId: sourceUserFieldId, + }, + }, + ], + views: [{ type: 'grid' }], + }); + hostTableId = hostTable.id; + + const source = await ctx.createRecord(sourceTable.id, { + [sourceNameFieldId]: 'A-1', + [sourceUserFieldId]: [aliceCell], + }); + + const host = await ctx.createRecord(hostTable.id, { + [hostNameFieldId]: 'B-1', + [hostLinkFieldId]: { id: source.id }, + }); + await ctx.drainOutbox(); + + const beforeRecords = await ctx.listRecords(hostTable.id); + const beforeLookup = beforeRecords.find((record) => record.id === host.id)?.fields[ + hostLookupFieldId + ]; + expect(beforeLookup).toMatchObject([aliceCell]); + + await ctx.updateRecord(sourceTable.id, source.id, { + [sourceUserFieldId]: [aliceCell, bobCell], + }); + await ctx.drainOutbox(); + + const afterRecords = await ctx.listRecords(hostTable.id); + const afterLookup = afterRecords.find((record) => record.id === host.id)?.fields[ + hostLookupFieldId + ]; + expect(afterLookup).toMatchObject([aliceCell, bobCell]); + } finally { + await deleteTableSafe(ctx, hostTableId); + await deleteTableSafe(ctx, sourceTableId); + } + }); }); diff --git a/packages/v2/e2e/src/paste.e2e.spec.ts b/packages/v2/e2e/src/paste.e2e.spec.ts index 76967619ed..7ca2a446f3 100644 --- a/packages/v2/e2e/src/paste.e2e.spec.ts +++ b/packages/v2/e2e/src/paste.e2e.spec.ts @@ -5,7 +5,9 @@ import { beforeAll, describe, expect, it } from 'vitest'; import { getSharedTestContext, type SharedTestContext } from './shared/globalTestContext'; import { setupGroupedLinkRangeFixture, + setupGroupedSingleSelectRangeFixture, type GroupedLinkRangeFixture, + type GroupedSingleSelectRangeFixture, } from './shared/groupedLinkRangeFixture'; /** @@ -3582,6 +3584,66 @@ describe('v2 http paste (e2e)', () => { }); }); + describe('paste with grouped singleSelect view order parity', () => { + const expectOnlySecondVisibleRowPasted = async ( + fixture: GroupedSingleSelectRangeFixture, + value: string + ) => { + const records = await ctx.listRecords(fixture.tableId); + const order2 = records.find((record) => record.id === fixture.recordIds.order2); + const order3 = records.find((record) => record.id === fixture.recordIds.order3); + const order4 = records.find((record) => record.id === fixture.recordIds.order4); + + // Visible order follows the view row order (reverse of creation), so the + // second visible row is 加单3. + expect(order4?.fields[fixture.nameFieldId]).toBe('加单4'); + expect(order3?.fields[fixture.nameFieldId]).toBe(value); + expect(order2?.fields[fixture.nameFieldId]).toBe('加单2'); + }; + + it('should paste to the visible row in a saved grouped view when row order differs', async () => { + const fixture = await setupGroupedSingleSelectRangeFixture(ctx, 'paste-saved-group', { + persistViewQuery: true, + }); + const value = '保存视图粘贴'; + + const result = await ctx.paste({ + tableId: fixture.tableId, + viewId: fixture.viewId, + ranges: [ + [0, 1], + [0, 1], + ], + content: [[value]], + }); + + expect(result.updatedCount).toBe(1); + await expectOnlySecondVisibleRowPasted(fixture, value); + }); + + it('should paste to the visible row in a personal grouped view request', async () => { + const fixture = await setupGroupedSingleSelectRangeFixture(ctx, 'paste-personal-group'); + const value = '个人视图粘贴'; + + const result = await ctx.paste({ + tableId: fixture.tableId, + viewId: fixture.viewId, + ranges: [ + [0, 1], + [0, 1], + ], + content: [[value]], + ignoreViewQuery: true, + groupBy: fixture.groupByAsc, + sort: fixture.sortAsc, + projection: fixture.projection, + }); + + expect(result.updatedCount).toBe(1); + await expectOnlySecondVisibleRowPasted(fixture, value); + }); + }); + describe('paste with multi-column sort', () => { let multiSortTableId: string; let multiSortViewId: string; @@ -3896,11 +3958,11 @@ describe('v2 http paste (e2e)', () => { it('should paste to correct row at large offset when sort values tie', async () => { const targetOffset = 400; - const orderColumn = `__row_${tieViewId}`; + // All sort values tie, so the visible order follows the view row order column. const expected = await sql<{ __id: string }>` SELECT "__id" FROM ${sql.table(tieDbTableName)} - ORDER BY ${sql.ref(orderColumn)} ASC + ORDER BY ${sql.ref(`__row_${tieViewId}`)} ASC OFFSET ${targetOffset} LIMIT 1 `.execute(ctx.testContainer.db); diff --git a/packages/v2/e2e/src/shared/globalTestContext.ts b/packages/v2/e2e/src/shared/globalTestContext.ts index ce7e5e1f70..871fb2f222 100644 --- a/packages/v2/e2e/src/shared/globalTestContext.ts +++ b/packages/v2/e2e/src/shared/globalTestContext.ts @@ -138,15 +138,39 @@ export interface SharedTestContext { deleteRecords: (tableId: string, recordIds: string[]) => Promise; listRecords: ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: { + limit?: number; + offset?: number; + baseId?: string; + viewId?: string; + ignoreViewQuery?: boolean; + sort?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + groupBy?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + } ) => Promise }>>; listRecordsWithoutDrain: ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: { + limit?: number; + offset?: number; + baseId?: string; + viewId?: string; + ignoreViewQuery?: boolean; + sort?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + groupBy?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + } ) => Promise }>>; listRecordsWithPagination: ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: { + limit?: number; + offset?: number; + baseId?: string; + viewId?: string; + ignoreViewQuery?: boolean; + sort?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + groupBy?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + } ) => Promise>; getTableById: ( tableId: string, @@ -704,12 +728,26 @@ const initSharedContext = async ( const listRecordsWithPaginationWithoutDrain = async ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: { + limit?: number; + offset?: number; + baseId?: string; + viewId?: string; + ignoreViewQuery?: boolean; + sort?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + groupBy?: Array<{ fieldId: string; order: 'asc' | 'desc' }>; + } ) => { const params = new URLSearchParams({ tableId }); if (options?.limit !== undefined) params.set('limit', String(options.limit)); if (options?.offset !== undefined) params.set('offset', String(options.offset)); if (options?.baseId) params.set('baseId', options.baseId); + if (options?.viewId) params.set('viewId', options.viewId); + if (options?.ignoreViewQuery !== undefined) { + params.set('ignoreViewQuery', String(options.ignoreViewQuery)); + } + if (options?.sort) params.set('sort', JSON.stringify(options.sort)); + if (options?.groupBy) params.set('groupBy', JSON.stringify(options.groupBy)); const response = await fetch(`${baseUrl}/tables/listRecords?${params.toString()}`, { method: 'GET', headers: { 'content-type': 'application/json' }, @@ -723,7 +761,7 @@ const initSharedContext = async ( const listRecordsWithPagination = async ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: Parameters[1] ) => { await drainOutbox(); return listRecordsWithPaginationWithoutDrain(tableId, options); @@ -731,7 +769,7 @@ const initSharedContext = async ( const listRecordsWithoutDrain = async ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: Parameters[1] ) => { const result = await listRecordsWithPaginationWithoutDrain(tableId, options); return result.records; @@ -739,7 +777,7 @@ const initSharedContext = async ( const listRecords = async ( tableId: string, - options?: { limit?: number; offset?: number; baseId?: string } + options?: Parameters[1] ) => { const result = await listRecordsWithPagination(tableId, options); return result.records; diff --git a/packages/v2/e2e/src/shared/groupedLinkRangeFixture.ts b/packages/v2/e2e/src/shared/groupedLinkRangeFixture.ts index 289f606ef5..2c2214ccee 100644 --- a/packages/v2/e2e/src/shared/groupedLinkRangeFixture.ts +++ b/packages/v2/e2e/src/shared/groupedLinkRangeFixture.ts @@ -1,6 +1,8 @@ +import { sql } from 'kysely'; import type { SharedTestContext } from './globalTestContext'; type GroupByClause = { fieldId: string; order: 'asc' | 'desc' }; +type SortClause = { fieldId: string; order: 'asc' | 'desc' }; export interface GroupedLinkRangeFixture { tableId: string; @@ -18,6 +20,24 @@ export interface GroupedLinkRangeFixture { }; } +export interface GroupedSingleSelectRangeFixture { + tableId: string; + viewId: string; + nameFieldId: string; + storeFieldId: string; + timeFieldId: string; + groupByAsc: GroupByClause[]; + sortAsc: SortClause[]; + projection: string[]; + expectedVisibleOrderIds: string[]; + recordIds: { + order1: string; + order2: string; + order3: string; + order4: string; + }; +} + export const setupGroupedLinkRangeFixture = async ( ctx: SharedTestContext, label: string @@ -110,3 +130,101 @@ export const setupGroupedLinkRangeFixture = async ( }, }; }; + +export const setupGroupedSingleSelectRangeFixture = async ( + ctx: SharedTestContext, + label: string, + options: { persistViewQuery?: boolean } = {} +): Promise => { + const suffix = `${label}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + + const table = await ctx.createTable({ + baseId: ctx.baseId, + name: `Grouped Range Orders ${suffix}`, + fields: [ + { name: '编码', type: 'singleLineText', isPrimary: true }, + { + name: '配送店铺', + type: 'singleSelect', + options: { choices: [{ name: '下沙店', color: 'greenLight1' }] }, + }, + { name: '送出时间', type: 'singleLineText' }, + { name: '日期', type: 'date' }, + ], + views: [{ type: 'grid' }], + }); + + const viewId = table.views[0]?.id; + const nameFieldId = table.fields.find((field) => field.isPrimary)?.id; + const storeFieldId = table.fields.find((field) => field.name === '配送店铺')?.id; + const timeFieldId = table.fields.find((field) => field.name === '送出时间')?.id; + const dateFieldId = table.fields.find((field) => field.name === '日期')?.id; + + if (!viewId || !nameFieldId || !storeFieldId || !timeFieldId || !dateFieldId) { + throw new Error('Failed to resolve grouped single-select fixture metadata'); + } + + const records = []; + for (const name of ['加单1', '加单2', '加单3', '加单4']) { + records.push( + await ctx.createRecord(table.id, { + [nameFieldId]: name, + [storeFieldId]: '下沙店', + [timeFieldId]: '16:30', + [dateFieldId]: '2026-06-01', + }) + ); + } + + const tableMeta = await ctx.testContainer.db + .selectFrom('table_meta') + .select('db_table_name') + .where('id', '=', table.id) + .executeTakeFirst(); + const dbTableName = tableMeta?.db_table_name; + if (!dbTableName) { + throw new Error('Failed to resolve grouped single-select fixture storage table'); + } + + const orderColumn = `__row_${viewId}`; + await sql` + ALTER TABLE ${sql.table(dbTableName)} + ADD COLUMN IF NOT EXISTS ${sql.id(orderColumn)} double precision + `.execute(ctx.testContainer.db); + await sql` + UPDATE ${sql.table(dbTableName)} + SET ${sql.ref(orderColumn)} = 5 - ${sql.ref('__auto_number')} + `.execute(ctx.testContainer.db); + + const groupByAsc = [{ fieldId: storeFieldId, order: 'asc' }] as const; + const sortAsc = [{ fieldId: timeFieldId, order: 'asc' }] as const; + + if (options.persistViewQuery) { + await ctx.testContainer.db + .updateTable('view') + .set({ + group: JSON.stringify(groupByAsc), + sort: JSON.stringify({ sortObjs: sortAsc, manualSort: false }), + }) + .where('id', '=', viewId) + .execute(); + } + + return { + tableId: table.id, + viewId, + nameFieldId, + storeFieldId, + timeFieldId, + groupByAsc: [...groupByAsc], + sortAsc: [...sortAsc], + projection: [nameFieldId, storeFieldId, timeFieldId, dateFieldId], + expectedVisibleOrderIds: records.map((record) => record.id), + recordIds: { + order1: records[0].id, + order2: records[1].id, + order3: records[2].id, + order4: records[3].id, + }, + }; +}; diff --git a/packages/v2/e2e/src/update-field/computed/dependency-cascade.spec.ts b/packages/v2/e2e/src/update-field/computed/dependency-cascade.spec.ts index 976ab64c78..0223c1a60c 100644 --- a/packages/v2/e2e/src/update-field/computed/dependency-cascade.spec.ts +++ b/packages/v2/e2e/src/update-field/computed/dependency-cascade.spec.ts @@ -267,7 +267,6 @@ describe('update-field: computed dependency cascades', () => { const hostMatchFieldId = createFieldId(); const conditionalLookupFieldId = createFieldId(); - const formulaFieldId = createFieldId(); const host = await ctx.createTable({ baseId: ctx.baseId, name: createName('dep-cascade-cond-lookup-formula-host'), @@ -296,18 +295,25 @@ describe('update-field: computed dependency cascades', () => { }, }, }, - { - type: 'formula', - id: formulaFieldId, - name: 'FormulaOverConditionalLookup', - options: { expression: `{${conditionalLookupFieldId}}` }, - }, ], records: [{ fields: { Name: 'H1', [hostMatchFieldId]: 1 } }], }); await ctx.drainOutbox(); + const formulaFieldId = createFieldId(); + await ctx.createField({ + baseId: ctx.baseId, + tableId: host.id, + field: { + type: 'formula', + id: formulaFieldId, + name: 'FormulaOverConditionalLookup', + options: { expression: `{${conditionalLookupFieldId}}` }, + }, + }); + await ctx.drainOutbox(); + return { hostTableId: host.id, foreignTableId: foreign.id, diff --git a/packages/v2/e2e/src/update-field/select-option-record-action.spec.ts b/packages/v2/e2e/src/update-field/select-option-record-action.spec.ts new file mode 100644 index 0000000000..a822b00ca2 --- /dev/null +++ b/packages/v2/e2e/src/update-field/select-option-record-action.spec.ts @@ -0,0 +1,126 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { updateFieldOkResponseSchema } from '@teable/v2-contract-http'; +import { beforeAll, describe, expect, test } from 'vitest'; +import { getSharedTestContext, type SharedTestContext } from '../shared/globalTestContext'; + +let fieldIdCounter = 0; + +const createFieldId = () => { + const suffix = fieldIdCounter.toString(36).padStart(16, '0'); + fieldIdCounter += 1; + return `fld${suffix}`; +}; + +const isObjectRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +const getDomainEventName = (event: unknown): string | undefined => { + if (!isObjectRecord(event)) { + return undefined; + } + + const name = event['name']; + if (!isObjectRecord(name) || typeof name.toString !== 'function') { + return undefined; + } + + return name.toString(); +}; + +const getActionKey = (event: unknown): string | undefined => { + if (!isObjectRecord(event)) { + return undefined; + } + + const actionKey = event['actionKey']; + return typeof actionKey === 'string' ? actionKey : undefined; +}; + +const deleteTableSafe = async (ctx: SharedTestContext, tableId: string | undefined) => { + if (!tableId) return; + try { + await ctx.deleteTable(tableId); + } catch {} +}; + +describe('update-field: select option record action', () => { + let ctx: SharedTestContext; + + beforeAll(async () => { + ctx = await getSharedTestContext(); + }); + + test('emits record action trigger when removing a singleSelect option clears cells', async () => { + let tableId: string | undefined; + try { + const statusFieldId = createFieldId(); + const optionOpen = { id: 'choOpen', name: 'Open', color: 'blueBright' as const }; + const optionDone = { id: 'choDone', name: 'Done', color: 'greenBright' as const }; + + const table = await ctx.createTable({ + baseId: ctx.baseId, + name: 'Select Option Record Action', + fields: [ + { type: 'singleLineText', name: 'Name', isPrimary: true }, + { + type: 'singleSelect', + id: statusFieldId, + name: 'Status', + options: { choices: [optionOpen, optionDone] }, + }, + ], + }); + tableId = table.id; + + const clearedRecord = await ctx.createRecord(tableId, { [statusFieldId]: 'Open' }); + const retainedRecord = await ctx.createRecord(tableId, { [statusFieldId]: 'Done' }); + await ctx.drainOutbox(); + + const beforeEventCount = ctx.testContainer.eventBus.events().length; + + const response = await fetch(`${ctx.baseUrl}/tables/updateField`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + tableId, + fieldId: statusFieldId, + field: { + options: { + choices: [optionDone], + }, + }, + }), + }); + + expect(response.ok).toBe(true); + const rawBody = await response.json(); + const parsed = updateFieldOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success || !parsed.data.ok) { + throw new Error('Failed to parse updateField response'); + } + + const responseEventNames = parsed.data.data.events.map((event) => event.name); + expect(responseEventNames).toContain('FieldUpdated'); + expect(responseEventNames).not.toContain('TableActionTriggerRequested'); + + const records = await ctx.listRecords(tableId); + expect(records.find((record) => record.id === clearedRecord.id)?.fields[statusFieldId]).toBe( + null + ); + expect(records.find((record) => record.id === retainedRecord.id)?.fields[statusFieldId]).toBe( + 'Done' + ); + + const newEvents = ctx.testContainer.eventBus.events().slice(beforeEventCount); + const actionKeys = newEvents + .filter((event) => getDomainEventName(event) === 'TableActionTriggerRequested') + .map((event) => getActionKey(event)) + .filter((actionKey): actionKey is string => Boolean(actionKey)); + + expect(actionKeys).toEqual(expect.arrayContaining(['setRecord'])); + } finally { + await deleteTableSafe(ctx, tableId); + } + }); +}); diff --git a/packages/v2/e2e/src/update-field/user/update-properties.spec.ts b/packages/v2/e2e/src/update-field/user/update-properties.spec.ts index 920643a250..412197dc80 100644 --- a/packages/v2/e2e/src/update-field/user/update-properties.spec.ts +++ b/packages/v2/e2e/src/update-field/user/update-properties.spec.ts @@ -46,6 +46,16 @@ describe('update-field: user property updates', () => { return fieldId; }; + const getPersistedMultiplicity = async (fieldId: string) => { + const row = await ctx.testContainer.db + .selectFrom('field') + .select('is_multiple_cell_value') + .where('id', '=', fieldId) + .executeTakeFirst(); + + return row?.is_multiple_cell_value; + }; + beforeAll(async () => { ctx = await getSharedTestContext(); @@ -87,6 +97,7 @@ describe('update-field: user property updates', () => { if (isUserField(updatedField!)) { expect(updatedField.options?.isMultiple).toBe(true); } + expect(await getPersistedMultiplicity(fieldId)).toBe(true); const records = await ctx.listRecordsWithoutDrain(tableId); const v1 = records.find((r) => r.id === r1.id)?.fields[fieldId] as @@ -103,6 +114,49 @@ describe('update-field: user property updates', () => { await ctx.deleteRecords(tableId, [r1.id, r2.id]); }); + test('should refresh dependent formula when converting single user to multiple users', async () => { + const fieldId = await createUserField('Single User Formula Source', { + isMultiple: false, + shouldNotify: false, + }); + const formulaFieldId = createFieldId(); + await ctx.createField({ + baseId: ctx.baseId, + tableId, + field: { + type: 'formula', + id: formulaFieldId, + name: 'User Formula', + options: { expression: `{${fieldId}}` }, + }, + }); + const r1 = await ctx.createRecord(tableId, { + [fieldId]: { id: ctx.testUser.id, title: ctx.testUser.name, email: ctx.testUser.email }, + }); + + const beforeRecords = await ctx.listRecords(tableId); + expect(beforeRecords.find((r) => r.id === r1.id)?.fields[formulaFieldId]).toBe( + ctx.testUser.name + ); + + await ctx.updateField({ + tableId, + fieldId, + field: { type: 'user', options: { isMultiple: true, shouldNotify: false } }, + }); + + const records = await ctx.listRecords(tableId); + const record = records.find((r) => r.id === r1.id); + expect(record?.fields[fieldId]).toMatchObject([ + { id: ctx.testUser.id, title: ctx.testUser.name, email: ctx.testUser.email }, + ]); + expect(record?.fields[formulaFieldId]).toEqual([ctx.testUser.name]); + + await ctx.deleteField({ tableId, fieldId: formulaFieldId }); + await ctx.deleteField({ tableId, fieldId }); + await ctx.deleteRecords(tableId, [r1.id]); + }); + test('should convert multiple users to single user', async () => { const fieldId = await createUserField('Multiple User', { isMultiple: true, @@ -127,6 +181,7 @@ describe('update-field: user property updates', () => { if (isUserField(updatedField!)) { expect(updatedField.options?.isMultiple).toBe(false); } + expect(await getPersistedMultiplicity(fieldId)).toBe(false); const records = await ctx.listRecordsWithoutDrain(tableId); const value = records.find((r) => r.id === r1.id)?.fields[fieldId] as diff --git a/packages/v2/formula-sql-pg/src/DirectFieldReferences.spec.ts b/packages/v2/formula-sql-pg/src/DirectFieldReferences.spec.ts index 640b25ba0c..f83497e5e4 100644 --- a/packages/v2/formula-sql-pg/src/DirectFieldReferences.spec.ts +++ b/packages/v2/formula-sql-pg/src/DirectFieldReferences.spec.ts @@ -78,4 +78,14 @@ describe('direct field references', () => { } }); }); + + describe('attachment field display value extraction', () => { + it('should return attachment filenames, not raw JSON objects', async () => { + const context = await buildFormulaSnapshotContext(testTable, 'Direct_Attachment'); + + expect(context.result).toBe('10, 20'); + expect(context.result).not.toContain('"id"'); + expect(context.result).not.toContain('"mimetype"'); + }); + }); }); diff --git a/packages/v2/formula-sql-pg/src/FormulaSqlPgFunctions.ts b/packages/v2/formula-sql-pg/src/FormulaSqlPgFunctions.ts index f7a2a494f3..f94565bd72 100644 --- a/packages/v2/formula-sql-pg/src/FormulaSqlPgFunctions.ts +++ b/packages/v2/formula-sql-pg/src/FormulaSqlPgFunctions.ts @@ -747,7 +747,15 @@ export class FormulaSqlPgFunctions extends FormulaSqlPgExpressionBuilder { if (textExpr.isArray) { return this.vectorizeUnaryNumericArray(textExpr, (valueSql: string) => valueSql, 'value'); } - return this.coerceToNumber(textExpr, 'value'); + const numeric = this.coerceToNumber(textExpr, 'value'); + return makeExpr( + `COALESCE(${numeric.valueSql}, 0)`, + 'number', + false, + numeric.errorConditionSql, + numeric.errorMessageSql, + numeric.field + ); } private concatenate(params: SqlExpr[]): SqlExpr { diff --git a/packages/v2/formula-sql-pg/src/FormulaSqlPgTranslator.ts b/packages/v2/formula-sql-pg/src/FormulaSqlPgTranslator.ts index d7fdea72d3..c50041ea15 100644 --- a/packages/v2/formula-sql-pg/src/FormulaSqlPgTranslator.ts +++ b/packages/v2/formula-sql-pg/src/FormulaSqlPgTranslator.ts @@ -104,8 +104,12 @@ export class FormulaSqlPgTranslator { } renderSql(expr: SqlExpr): string { - if (!expr.errorConditionSql) return expr.valueSql; + const renderedValueSql = expr.displayValueSql ?? expr.valueSql; + if (!expr.errorConditionSql) return renderedValueSql; const errorMessage = expr.errorMessageSql ?? buildErrorLiteral('INTERNAL', 'unknown_error'); + if (expr.displayValueSql) { + return `CASE WHEN ${expr.errorConditionSql} THEN ${errorMessage} ELSE ${renderedValueSql} END`; + } if (expr.isArray) { return `CASE WHEN ${expr.errorConditionSql} THEN jsonb_build_array(${errorMessage}) ELSE ${expr.valueSql} END`; } diff --git a/packages/v2/formula-sql-pg/src/FormulaSqlPgVisitor.ts b/packages/v2/formula-sql-pg/src/FormulaSqlPgVisitor.ts index 466a17cd96..4bad682907 100644 --- a/packages/v2/formula-sql-pg/src/FormulaSqlPgVisitor.ts +++ b/packages/v2/formula-sql-pg/src/FormulaSqlPgVisitor.ts @@ -221,6 +221,21 @@ export class FormulaSqlPgVisitor return normalizeLookupLinkTitles(expr); } + if ( + expr.storageKind === 'json' && + expr.isArray && + expr.field?.type().equals(FieldType.attachment()) + ) { + const normalizedArray = normalizeJsonArraySql(expr); + return { + ...expr, + displayValueSql: `( + SELECT string_agg(${buildJsonObjectText('elem')}, ', ' ORDER BY ord) + FROM jsonb_array_elements(${normalizedArray}) WITH ORDINALITY AS arr(elem, ord) + )`, + }; + } + // For JSON object fields (button, link), extract the display value (title/name) // when directly referenced in a formula. This ensures that {Button} and {LinkField} // return the human-readable title instead of the raw JSON object. diff --git a/packages/v2/formula-sql-pg/src/SqlExpression.ts b/packages/v2/formula-sql-pg/src/SqlExpression.ts index 652d269218..7bc2a6f36d 100644 --- a/packages/v2/formula-sql-pg/src/SqlExpression.ts +++ b/packages/v2/formula-sql-pg/src/SqlExpression.ts @@ -5,6 +5,7 @@ export type SqlStorageKind = 'scalar' | 'json' | 'array'; export type SqlExpr = { valueSql: string; + displayValueSql?: string; valueType: SqlValueType; isArray: boolean; storageKind?: SqlStorageKind; diff --git a/packages/v2/formula-sql-pg/src/TranslatorEdgeCases.spec.ts b/packages/v2/formula-sql-pg/src/TranslatorEdgeCases.spec.ts index d55544ad35..d41c98fa85 100644 --- a/packages/v2/formula-sql-pg/src/TranslatorEdgeCases.spec.ts +++ b/packages/v2/formula-sql-pg/src/TranslatorEdgeCases.spec.ts @@ -24,6 +24,7 @@ beforeAll(async () => { { name: 'NestedFormula', expression: '{BaseFormula} + 10' }, { name: 'ErrorFormula', expression: '1 / 0' }, { name: 'FieldByName', expression: '{Number} + {SingleLineText}' }, + { name: 'ValueBlank', expression: 'VALUE(BLANK())' }, ]; testTable = await createFormulaTestTable(container, formulaFields, { profile: 'minimal', @@ -48,6 +49,11 @@ describe('FormulaSqlPgTranslator edge cases', () => { const result = await executeFormulaAsText(testTable, 'ErrorFormula'); expect(result).toContain('#ERROR:DIV0'); }); + + it('should treat blank VALUE input as zero', async () => { + const result = await executeFormulaAsText(testTable, 'ValueBlank'); + expect(result).toBe('0'); + }); }); describe('formula field resolution', () => { diff --git a/packages/v2/formula-sql-pg/src/__snapshots__/DirectFieldReferences.spec.ts.snap b/packages/v2/formula-sql-pg/src/__snapshots__/DirectFieldReferences.spec.ts.snap index d2f73482ba..8149694bd2 100644 --- a/packages/v2/formula-sql-pg/src/__snapshots__/DirectFieldReferences.spec.ts.snap +++ b/packages/v2/formula-sql-pg/src/__snapshots__/DirectFieldReferences.spec.ts.snap @@ -13,8 +13,16 @@ exports[`direct field references > direct reference to 'attachment' field 1`] = "rawValue": "[{"id": "att_1", "name": "10", "path": "/tmp/10", "size": 10, "token": "tok_1", "mimetype": "text/plain"}, {"id": "att_2", "name": "20", "path": "/tmp/20", "size": 20, "token": "tok_2", "mimetype": "text/plain"}]", }, }, - "result": "[{"id": "att_1", "name": "10", "path": "/tmp/10", "size": 10, "token": "tok_1", "mimetype": "text/plain"}, {"id": "att_2", "name": "20", "path": "/tmp/20", "size": 20, "token": "tok_2", "mimetype": "text/plain"}]", - "sql": ""t"."Attachment"", + "result": "10, 20", + "sql": "( + SELECT string_agg(COALESCE(elem->>'title', elem->>'name', elem #>> '{}'), ', ' ORDER BY ord) + FROM jsonb_array_elements((CASE + WHEN "t"."Attachment" IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(("t"."Attachment")::jsonb) = 'array' THEN ("t"."Attachment")::jsonb + WHEN jsonb_typeof(("t"."Attachment")::jsonb) = 'null' THEN '[]'::jsonb + ELSE jsonb_build_array(("t"."Attachment")::jsonb) + END)) WITH ORDINALITY AS arr(elem, ord) + )", } `; diff --git a/packages/v2/formula-sql-pg/src/__snapshots__/ErrorHandling.spec.ts.snap b/packages/v2/formula-sql-pg/src/__snapshots__/ErrorHandling.spec.ts.snap index 7b08c65ec4..f163f9a5b9 100644 --- a/packages/v2/formula-sql-pg/src/__snapshots__/ErrorHandling.spec.ts.snap +++ b/packages/v2/formula-sql-pg/src/__snapshots__/ErrorHandling.spec.ts.snap @@ -521,7 +521,7 @@ exports[`error handling > 'ValueTypeError': 'Value with non-convertible type' 1` }, }, "result": "#ERROR:TYPE:cannot_cast_to_number", - "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE (NULL::double precision)::text END", + "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE (COALESCE(NULL::double precision, 0))::text END", } `; diff --git a/packages/v2/formula-sql-pg/src/__snapshots__/NumericFunctions.spec.ts.snap b/packages/v2/formula-sql-pg/src/__snapshots__/NumericFunctions.spec.ts.snap index 0b577d0d36..0000a7a78a 100644 --- a/packages/v2/formula-sql-pg/src/__snapshots__/NumericFunctions.spec.ts.snap +++ b/packages/v2/formula-sql-pg/src/__snapshots__/NumericFunctions.spec.ts.snap @@ -12864,7 +12864,7 @@ exports[`numeric functions > 'Value' with 'autoNumber' 1`] = ` }, }, "result": "1", - "sql": "("t"."AutoNumber")::double precision", + "sql": "COALESCE(("t"."AutoNumber")::double precision, 0)", } `; @@ -12883,7 +12883,7 @@ exports[`numeric functions > 'Value' with 'button' 1`] = ` }, }, "result": "#ERROR:TYPE:cannot_cast_to_number", - "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE (NULL::double precision)::text END", + "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE (COALESCE(NULL::double precision, 0))::text END", } `; @@ -12902,7 +12902,7 @@ exports[`numeric functions > 'Value' with 'checkbox' 1`] = ` }, }, "result": "1", - "sql": "(CASE WHEN "t"."Checkbox" IS NULL THEN NULL WHEN "t"."Checkbox" THEN 1 ELSE 0 END)::double precision", + "sql": "COALESCE((CASE WHEN "t"."Checkbox" IS NULL THEN NULL WHEN "t"."Checkbox" THEN 1 ELSE 0 END)::double precision, 0)", } `; @@ -12986,8 +12986,8 @@ exports[`numeric functions > 'Value' with 'conditionalRollup' 1`] = ` "rawValue": null, }, }, - "result": null, - "sql": "("t"."ConditionalRollupType")::double precision", + "result": "0", + "sql": "COALESCE(("t"."ConditionalRollupType")::double precision, 0)", } `; @@ -13014,7 +13014,7 @@ exports[`numeric functions > 'Value' with 'createdBy' 1`] = ` WHEN jsonb_typeof(to_jsonb("t"."CreatedBy")) = 'object' THEN COALESCE(to_jsonb("t"."CreatedBy")->>'title', to_jsonb("t"."CreatedBy")->>'name', to_jsonb("t"."CreatedBy") #>> '{}') WHEN jsonb_typeof(to_jsonb("t"."CreatedBy")) = 'array' THEN NULL ELSE to_jsonb("t"."CreatedBy") #>> '{}' - END) AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT (CASE + END) AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT (CASE WHEN jsonb_typeof(to_jsonb("t"."CreatedBy")) = 'object' THEN COALESCE(to_jsonb("t"."CreatedBy")->>'title', to_jsonb("t"."CreatedBy")->>'name', to_jsonb("t"."CreatedBy") #>> '{}') WHEN jsonb_typeof(to_jsonb("t"."CreatedBy")) = 'array' THEN NULL ELSE to_jsonb("t"."CreatedBy") #>> '{}' @@ -13026,7 +13026,7 @@ exports[`numeric functions > 'Value' with 'createdBy' 1`] = ` WHEN jsonb_typeof(to_jsonb("t"."CreatedBy")) = 'object' THEN COALESCE(to_jsonb("t"."CreatedBy")->>'title', to_jsonb("t"."CreatedBy")->>'name', to_jsonb("t"."CreatedBy") #>> '{}') WHEN jsonb_typeof(to_jsonb("t"."CreatedBy")) = 'array' THEN NULL ELSE to_jsonb("t"."CreatedBy") #>> '{}' - END) AS val) AS v) END))::text END", + END) AS val) AS v) END), 0))::text END", } `; @@ -13049,7 +13049,7 @@ exports[`numeric functions > 'Value' with 'createdTime' 1`] = ` }, }, "result": "#ERROR:TYPE:cannot_cast_datetime_to_number_value", - "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_datetime_to_number_value' ELSE (NULL::double precision)::text END", + "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_datetime_to_number_value' ELSE (COALESCE(NULL::double precision, 0))::text END", } `; @@ -13072,7 +13072,7 @@ exports[`numeric functions > 'Value' with 'date' 1`] = ` }, }, "result": "#ERROR:TYPE:cannot_cast_datetime_to_number_value", - "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_datetime_to_number_value' ELSE (NULL::double precision)::text END", + "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_datetime_to_number_value' ELSE (COALESCE(NULL::double precision, 0))::text END", } `; @@ -13094,7 +13094,7 @@ exports[`numeric functions > 'Value' with 'formula' 1`] = ` }, }, "result": "10", - "sql": "(10)::double precision", + "sql": "COALESCE((10)::double precision, 0)", } `; @@ -13121,7 +13121,7 @@ exports[`numeric functions > 'Value' with 'lastModifiedBy' 1`] = ` WHEN jsonb_typeof(to_jsonb("t"."LastModifiedBy")) = 'object' THEN COALESCE(to_jsonb("t"."LastModifiedBy")->>'title', to_jsonb("t"."LastModifiedBy")->>'name', to_jsonb("t"."LastModifiedBy") #>> '{}') WHEN jsonb_typeof(to_jsonb("t"."LastModifiedBy")) = 'array' THEN NULL ELSE to_jsonb("t"."LastModifiedBy") #>> '{}' - END) AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT (CASE + END) AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT (CASE WHEN jsonb_typeof(to_jsonb("t"."LastModifiedBy")) = 'object' THEN COALESCE(to_jsonb("t"."LastModifiedBy")->>'title', to_jsonb("t"."LastModifiedBy")->>'name', to_jsonb("t"."LastModifiedBy") #>> '{}') WHEN jsonb_typeof(to_jsonb("t"."LastModifiedBy")) = 'array' THEN NULL ELSE to_jsonb("t"."LastModifiedBy") #>> '{}' @@ -13133,7 +13133,7 @@ exports[`numeric functions > 'Value' with 'lastModifiedBy' 1`] = ` WHEN jsonb_typeof(to_jsonb("t"."LastModifiedBy")) = 'object' THEN COALESCE(to_jsonb("t"."LastModifiedBy")->>'title', to_jsonb("t"."LastModifiedBy")->>'name', to_jsonb("t"."LastModifiedBy") #>> '{}') WHEN jsonb_typeof(to_jsonb("t"."LastModifiedBy")) = 'array' THEN NULL ELSE to_jsonb("t"."LastModifiedBy") #>> '{}' - END) AS val) AS v) END))::text END", + END) AS val) AS v) END), 0))::text END", } `; @@ -13156,7 +13156,7 @@ exports[`numeric functions > 'Value' with 'lastModifiedTime' 1`] = ` }, }, "result": "#ERROR:TYPE:cannot_cast_datetime_to_number_value", - "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_datetime_to_number_value' ELSE (NULL::double precision)::text END", + "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_datetime_to_number_value' ELSE (COALESCE(NULL::double precision, 0))::text END", } `; @@ -13175,7 +13175,7 @@ exports[`numeric functions > 'Value' with 'link' 1`] = ` }, }, "result": "#ERROR:TYPE:cannot_cast_to_number", - "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE (NULL::double precision)::text END", + "sql": "CASE WHEN TRUE THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE (COALESCE(NULL::double precision, 0))::text END", } `; @@ -13194,11 +13194,11 @@ exports[`numeric functions > 'Value' with 'longText' 1`] = ` }, }, "result": "10", - "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."LongText" AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."LongText" AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."LongText" AS val) AS v) THEN NULL ELSE (SELECT (CASE + "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."LongText" AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."LongText" AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."LongText" AS val) AS v) THEN NULL ELSE (SELECT (CASE WHEN NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NULL THEN NULL WHEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric')) THEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'))::double precision ELSE NULL - END) FROM (SELECT "t"."LongText" AS val) AS v) END))::text END", + END) FROM (SELECT "t"."LongText" AS val) AS v) END), 0))::text END", } `; @@ -13317,7 +13317,7 @@ exports[`numeric functions > 'Value' with 'number' 1`] = ` }, }, "result": "10", - "sql": "("t"."Number")::double precision", + "sql": "COALESCE(("t"."Number")::double precision, 0)", } `; @@ -13336,7 +13336,7 @@ exports[`numeric functions > 'Value' with 'rating' 1`] = ` }, }, "result": "4", - "sql": "("t"."Rating")::double precision", + "sql": "COALESCE(("t"."Rating")::double precision, 0)", } `; @@ -13357,8 +13357,8 @@ exports[`numeric functions > 'Value' with 'rollup' 1`] = ` "rawValue": null, }, }, - "result": null, - "sql": "("t"."RollupType")::double precision", + "result": "0", + "sql": "COALESCE(("t"."RollupType")::double precision, 0)", } `; @@ -13377,11 +13377,11 @@ exports[`numeric functions > 'Value' with 'singleLineText' 1`] = ` }, }, "result": "10", - "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleLineText" AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleLineText" AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleLineText" AS val) AS v) THEN NULL ELSE (SELECT (CASE + "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleLineText" AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleLineText" AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleLineText" AS val) AS v) THEN NULL ELSE (SELECT (CASE WHEN NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NULL THEN NULL WHEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric')) THEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'))::double precision ELSE NULL - END) FROM (SELECT "t"."SingleLineText" AS val) AS v) END))::text END", + END) FROM (SELECT "t"."SingleLineText" AS val) AS v) END), 0))::text END", } `; @@ -13400,11 +13400,11 @@ exports[`numeric functions > 'Value' with 'singleSelect' 1`] = ` }, }, "result": "10", - "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleSelect" AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleSelect" AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleSelect" AS val) AS v) THEN NULL ELSE (SELECT (CASE + "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleSelect" AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleSelect" AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT "t"."SingleSelect" AS val) AS v) THEN NULL ELSE (SELECT (CASE WHEN NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NULL THEN NULL WHEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric')) THEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'))::double precision ELSE NULL - END) FROM (SELECT "t"."SingleSelect" AS val) AS v) END))::text END", + END) FROM (SELECT "t"."SingleSelect" AS val) AS v) END), 0))::text END", } `; @@ -13451,11 +13451,11 @@ exports[`numeric functions > 'ValueBad' uses constant input 1`] = ` "funcId": "ValueBad", "inputs": {}, "result": "#ERROR:TYPE:cannot_cast_to_number", - "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT 'not-a-number' AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT 'not-a-number' AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT 'not-a-number' AS val) AS v) THEN NULL ELSE (SELECT (CASE + "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT 'not-a-number' AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT 'not-a-number' AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT 'not-a-number' AS val) AS v) THEN NULL ELSE (SELECT (CASE WHEN NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NULL THEN NULL WHEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric')) THEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'))::double precision ELSE NULL - END) FROM (SELECT 'not-a-number' AS val) AS v) END))::text END", + END) FROM (SELECT 'not-a-number' AS val) AS v) END), 0))::text END", } `; @@ -13465,10 +13465,10 @@ exports[`numeric functions > 'ValueComma' uses constant input 1`] = ` "funcId": "ValueComma", "inputs": {}, "result": "1000", - "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT '1,000' AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT '1,000' AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE ((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT '1,000' AS val) AS v) THEN NULL ELSE (SELECT (CASE + "sql": "CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT '1,000' AS val) AS v) THEN CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT '1,000' AS val) AS v) THEN '#ERROR:TYPE:cannot_cast_to_number' ELSE '#ERROR:TYPE:cannot_cast_to_number' END ELSE (COALESCE((CASE WHEN (SELECT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NOT (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric'))) FROM (SELECT '1,000' AS val) AS v) THEN NULL ELSE (SELECT (CASE WHEN NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NULL THEN NULL WHEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)') IS NOT NULL AND NOT (NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') IS NOT NULL AND NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') ~ '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)[eE][+-]?\\d+') AND __teable_input_is_valid(SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'), 'numeric')) THEN (SUBSTRING(NULLIF(REGEXP_REPLACE(BTRIM((v.val)::text), '[,\\s]', '', 'g'), '') FROM '^([+-]?\\d+\\.?\\d*|[+-]?\\d*\\.\\d+)'))::double precision ELSE NULL - END) FROM (SELECT '1,000' AS val) AS v) END))::text END", + END) FROM (SELECT '1,000' AS val) AS v) END), 0))::text END", } `; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27a1bd6962..46dbf47c5b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,8 +187,8 @@ importers: specifier: 0.49.0 version: 0.49.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-pino': - specifier: 0.49.0 - version: 0.49.0(@opentelemetry/api@1.9.0) + specifier: 0.54.0 + version: 0.54.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-runtime-node': specifier: 0.24.0 version: 0.24.0(@opentelemetry/api@1.9.0) @@ -369,6 +369,9 @@ importers: lodash: specifier: 4.17.21 version: 4.17.21 + markdown-it: + specifier: 14.1.0 + version: 14.1.0 mime-types: specifier: 2.1.35 version: 2.1.35 @@ -667,6 +670,9 @@ importers: istanbul-merge: specifier: 2.0.0 version: 2.0.0 + neverthrow: + specifier: 8.2.0 + version: 8.2.0 npm-run-all2: specifier: 6.1.2 version: 6.1.2 @@ -8142,10 +8148,6 @@ packages: resolution: {integrity: sha512-IxcFDP1IGMDemVFG2by/AMK+/o6EuBQ8idUq3xZ6MxgQGeumYZuX5OwR0h9HuvcUc/JPjQGfU5OHKIKYDJcXeA==} engines: {node: '>=8.0.0'} - '@opentelemetry/api-logs@0.202.0': - resolution: {integrity: sha512-fTBjMqKCfotFWfLzaKyhjLvyEyq5vDKTTFfBmx21btv3gvy8Lq6N5Dh2OzqeuN4DjtpSvNT1uNVfg08eD2Rfxw==} - engines: {node: '>=8.0.0'} - '@opentelemetry/api-logs@0.203.0': resolution: {integrity: sha512-9B9RU0H7Ya1Dx/Rkyc4stuBZSGVQF27WigitInx2QQoj6KUpEFYPKoWjdFTunJYxmXmh17HeBvbMa1EhGyPmqQ==} engines: {node: '>=8.0.0'} @@ -8154,6 +8156,10 @@ packages: resolution: {integrity: sha512-DqxY8yoAaiBPivoJD4UtgrMS8gEmzZ5lnaxzPojzLVHBGqPxgWm4zcuvcUHZiqQ6kRX2Klel2r9y8cA2HAtqpw==} engines: {node: '>=8.0.0'} + '@opentelemetry/api-logs@0.207.0': + resolution: {integrity: sha512-lAb0jQRVyleQQGiuuvCOTDVspc14nx6XJjP4FspJ1sNARo3Regq4ZZbrc3rN4b1TYSuUCvgH+UXUPug4SLOqEQ==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api-logs@0.208.0': resolution: {integrity: sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==} engines: {node: '>=8.0.0'} @@ -8544,8 +8550,8 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/instrumentation-pino@0.49.0': - resolution: {integrity: sha512-nngcqUnIeVnDvRMf6fixYwlMbTNzCVGv93CacyR/8TL/pjyumje020PC5q7b6CfcTdToiD5GMTMKvWBiTd08cA==} + '@opentelemetry/instrumentation-pino@0.54.0': + resolution: {integrity: sha512-LrTIfO7fzIOHNiwEb4CYN+jtFr5M/5yUMLR5ZW10gvyIT25m/L0BTsulbTvIHjUbU8CVWG9iJENKyVOK6PMTqg==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 @@ -8598,12 +8604,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/instrumentation@0.202.0': - resolution: {integrity: sha512-Uz3BxZWPgDwgHM2+vCKEQRh0R8WKrd/q6Tus1vThRClhlPO39Dyz7mDrOr6KuqGXAlBQ1e5Tnymzri4RMZNaWA==} - engines: {node: ^18.19.0 || >=20.6.0} - peerDependencies: - '@opentelemetry/api': ^1.3.0 - '@opentelemetry/instrumentation@0.203.0': resolution: {integrity: sha512-ke1qyM+3AK2zPuBPb6Hk/GCsc5ewbLvPNkEuELx/JmANeEp6ZjnZ+wypPAJSucTw0wvCGrUaibDSdcrGFoWxKQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -8616,6 +8616,12 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation@0.207.0': + resolution: {integrity: sha512-y6eeli9+TLKnznrR8AZlQMSJT7wILpXH+6EYq5Vf/4Ao+huI7EedxQHwRgVUOMLFbe7VFDvHJrX9/f4lcwnJsA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation@0.208.0': resolution: {integrity: sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==} engines: {node: ^18.19.0 || >=20.6.0} @@ -17686,6 +17692,10 @@ packages: resolution: {integrity: sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==} engines: {node: '>=0.10.0'} + markdown-it@14.1.0: + resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} + hasBin: true + markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} @@ -17787,6 +17797,9 @@ packages: mdn-data@2.12.2: resolution: {integrity: sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==} + mdurl@2.0.0: + resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} + media-typer@0.3.0: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} @@ -27640,15 +27653,15 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs@0.202.0': + '@opentelemetry/api-logs@0.203.0': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs@0.203.0': + '@opentelemetry/api-logs@0.204.0': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs@0.204.0': + '@opentelemetry/api-logs@0.207.0': dependencies: '@opentelemetry/api': 1.9.0 @@ -28186,12 +28199,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/instrumentation-pino@0.49.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/instrumentation-pino@0.54.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs': 0.202.0 - '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) - '@opentelemetry/instrumentation': 0.202.0(@opentelemetry/api@1.9.0) + '@opentelemetry/api-logs': 0.207.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation': 0.207.0(@opentelemetry/api@1.9.0) transitivePeerDependencies: - supports-color @@ -28265,15 +28278,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/instrumentation@0.202.0(@opentelemetry/api@1.9.0)': - dependencies: - '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs': 0.202.0 - import-in-the-middle: 1.11.3 - require-in-the-middle: 7.4.0 - transitivePeerDependencies: - - supports-color - '@opentelemetry/instrumentation@0.203.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -28292,6 +28296,15 @@ snapshots: transitivePeerDependencies: - supports-color + '@opentelemetry/instrumentation@0.207.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.207.0 + import-in-the-middle: 2.0.4 + require-in-the-middle: 8.0.1 + transitivePeerDependencies: + - supports-color + '@opentelemetry/instrumentation@0.208.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -31827,7 +31840,7 @@ snapshots: '@types/acorn@4.0.6': dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 '@types/archiver@6.0.3': dependencies: @@ -32054,16 +32067,16 @@ snapshots: '@types/eslint-scope@3.7.7': dependencies: '@types/eslint': 9.6.1 - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 '@types/eslint@9.6.1': dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 '@types/json-schema': 7.0.15 '@types/estree-jsx@1.0.5': dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 '@types/estree@0.0.39': {} @@ -38765,7 +38778,7 @@ snapshots: is-reference@1.2.1: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 is-regex@1.2.1: dependencies: @@ -39678,6 +39691,15 @@ snapshots: dependencies: object-visit: 1.0.1 + markdown-it@14.1.0: + dependencies: + argparse: 2.0.1 + entities: 4.5.0 + linkify-it: 5.0.0 + mdurl: 2.0.0 + punycode.js: 2.3.1 + uc.micro: 2.1.0 + markdown-table@3.0.4: {} markdown-to-jsx@7.3.2(react@18.3.1): @@ -39905,6 +39927,8 @@ snapshots: mdn-data@2.12.2: optional: true + mdurl@2.0.0: {} + media-typer@0.3.0: {} mem@8.1.1: @@ -40086,7 +40110,7 @@ snapshots: micromark-extension-mdx-expression@3.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 devlop: 1.1.0 micromark-factory-mdx-expression: 2.0.2 micromark-factory-space: 2.0.1 @@ -40098,7 +40122,7 @@ snapshots: micromark-extension-mdx-jsx@3.0.1: dependencies: '@types/acorn': 4.0.6 - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 micromark-factory-mdx-expression: 2.0.2 @@ -40115,7 +40139,7 @@ snapshots: micromark-extension-mdxjs-esm@3.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 devlop: 1.1.0 micromark-core-commonmark: 2.0.2 micromark-util-character: 2.1.1 @@ -40151,7 +40175,7 @@ snapshots: micromark-factory-mdx-expression@2.0.2: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 devlop: 1.1.0 micromark-factory-space: 2.0.1 micromark-util-character: 2.1.1 @@ -40216,7 +40240,7 @@ snapshots: micromark-util-events-to-acorn@2.0.2: dependencies: '@types/acorn': 4.0.6 - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 '@types/unist': 3.0.3 devlop: 1.1.0 estree-util-visit: 2.0.0 @@ -46543,7 +46567,7 @@ snapshots: webpack@5.90.1(@swc/core@1.13.3(@swc/helpers@0.5.17))(esbuild@0.23.0): dependencies: '@types/eslint-scope': 3.7.7 - '@types/estree': 1.0.8 + '@types/estree': 1.0.6 '@webassemblyjs/ast': 1.14.1 '@webassemblyjs/wasm-edit': 1.14.1 '@webassemblyjs/wasm-parser': 1.14.1