Harden Linear XLSX end to end

src/db/repositories/batched-insert.persistence.ts

@@ -0,0 +1,15 @@
+import type { AppDatabase } from "../database-context.js";
+
+const defaultInsertBatchSize = 200;
+
+export function insertManyInBatches<TTable extends Parameters<AppDatabase["insert"]>[0]>(
+  database: AppDatabase,
+  table: TTable,
+  values: readonly unknown[],
+  batchSize = defaultInsertBatchSize
+) {
+  if (!Number.isSafeInteger(batchSize) || batchSize < 1) throw new Error("Insert batch size must be a positive integer.");
+  for (let index = 0; index < values.length; index += batchSize) {
+    database.insert(table).values(values.slice(index, index + batchSize) as never).run();
+  }
+}

src/db/repositories/external-follow-up-import-project-command.repository.ts

@@ -21,6 +21,7 @@ import { rooms } from "../schema/rooms.js";
 import { toCircuitDeviceRowSnapshot } from "./circuit-device-row-structure.persistence.js";
 import { ExternalModelStateRepository } from "./external-model-state.repository.js";
 import { executeProjectCommandTransaction } from "./project-command-transaction.persistence.js";
+import { insertManyInBatches } from "./batched-insert.persistence.js";
 
 export class ExternalFollowUpImportProjectCommandRepository implements ExternalFollowUpImportProjectCommandStore {
   constructor(private readonly database: AppDatabase) {}
@@ -80,7 +81,7 @@ function assertRows(database: AppDatabase, projectId: string, target: ExternalMo
   }
 }
 function replaceConfiguration(database: AppDatabase, projectId: string, target: typeof externalModelConfigurations.$inferInsert) { database.delete(externalModelConfigurations).where(eq(externalModelConfigurations.projectId, projectId)).run(); database.insert(externalModelConfigurations).values(target).run(); }
-function replaceState(database: AppDatabase, projectId: string, target: ExternalModelStateSnapshot) { database.delete(externalModelSources).where(eq(externalModelSources.projectId, projectId)).run(); if (!target.source) return; database.insert(externalModelSources).values(target.source).run(); database.insert(externalImportBatches).values(target.importBatches.map(({ originalContentBase64, ...batch }) => ({ ...batch, originalBytes: originalContentBase64 === null ? null : Buffer.from(originalContentBase64, "base64") }))).run(); if (target.roomMappings.length) database.insert(externalRoomMappings).values(target.roomMappings).run(); if (target.objects.length) database.insert(externalModelObjects).values(target.objects).run(); }
+function replaceState(database: AppDatabase, projectId: string, target: ExternalModelStateSnapshot) { database.delete(externalModelSources).where(eq(externalModelSources.projectId, projectId)).run(); if (!target.source) return; database.insert(externalModelSources).values(target.source).run(); insertManyInBatches(database, externalImportBatches, target.importBatches.map(({ originalContentBase64, ...batch }) => ({ ...batch, originalBytes: originalContentBase64 === null ? null : Buffer.from(originalContentBase64, "base64") }))); insertManyInBatches(database, externalRoomMappings, target.roomMappings); insertManyInBatches(database, externalModelObjects, target.objects); }
 function ids(entries: Array<{ id: string }>) { return new Set(entries.map((entry) => entry.id)); }
 function same(left: unknown, right: unknown) { return canonical(left) === canonical(right); }
 function canonical(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; if (value !== null && typeof value === "object") { const record = value as Record<string, unknown>; return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`).join(",")}}`; } return JSON.stringify(value); }

src/db/repositories/external-initial-import-project-command.repository.ts

@@ -20,6 +20,7 @@ import { projects } from "../schema/projects.js";
 import { rooms } from "../schema/rooms.js";
 import { executeProjectCommandTransaction } from "./project-command-transaction.persistence.js";
 import { ExternalModelStateRepository } from "./external-model-state.repository.js";
+import { insertManyInBatches } from "./batched-insert.persistence.js";
 
 export class ExternalInitialImportProjectCommandRepository implements ExternalInitialImportProjectCommandStore {
   constructor(private readonly database: AppDatabase) {}
@@ -76,9 +77,9 @@ function replaceState(database: AppDatabase, projectId: string, target: External
   database.delete(externalModelSources).where(eq(externalModelSources.projectId, projectId)).run();
   if (!target.source) return;
   database.insert(externalModelSources).values(target.source).run();
-  database.insert(externalImportBatches).values(target.importBatches.map(({ originalContentBase64, ...batch }) => ({ ...batch, originalBytes: originalContentBase64 === null ? null : Buffer.from(originalContentBase64, "base64") }))).run();
-  if (target.roomMappings.length) database.insert(externalRoomMappings).values(target.roomMappings).run();
-  if (target.objects.length) database.insert(externalModelObjects).values(target.objects).run();
+  insertManyInBatches(database, externalImportBatches, target.importBatches.map(({ originalContentBase64, ...batch }) => ({ ...batch, originalBytes: originalContentBase64 === null ? null : Buffer.from(originalContentBase64, "base64") })));
+  insertManyInBatches(database, externalRoomMappings, target.roomMappings);
+  insertManyInBatches(database, externalModelObjects, target.objects);
 }
 
 function decodeCanonicalBase64(value: string) { const bytes = Buffer.from(value, "base64"); if (bytes.toString("base64") !== value) throw new Error("Der XLSX-Inhalt ist nicht kanonisch Base64-kodiert."); return bytes; }

src/db/repositories/project-state-restore-command.repository.ts

@@ -45,6 +45,7 @@ import { executeProjectCommandTransaction } from "./project-command-transaction.
 import {
   readProjectStateSnapshot,
 } from "./project-state-snapshot.persistence.js";
+import { insertManyInBatches } from "./batched-insert.persistence.js";
 
 export class ProjectStateRestoreCommandRepository
   implements ProjectStateRestoreCommandStore
@@ -428,11 +429,5 @@ function insertMany<TTable extends Parameters<AppDatabase["insert"]>[0]>(
   table: TTable,
   values: unknown[]
 ) {
-  if (values.length === 0) {
-    return;
-  }
-  database
-    .insert(table)
-    .values(values as never)
-    .run();
+  insertManyInBatches(database, table, values);
 }

src/db/repositories/project-transfer.repository.ts

@@ -37,6 +37,7 @@ import {
   readProjectStateSnapshot,
 } from "./project-state-snapshot.persistence.js";
 import { serializeProjectStateSnapshot } from "../../domain/models/project-state-snapshot.model.js";
+import { insertManyInBatches } from "./batched-insert.persistence.js";
 
 export class ProjectTransferRepository implements ProjectTransferStore {
   constructor(private readonly database: AppDatabase) {}
@@ -256,7 +257,5 @@ function insertMany<TTable extends Parameters<AppDatabase["insert"]>[0]>(
   table: TTable,
   values: unknown[]
 ) {
-  if (values.length > 0) {
-    database.insert(table).values(values as never).run();
-  }
+  insertManyInBatches(database, table, values);
 }

src/frontend/components/revit-object-table-model.ts

@@ -0,0 +1,25 @@
+import type { ExternalProjectObjectDto } from "../types";
+
+export type RevitObjectSort = "family" | "room" | "worksharing";
+
+export function filterAndSortRevitObjects(
+  objects: readonly ExternalProjectObjectDto[],
+  input: { search: string; boardFilter: string; assignmentFilter: string; sort: RevitObjectSort }
+) {
+  const needle = input.search.trim().toLocaleLowerCase("de-DE");
+  return objects.filter((object) => {
+    if (input.boardFilter !== "all" && (input.boardFilter === "none" ? object.distributionBoardId !== null : object.distributionBoardId !== input.boardFilter)) return false;
+    if (input.assignmentFilter === "unassigned" && object.circuitDeviceRowId !== null) return false;
+    if (input.assignmentFilter === "linked" && object.circuitDeviceRowId === null) return false;
+    return !needle || [object.worksharingId, object.familyAndType, object.selectionMarker, object.sourceRoomNumber, object.sourceRoomName, object.planningValues.displayName ?? ""].some((value) => value.toLocaleLowerCase("de-DE").includes(needle));
+  }).sort((leftObject, rightObject) => {
+    const left = input.sort === "family" ? leftObject.familyAndType : input.sort === "room" ? `${leftObject.sourceRoomNumber}\u0000${leftObject.sourceRoomName}` : leftObject.worksharingId;
+    const right = input.sort === "family" ? rightObject.familyAndType : input.sort === "room" ? `${rightObject.sourceRoomNumber}\u0000${rightObject.sourceRoomName}` : rightObject.worksharingId;
+    return left.localeCompare(right, "de", { numeric: true }) || leftObject.worksharingId.localeCompare(rightObject.worksharingId);
+  });
+}
+
+export function createVirtualRevitObjectWindow<T>(rows: readonly T[], scrollTop: number, rowHeight: number, viewportHeight: number, overscan = 5) {
+  const first = Math.max(0, Math.floor(scrollTop / rowHeight) - overscan);
+  return { first, rows: rows.slice(first, first + Math.ceil(viewportHeight / rowHeight) + overscan * 2) };
+}

src/frontend/components/revit-object-table.tsx

@@ -3,6 +3,7 @@
 import { useEffect, useMemo, useState } from "react";
 import type { ExternalProjectObjectsDto, ExternalReturnScopeDto } from "../types";
 import { createExternalReturnWorkbook, deleteMissingExternalProjectObjects, getExternalProjectObjects, updateExternalProjectObjects } from "../utils/api";
+import { createVirtualRevitObjectWindow, filterAndSortRevitObjects, type RevitObjectSort } from "./revit-object-table-model";
 
 const rowHeight = 44;
 const viewportHeight = 520;
@@ -12,7 +13,7 @@ export function RevitObjectTable({ projectId, onRevision }: { projectId: string;
   const [search, setSearch] = useState("");
   const [boardFilter, setBoardFilter] = useState("all");
   const [assignmentFilter, setAssignmentFilter] = useState("all");
-  const [sort, setSort] = useState<"family" | "room" | "worksharing">("family");
+  const [sort, setSort] = useState<RevitObjectSort>("family");
   const [selected, setSelected] = useState<Set<string>>(new Set());
   const [scrollTop, setScrollTop] = useState(0);
   const [bulkBoard, setBulkBoard] = useState("");
@@ -33,20 +34,9 @@ export function RevitObjectTable({ projectId, onRevision }: { projectId: string;
 
   const rows = useMemo(() => {
     if (!data) return [];
-    const needle = search.trim().toLocaleLowerCase("de-DE");
-    return data.objects.filter((object) => {
-      if (boardFilter !== "all" && (boardFilter === "none" ? object.distributionBoardId !== null : object.distributionBoardId !== boardFilter)) return false;
-      if (assignmentFilter === "unassigned" && object.circuitDeviceRowId !== null) return false;
-      if (assignmentFilter === "linked" && object.circuitDeviceRowId === null) return false;
-      return !needle || [object.worksharingId, object.familyAndType, object.selectionMarker, object.sourceRoomNumber, object.sourceRoomName, object.planningValues.displayName ?? ""].some((value) => value.toLocaleLowerCase("de-DE").includes(needle));
-    }).sort((a, b) => {
-      const left = sort === "family" ? a.familyAndType : sort === "room" ? `${a.sourceRoomNumber}\u0000${a.sourceRoomName}` : a.worksharingId;
-      const right = sort === "family" ? b.familyAndType : sort === "room" ? `${b.sourceRoomNumber}\u0000${b.sourceRoomName}` : b.worksharingId;
-      return left.localeCompare(right, "de", { numeric: true }) || a.worksharingId.localeCompare(b.worksharingId);
-    });
+    return filterAndSortRevitObjects(data.objects, { search, boardFilter, assignmentFilter, sort });
   }, [data, search, boardFilter, assignmentFilter, sort]);
-  const first = Math.max(0, Math.floor(scrollTop / rowHeight) - 5);
-  const visible = rows.slice(first, first + Math.ceil(viewportHeight / rowHeight) + 10);
+  const { first, rows: visible } = createVirtualRevitObjectWindow(rows, scrollTop, rowHeight, viewportHeight);
   const selectable = rows.filter((object) => object.circuitDeviceRowId === null);
   const returnSelectable = rows.filter((object) => object.presenceStatus === "present");
   const selectedHasLinkedObject = data?.objects.some((object) => selected.has(object.id) && object.circuitDeviceRowId !== null) ?? false;