forked from jappel/leistungsbilanz-ts
Add Revit CSV transport
This commit is contained in:
parent
43cee890e7
commit
bfe8b790b2
4 changed files with 521 additions and 5 deletions
350
src/external-model/csv/external-csv-transport.ts
Normal file
350
src/external-model/csv/external-csv-transport.ts
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
import { assertExternalCsvConfiguration } from "./external-csv-configuration.js";
|
||||
import type {
|
||||
ExternalCsvCell,
|
||||
ExternalCsvConfiguration,
|
||||
ExternalCsvDocument,
|
||||
ExternalCsvLineEnding,
|
||||
ExternalCsvRow,
|
||||
} from "./external-csv-contracts.js";
|
||||
|
||||
export type ExternalCsvParseErrorCode =
|
||||
| "invalid-encoding"
|
||||
| "invalid-csv"
|
||||
| "header-not-found"
|
||||
| "ambiguous-header"
|
||||
| "invalid-ifc-guid"
|
||||
| "duplicate-ifc-guid";
|
||||
|
||||
export class ExternalCsvParseError extends Error {
|
||||
constructor(
|
||||
public readonly code: ExternalCsvParseErrorCode,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ExternalCsvParseError";
|
||||
}
|
||||
}
|
||||
|
||||
export function parseExternalCsv(
|
||||
input: Uint8Array,
|
||||
configuration: ExternalCsvConfiguration
|
||||
): ExternalCsvDocument {
|
||||
assertExternalCsvConfiguration(configuration);
|
||||
const hasBom = hasUtf8Bom(input);
|
||||
const contentBytes = hasBom ? input.subarray(3) : input;
|
||||
let text: string;
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true }).decode(contentBytes);
|
||||
} catch {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-encoding",
|
||||
"CSV is not valid UTF-8."
|
||||
);
|
||||
}
|
||||
|
||||
const parsed = parseRows(text, configuration.delimiter);
|
||||
const headerMatches = findHeaderRows(parsed.rows, configuration);
|
||||
if (headerMatches.length === 0) {
|
||||
throw new ExternalCsvParseError(
|
||||
"header-not-found",
|
||||
"CSV does not contain exactly the configured required columns in one header row."
|
||||
);
|
||||
}
|
||||
if (headerMatches.length > 1) {
|
||||
throw new ExternalCsvParseError(
|
||||
"ambiguous-header",
|
||||
"CSV contains more than one possible header row."
|
||||
);
|
||||
}
|
||||
|
||||
const headerRowIndex = headerMatches[0];
|
||||
const configuredColumnNames = [
|
||||
...Object.values(configuration.columns),
|
||||
...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn),
|
||||
];
|
||||
const headerValues = parsed.rows[headerRowIndex].map((cell) => cell.value);
|
||||
if (
|
||||
configuredColumnNames.some(
|
||||
(columnName) => headerValues.filter((value) => value === columnName).length !== 1
|
||||
)
|
||||
) {
|
||||
throw new ExternalCsvParseError(
|
||||
"ambiguous-header",
|
||||
"CSV header contains a configured column more than once."
|
||||
);
|
||||
}
|
||||
const headerLookup = createHeaderLookup(parsed.rows[headerRowIndex]);
|
||||
const ifcGuidColumnIndex = headerLookup.get(configuration.columns.ifcGuid)!;
|
||||
const objectBearingColumnIndexes = [
|
||||
configuration.columns.roomNumber,
|
||||
configuration.columns.roomName,
|
||||
configuration.columns.familyAndType,
|
||||
configuration.columns.selectionMarker,
|
||||
configuration.columns.power,
|
||||
configuration.columns.quantity,
|
||||
...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn),
|
||||
].map((column) => headerLookup.get(column)!);
|
||||
const seenIfcGuids = new Set<string>();
|
||||
|
||||
const rows: ExternalCsvRow[] = parsed.rows.map((cells, index) => {
|
||||
if (index < headerRowIndex) {
|
||||
return { index, cells, classification: "metadata" };
|
||||
}
|
||||
if (index === headerRowIndex) {
|
||||
return { index, cells, classification: "header" };
|
||||
}
|
||||
|
||||
const rawIfcGuid = cells[ifcGuidColumnIndex]?.value ?? "";
|
||||
const ifcGuid = rawIfcGuid.trim();
|
||||
if (ifcGuid) {
|
||||
if (ifcGuid !== rawIfcGuid || !isIfcGuid(ifcGuid)) {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-ifc-guid",
|
||||
`CSV row ${index + 1} contains an invalid IfcGUID.`
|
||||
);
|
||||
}
|
||||
if (seenIfcGuids.has(ifcGuid)) {
|
||||
throw new ExternalCsvParseError(
|
||||
"duplicate-ifc-guid",
|
||||
`CSV contains the IfcGUID more than once: ${ifcGuid}`
|
||||
);
|
||||
}
|
||||
seenIfcGuids.add(ifcGuid);
|
||||
return { index, cells, classification: "object" };
|
||||
}
|
||||
|
||||
const looksLikeObject = objectBearingColumnIndexes.some(
|
||||
(columnIndex) => (cells[columnIndex]?.value.trim() ?? "") !== ""
|
||||
);
|
||||
return {
|
||||
index,
|
||||
cells,
|
||||
classification: looksLikeObject ? "suspect-object" : "passthrough",
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
dialect: {
|
||||
encoding: "utf-8",
|
||||
hasBom,
|
||||
delimiter: configuration.delimiter,
|
||||
lineEnding: resolveLineEnding(parsed.lineEndings),
|
||||
quoteCharacter: '"',
|
||||
quoteAllFields:
|
||||
rows.flatMap((row) => row.cells).length > 0 &&
|
||||
rows.flatMap((row) => row.cells).every((cell) => cell.wasQuoted),
|
||||
hasTrailingLineEnding: parsed.hasTrailingLineEnding,
|
||||
},
|
||||
headerRowIndex,
|
||||
rows,
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeExternalCsv(document: ExternalCsvDocument): Uint8Array {
|
||||
assertSerializableDocument(document);
|
||||
const serializedRows = document.rows.map((row) =>
|
||||
row.cells
|
||||
.map((cell) => serializeCell(cell, document.dialect.delimiter))
|
||||
.join(document.dialect.delimiter)
|
||||
);
|
||||
let text = serializedRows.join(document.dialect.lineEnding);
|
||||
if (document.dialect.hasTrailingLineEnding && document.rows.length > 0) {
|
||||
text += document.dialect.lineEnding;
|
||||
}
|
||||
const encoded = new TextEncoder().encode(text);
|
||||
if (!document.dialect.hasBom) {
|
||||
return encoded;
|
||||
}
|
||||
const result = new Uint8Array(encoded.length + 3);
|
||||
result.set([0xef, 0xbb, 0xbf]);
|
||||
result.set(encoded, 3);
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseRows(text: string, delimiter: string) {
|
||||
const rows: ExternalCsvCell[][] = [];
|
||||
const lineEndings: ExternalCsvLineEnding[] = [];
|
||||
let cells: ExternalCsvCell[] = [];
|
||||
let value = "";
|
||||
let wasQuoted = false;
|
||||
let inQuotes = false;
|
||||
let afterClosingQuote = false;
|
||||
let fieldStarted = false;
|
||||
let hasTrailingLineEnding = false;
|
||||
|
||||
const finishCell = () => {
|
||||
cells.push({ value, wasQuoted });
|
||||
value = "";
|
||||
wasQuoted = false;
|
||||
inQuotes = false;
|
||||
afterClosingQuote = false;
|
||||
fieldStarted = false;
|
||||
};
|
||||
const finishRow = (lineEnding: ExternalCsvLineEnding) => {
|
||||
finishCell();
|
||||
rows.push(cells);
|
||||
cells = [];
|
||||
lineEndings.push(lineEnding);
|
||||
hasTrailingLineEnding = true;
|
||||
};
|
||||
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const character = text[index];
|
||||
if (inQuotes) {
|
||||
if (character === '"') {
|
||||
if (text[index + 1] === '"') {
|
||||
value += '"';
|
||||
index += 1;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
afterClosingQuote = true;
|
||||
}
|
||||
} else {
|
||||
value += character;
|
||||
}
|
||||
hasTrailingLineEnding = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (afterClosingQuote) {
|
||||
if (character !== delimiter && character !== "\r" && character !== "\n") {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-csv",
|
||||
"CSV contains characters after a closing quote."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (character === delimiter) {
|
||||
finishCell();
|
||||
hasTrailingLineEnding = false;
|
||||
continue;
|
||||
}
|
||||
if (character === "\r" || character === "\n") {
|
||||
const lineEnding: ExternalCsvLineEnding =
|
||||
character === "\r" && text[index + 1] === "\n"
|
||||
? "\r\n"
|
||||
: character;
|
||||
if (lineEnding === "\r\n") {
|
||||
index += 1;
|
||||
}
|
||||
finishRow(lineEnding);
|
||||
continue;
|
||||
}
|
||||
if (character === '"') {
|
||||
if (fieldStarted || value.length > 0 || afterClosingQuote) {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-csv",
|
||||
"CSV contains a quote inside an unquoted field."
|
||||
);
|
||||
}
|
||||
wasQuoted = true;
|
||||
inQuotes = true;
|
||||
fieldStarted = true;
|
||||
hasTrailingLineEnding = false;
|
||||
continue;
|
||||
}
|
||||
if (afterClosingQuote) {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-csv",
|
||||
"CSV contains invalid data after a quoted field."
|
||||
);
|
||||
}
|
||||
value += character;
|
||||
fieldStarted = true;
|
||||
hasTrailingLineEnding = false;
|
||||
}
|
||||
|
||||
if (inQuotes) {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-csv",
|
||||
"CSV contains an unterminated quoted field."
|
||||
);
|
||||
}
|
||||
if (!hasTrailingLineEnding || cells.length > 0 || fieldStarted || value.length > 0) {
|
||||
finishCell();
|
||||
rows.push(cells);
|
||||
}
|
||||
|
||||
return { rows, lineEndings, hasTrailingLineEnding };
|
||||
}
|
||||
|
||||
function findHeaderRows(
|
||||
rows: ExternalCsvCell[][],
|
||||
configuration: ExternalCsvConfiguration
|
||||
) {
|
||||
const requiredColumns = new Set([
|
||||
...Object.values(configuration.columns),
|
||||
...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn),
|
||||
]);
|
||||
const matches: number[] = [];
|
||||
rows.forEach((row, index) => {
|
||||
const values = new Set(row.map((cell) => cell.value));
|
||||
if ([...requiredColumns].every((column) => values.has(column))) {
|
||||
matches.push(index);
|
||||
}
|
||||
});
|
||||
return matches;
|
||||
}
|
||||
|
||||
function createHeaderLookup(cells: ExternalCsvCell[]) {
|
||||
return new Map(cells.map((cell, index) => [cell.value, index]));
|
||||
}
|
||||
|
||||
function isIfcGuid(value: string) {
|
||||
return /^[0-9A-Za-z_$]{22}$/.test(value);
|
||||
}
|
||||
|
||||
function resolveLineEnding(lineEndings: ExternalCsvLineEnding[]): ExternalCsvLineEnding {
|
||||
if (lineEndings.length === 0) {
|
||||
return "\r\n";
|
||||
}
|
||||
const unique = new Set(lineEndings);
|
||||
if (unique.size !== 1) {
|
||||
throw new ExternalCsvParseError(
|
||||
"invalid-csv",
|
||||
"CSV contains mixed line endings that cannot be preserved reliably."
|
||||
);
|
||||
}
|
||||
return lineEndings[0];
|
||||
}
|
||||
|
||||
function serializeCell(cell: ExternalCsvCell, delimiter: string) {
|
||||
const mustQuote =
|
||||
cell.wasQuoted ||
|
||||
cell.value.includes(delimiter) ||
|
||||
cell.value.includes('"') ||
|
||||
cell.value.includes("\r") ||
|
||||
cell.value.includes("\n");
|
||||
if (!mustQuote) {
|
||||
return cell.value;
|
||||
}
|
||||
return `"${cell.value.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
function assertSerializableDocument(document: ExternalCsvDocument) {
|
||||
if (document.dialect.encoding !== "utf-8") {
|
||||
throw new Error("Only UTF-8 external CSV documents can be serialized.");
|
||||
}
|
||||
if (
|
||||
document.headerRowIndex < 0 ||
|
||||
document.headerRowIndex >= document.rows.length ||
|
||||
document.rows[document.headerRowIndex]?.classification !== "header"
|
||||
) {
|
||||
throw new Error("External CSV document has an invalid header row.");
|
||||
}
|
||||
document.rows.forEach((row, index) => {
|
||||
if (row.index !== index) {
|
||||
throw new Error("External CSV row indexes must be contiguous and stable.");
|
||||
}
|
||||
for (const cell of row.cells) {
|
||||
if (typeof cell.value !== "string" || typeof cell.wasQuoted !== "boolean") {
|
||||
throw new Error("External CSV document contains an invalid cell.");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function hasUtf8Bom(input: Uint8Array) {
|
||||
return input.length >= 3 && input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue