Build the agent-native compute CRM platform

2026-08-13 01:39:01 -07:00
parent bfd2f8d95a
commit 853bde2265
160 changed files with 61812 additions and 483 deletions
@@ -0,0 +1,314 @@
+import { inflateRawSync } from 'node:zlib';
+
+export const MAX_IMPORT_FILE_BYTES = 5 * 1024 * 1024;
+export const MAX_IMPORT_ROWS = 2_000;
+export const MAX_IMPORT_COLUMNS = 100;
+export const MAX_IMPORT_CELL_CHARS = 10_000;
+const MAX_XLSX_ENTRIES = 256;
+const MAX_XLSX_UNCOMPRESSED_BYTES = 20 * 1024 * 1024;
+
+export interface ParsedTable {
+  fileName: string;
+  sheetName: string | null;
+  headers: string[];
+  rows: string[][];
+  warnings: string[];
+}
+
+export function parseTabularFile(input: {
+  fileName: string;
+  mimeType?: string;
+  bytes: Uint8Array;
+}): ParsedTable {
+  if (input.bytes.byteLength === 0) throw new Error('The selected file is empty.');
+  if (input.bytes.byteLength > MAX_IMPORT_FILE_BYTES) {
+    throw new Error('Import files may not exceed 5 MB.');
+  }
+  const fileName = input.fileName.trim().slice(0, 255);
+  const lowerName = fileName.toLowerCase();
+  if (lowerName.endsWith('.csv') || input.mimeType === 'text/csv') {
+    const text = new TextDecoder('utf-8', { fatal: true }).decode(input.bytes);
+    const { headers, rows, warnings } = parseCsv(text);
+    return { fileName, sheetName: null, headers, rows, warnings };
+  }
+  if (
+    lowerName.endsWith('.xlsx') ||
+    input.mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+  ) {
+    const { sheetName, headers, rows, warnings } = parseXlsx(input.bytes);
+    return { fileName, sheetName, headers, rows, warnings };
+  }
+  throw new Error('Use a UTF-8 CSV or .xlsx workbook. Legacy .xls files are not supported.');
+}
+
+export function parseCsv(source: string): Omit<ParsedTable, 'fileName' | 'sheetName'> {
+  if (source.includes('\0')) throw new Error('The CSV contains invalid null bytes.');
+  const table: string[][] = [];
+  let row: string[] = [];
+  let field = '';
+  let quoted = false;
+
+  const pushField = () => {
+    if (field.length > MAX_IMPORT_CELL_CHARS) throw new Error('A cell exceeds 10,000 characters.');
+    row.push(field);
+    field = '';
+    if (row.length > MAX_IMPORT_COLUMNS) throw new Error('Imports may not exceed 100 columns.');
+  };
+  const pushRow = () => {
+    pushField();
+    table.push(row);
+    row = [];
+    if (table.length > MAX_IMPORT_ROWS + 1) throw new Error('Imports may not exceed 2,000 data rows.');
+  };
+
+  const text = source.charCodeAt(0) === 0xfeff ? source.slice(1) : source;
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text[index]!;
+    if (quoted) {
+      if (character === '"') {
+        if (text[index + 1] === '"') {
+          field += '"';
+          index += 1;
+        } else {
+          quoted = false;
+        }
+      } else {
+        field += character;
+      }
+      continue;
+    }
+    if (character === '"' && field.length === 0) quoted = true;
+    else if (character === ',') pushField();
+    else if (character === '\n') pushRow();
+    else if (character === '\r' && text[index + 1] === '\n') continue;
+    else if (character === '\r') pushRow();
+    else field += character;
+  }
+  if (quoted) throw new Error('The CSV ends inside a quoted cell.');
+  if (field.length > 0 || row.length > 0) pushRow();
+
+  return normaliseTabularRows(table, []);
+}
+
+function parseXlsx(bytes: Uint8Array): {
+  sheetName: string;
+  headers: string[];
+  rows: string[][];
+  warnings: string[];
+} {
+  const entries = readZip(bytes);
+  const workbook = readXml(entries, 'xl/workbook.xml');
+  const relationships = readXml(entries, 'xl/_rels/workbook.xml.rels');
+  rejectActiveXml(workbook);
+  rejectActiveXml(relationships);
+
+  const sheets = [...workbook.matchAll(/<sheet\b[^>]*>/gi)]
+    .map((match) => ({
+      name: xmlAttribute(match[0], 'name'),
+      relationshipId: xmlAttribute(match[0], 'r:id'),
+      hidden: ['hidden', 'veryHidden'].includes(xmlAttribute(match[0], 'state') ?? ''),
+    }))
+    .filter((sheet) => sheet.name && sheet.relationshipId && !sheet.hidden);
+  const firstSheet = sheets[0];
+  if (!firstSheet?.name || !firstSheet.relationshipId) {
+    throw new Error('The workbook has no visible worksheet.');
+  }
+  const relationship = [...relationships.matchAll(/<Relationship\b[^>]*>/gi)]
+    .map((match) => match[0])
+    .find((tag) => xmlAttribute(tag, 'Id') === firstSheet.relationshipId);
+  const target = relationship ? xmlAttribute(relationship, 'Target') : null;
+  if (!target) throw new Error('The workbook worksheet relationship is invalid.');
+  const sheetPath = normaliseZipPath(target.startsWith('/') ? target.slice(1) : `xl/${target}`);
+  const sharedStrings = entries.has('xl/sharedStrings.xml')
+    ? parseSharedStrings(readXml(entries, 'xl/sharedStrings.xml'))
+    : [];
+  const worksheet = readXml(entries, sheetPath);
+  const parsed = parseWorksheetXml(worksheet, sharedStrings);
+  return { sheetName: decodeXml(firstSheet.name), ...parsed };
+}
+
+export function parseWorksheetXml(
+  worksheet: string,
+  sharedStrings: readonly string[] = [],
+): Omit<ParsedTable, 'fileName' | 'sheetName'> {
+  rejectActiveXml(worksheet);
+  const table: string[][] = [];
+  const warnings: string[] = [];
+  let sawFormula = false;
+  for (const rowMatch of worksheet.matchAll(/<row\b[^>]*>([\s\S]*?)<\/row>/gi)) {
+    const row: string[] = [];
+    let sequentialColumn = 0;
+    for (const cellMatch of rowMatch[1]!.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/gi)) {
+      const attributes = cellMatch[1]!;
+      const body = cellMatch[2]!;
+      const reference = xmlAttribute(attributes, 'r');
+      const column = reference ? columnFromReference(reference) : sequentialColumn;
+      if (column >= MAX_IMPORT_COLUMNS) throw new Error('Imports may not exceed 100 columns.');
+      const type = xmlAttribute(attributes, 't') ?? 'n';
+      const rawValue = body.match(/<v\b[^>]*>([\s\S]*?)<\/v>/i)?.[1] ?? '';
+      if (/<f\b/i.test(body)) sawFormula = true;
+      let value: string;
+      if (type === 's') value = sharedStrings[Number(rawValue)] ?? '';
+      else if (type === 'inlineStr') value = extractTextNodes(body);
+      else if (type === 'b') value = rawValue === '1' ? 'true' : 'false';
+      else value = decodeXml(rawValue);
+      if (value.length > MAX_IMPORT_CELL_CHARS) throw new Error('A cell exceeds 10,000 characters.');
+      row[column] = value;
+      sequentialColumn = column + 1;
+    }
+    if (row.some((value) => value !== undefined && value !== '')) table.push(row);
+    if (table.length > MAX_IMPORT_ROWS + 1) throw new Error('Imports may not exceed 2,000 data rows.');
+  }
+  if (sawFormula) {
+    warnings.push('Formula cells were not executed; only cached values stored in the workbook were read.');
+  }
+  return normaliseTabularRows(table, warnings);
+}
+
+export function normaliseTabularRows(
+  table: string[][],
+  warnings: string[],
+): Omit<ParsedTable, 'fileName' | 'sheetName'> {
+  while (table.length > 0 && table.at(-1)!.every((cell) => !cell?.trim())) table.pop();
+  const headerRow = table.shift();
+  if (!headerRow) throw new Error('The file has no header row.');
+  let width = headerRow.length;
+  while (width > 0 && !headerRow[width - 1]?.trim()) width -= 1;
+  if (width === 0) throw new Error('The file has no named columns.');
+  const headers = headerRow.slice(0, width).map((header) => header.trim());
+  if (headers.some((header) => !header)) throw new Error('Every imported column needs a header.');
+  const normalised = headers.map((header) => header.toLocaleLowerCase());
+  if (new Set(normalised).size !== normalised.length) {
+    throw new Error('Column headers must be unique, ignoring letter case.');
+  }
+  const rows = table
+    .map((sourceRow) => Array.from({ length: width }, (_, index) => sourceRow[index] ?? ''))
+    .filter((sourceRow) => sourceRow.some((cell) => cell.trim() !== ''));
+  if (rows.length === 0) throw new Error('The file has headers but no data rows.');
+  if (rows.length > MAX_IMPORT_ROWS) throw new Error('Imports may not exceed 2,000 data rows.');
+  if (rows.some((sourceRow) => sourceRow.some((cell) => /^[=+@]/.test(cell.trim())))) {
+    warnings.push('Formula-like text from the source remains inert text and is never executed.');
+  }
+  return { headers, rows, warnings };
+}
+
+function readZip(bytes: Uint8Array): Map<string, Uint8Array> {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  let end = -1;
+  for (let offset = bytes.byteLength - 22; offset >= Math.max(0, bytes.byteLength - 65_557); offset -= 1) {
+    if (view.getUint32(offset, true) === 0x06054b50) {
+      end = offset;
+      break;
+    }
+  }
+  if (end < 0) throw new Error('The .xlsx ZIP directory is invalid.');
+  const count = view.getUint16(end + 10, true);
+  const centralOffset = view.getUint32(end + 16, true);
+  if (count > MAX_XLSX_ENTRIES || count === 0xffff || centralOffset === 0xffffffff) {
+    throw new Error('The workbook archive is too large or uses unsupported ZIP64 metadata.');
+  }
+  const entries = new Map<string, Uint8Array>();
+  let offset = centralOffset;
+  let totalUncompressed = 0;
+  for (let index = 0; index < count; index += 1) {
+    if (offset + 46 > bytes.byteLength || view.getUint32(offset, true) !== 0x02014b50) {
+      throw new Error('The .xlsx ZIP directory is corrupt.');
+    }
+    const flags = view.getUint16(offset + 8, true);
+    const method = view.getUint16(offset + 10, true);
+    const compressedSize = view.getUint32(offset + 20, true);
+    const uncompressedSize = view.getUint32(offset + 24, true);
+    const nameLength = view.getUint16(offset + 28, true);
+    const extraLength = view.getUint16(offset + 30, true);
+    const commentLength = view.getUint16(offset + 32, true);
+    const localOffset = view.getUint32(offset + 42, true);
+    const name = new TextDecoder().decode(bytes.subarray(offset + 46, offset + 46 + nameLength));
+    const safeName = normaliseZipPath(name);
+    if ((flags & 1) !== 0) throw new Error('Encrypted workbooks are not supported.');
+    if (method !== 0 && method !== 8) throw new Error('The workbook uses unsupported ZIP compression.');
+    totalUncompressed += uncompressedSize;
+    if (totalUncompressed > MAX_XLSX_UNCOMPRESSED_BYTES) {
+      throw new Error('The expanded workbook may not exceed 20 MB.');
+    }
+    if (localOffset + 30 > bytes.byteLength || view.getUint32(localOffset, true) !== 0x04034b50) {
+      throw new Error('The workbook contains an invalid ZIP entry.');
+    }
+    const localNameLength = view.getUint16(localOffset + 26, true);
+    const localExtraLength = view.getUint16(localOffset + 28, true);
+    const dataOffset = localOffset + 30 + localNameLength + localExtraLength;
+    const compressed = bytes.subarray(dataOffset, dataOffset + compressedSize);
+    if (compressed.byteLength !== compressedSize) throw new Error('The workbook ZIP entry is truncated.');
+    const output = method === 0
+      ? Uint8Array.from(compressed)
+      : inflateRawSync(compressed, { maxOutputLength: MAX_XLSX_UNCOMPRESSED_BYTES });
+    if (output.byteLength !== uncompressedSize) throw new Error('The workbook ZIP entry size is inconsistent.');
+    entries.set(safeName, output);
+    offset += 46 + nameLength + extraLength + commentLength;
+  }
+  return entries;
+}
+
+function readXml(entries: Map<string, Uint8Array>, name: string): string {
+  const bytes = entries.get(name);
+  if (!bytes) throw new Error(`The workbook is missing ${name}.`);
+  return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+}
+
+function rejectActiveXml(xml: string): void {
+  if (/<!DOCTYPE|<!ENTITY/i.test(xml)) throw new Error('Workbook XML declarations are not allowed.');
+}
+
+function normaliseZipPath(path: string): string {
+  const segments: string[] = [];
+  for (const segment of path.replace(/\\/g, '/').split('/')) {
+    if (!segment || segment === '.') continue;
+    if (segment === '..') {
+      if (segments.length === 0) throw new Error('The workbook contains an unsafe ZIP path.');
+      segments.pop();
+    } else {
+      segments.push(segment);
+    }
+  }
+  return segments.join('/');
+}
+
+function xmlAttribute(tag: string, name: string): string | null {
+  const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const match = tag.match(new RegExp(`${escaped}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, 'i'));
+  return match ? decodeXml(match[1] ?? match[2] ?? '') : null;
+}
+
+function parseSharedStrings(xml: string): string[] {
+  rejectActiveXml(xml);
+  return [...xml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/gi)].map((match) =>
+    extractTextNodes(match[1]!),
+  );
+}
+
+function extractTextNodes(xml: string): string {
+  return [...xml.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/gi)]
+    .map((match) => decodeXml(match[1]!))
+    .join('');
+}
+
+function decodeXml(value: string): string {
+  return value.replace(/&(?:#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (entity) => {
+    if (entity === '&amp;') return '&';
+    if (entity === '&lt;') return '<';
+    if (entity === '&gt;') return '>';
+    if (entity === '&quot;') return '"';
+    if (entity === '&apos;') return "'";
+    const numeric = entity.startsWith('&#x')
+      ? Number.parseInt(entity.slice(3, -1), 16)
+      : Number.parseInt(entity.slice(2, -1), 10);
+    return Number.isFinite(numeric) ? String.fromCodePoint(numeric) : entity;
+  });
+}
+
+function columnFromReference(reference: string): number {
+  const letters = reference.match(/^[A-Za-z]+/)?.[0];
+  if (!letters) throw new Error('The worksheet contains an invalid cell reference.');
+  let column = 0;
+  for (const letter of letters.toUpperCase()) column = column * 26 + letter.charCodeAt(0) - 64;
+  return column - 1;
+}