This commit is contained in:
@@ -0,0 +1,315 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useMutation, useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
IMPORT_ENTITIES,
|
||||
IMPORT_ENTITY_DEFINITIONS,
|
||||
TEAMS,
|
||||
type ImportEntity,
|
||||
type PermissionGrant,
|
||||
} from '@pig/core';
|
||||
import { AlertTriangle, CheckCircle2, FileSpreadsheet, LoaderCircle, Upload } from 'lucide-react';
|
||||
import { Badge, Button, Card, CardContent, CardHeader, CardTitle, EmptyState, Input } from '@/components/ui';
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectGroup,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Separator } from '@/components/ui/separator';
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table';
|
||||
import { ApiError, get, post } from '@/lib/api';
|
||||
import { can } from '@/lib/permissions';
|
||||
import { usePageTitle } from '@/lib/title';
|
||||
import { NotionImportSource, type ImportedTable } from '@/components/NotionImportSource';
|
||||
import { GoogleSheetsSource } from '@/components/GoogleSheetsSource';
|
||||
|
||||
type ParsedTable = ImportedTable;
|
||||
|
||||
interface PreviewRow {
|
||||
rowNumber: number;
|
||||
key: string;
|
||||
action: 'create' | 'update' | 'error';
|
||||
recordId: string | null;
|
||||
values: Record<string, unknown>;
|
||||
errors: { field: string | null; message: string }[];
|
||||
}
|
||||
|
||||
interface Preview {
|
||||
digest: string;
|
||||
rows: PreviewRow[];
|
||||
counts: { create: number; update: number; error: number };
|
||||
}
|
||||
|
||||
interface CommitResult {
|
||||
created: number;
|
||||
updated: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
const MAX_FILE_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
export function Imports() {
|
||||
usePageTitle('Import data');
|
||||
const [entity, setEntity] = useState<ImportEntity>('account');
|
||||
const [source, setSource] = useState<'file' | 'notion' | 'google'>('file');
|
||||
const [parsed, setParsed] = useState<ParsedTable | null>(null);
|
||||
const [mapping, setMapping] = useState<Record<string, string>>({});
|
||||
const [keySourceColumn, setKeySourceColumn] = useState('');
|
||||
const [preview, setPreview] = useState<Preview | null>(null);
|
||||
const [fileError, setFileError] = useState<string | null>(null);
|
||||
const { data: me } = useQuery({
|
||||
queryKey: ['me'],
|
||||
queryFn: () => get<{ permissions: PermissionGrant[] }>('/api/me'),
|
||||
});
|
||||
const allowed = TEAMS.some((team) => can(me, 'data:import', team));
|
||||
const definition = IMPORT_ENTITY_DEFINITIONS[entity];
|
||||
|
||||
const adoptTable = (table: ParsedTable) => {
|
||||
setParsed(table);
|
||||
setPreview(null);
|
||||
setFileError(null);
|
||||
const nextMapping: Record<string, string> = {};
|
||||
const normalisedHeaders = new Map(table.headers.map((header) => [normalise(header), header]));
|
||||
for (const field of definition.fields) {
|
||||
const source = normalisedHeaders.get(normalise(field.key)) ?? normalisedHeaders.get(normalise(field.label));
|
||||
if (source) nextMapping[field.key] = source;
|
||||
}
|
||||
setMapping(nextMapping);
|
||||
setKeySourceColumn(table.headers[0] ?? '');
|
||||
};
|
||||
|
||||
const parse = useMutation({
|
||||
mutationFn: async (file: File) => {
|
||||
if (file.size > MAX_FILE_BYTES) throw new Error('Import files may not exceed 5 MB.');
|
||||
return post<ParsedTable>('/api/imports/parse', {
|
||||
fileName: file.name,
|
||||
mimeType: file.type || undefined,
|
||||
base64: arrayBufferToBase64(await file.arrayBuffer()),
|
||||
});
|
||||
},
|
||||
onSuccess: adoptTable,
|
||||
onError: (error) => setFileError(errorMessage(error)),
|
||||
});
|
||||
|
||||
const plan = useMemo(() => parsed ? {
|
||||
entity,
|
||||
sourceName: parsed.fileName,
|
||||
headers: parsed.headers,
|
||||
rows: parsed.rows,
|
||||
mapping,
|
||||
keySourceColumn,
|
||||
} : null, [entity, keySourceColumn, mapping, parsed]);
|
||||
const dryRun = useMutation({
|
||||
mutationFn: () => {
|
||||
if (!plan) throw new Error('Choose and map a file first.');
|
||||
return post<Preview>('/api/imports/preview', plan);
|
||||
},
|
||||
onSuccess: setPreview,
|
||||
});
|
||||
const commit = useMutation({
|
||||
mutationFn: () => {
|
||||
if (!plan || !preview) throw new Error('Run the dry run first.');
|
||||
return post<CommitResult>('/api/imports/commit', { ...plan, previewDigest: preview.digest });
|
||||
},
|
||||
});
|
||||
|
||||
const resetForEntity = (next: ImportEntity) => {
|
||||
setEntity(next);
|
||||
setParsed(null);
|
||||
setMapping({});
|
||||
setKeySourceColumn('');
|
||||
setPreview(null);
|
||||
setFileError(null);
|
||||
commit.reset();
|
||||
};
|
||||
|
||||
if (me && !allowed) {
|
||||
return <Card><EmptyState title="Import access required" description="A team administrator with data-import permission must run spreadsheet imports." /></Card>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-5">
|
||||
<header>
|
||||
<h1 className="text-xl font-semibold tracking-tight sm:text-2xl">Import data</h1>
|
||||
<p className="mt-1 max-w-2xl text-sm text-muted">Map a CSV or Excel table into PIG, inspect every create or update, then commit the reviewed plan atomically.</p>
|
||||
</header>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2 xl:grid-cols-4">
|
||||
{IMPORT_ENTITIES.map((candidate) => {
|
||||
const candidateDefinition = IMPORT_ENTITY_DEFINITIONS[candidate];
|
||||
return (
|
||||
<button
|
||||
key={candidate}
|
||||
type="button"
|
||||
aria-pressed={entity === candidate}
|
||||
onClick={() => resetForEntity(candidate)}
|
||||
className={entity === candidate ? 'tap card min-w-0 border-accent p-4 text-left ring-1 ring-accent' : 'tap card min-w-0 p-4 text-left'}
|
||||
>
|
||||
<p className="font-semibold">{candidateDefinition.label}</p>
|
||||
<p className="mt-1 text-xs leading-relaxed text-muted">{candidateDefinition.description}</p>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex items-center gap-3">
|
||||
<FileSpreadsheet className="size-5 text-muted" aria-hidden />
|
||||
<div><CardTitle className="text-base">1. Choose source</CardTitle><p className="mt-1 text-xs text-muted">Import {definition.label.toLocaleLowerCase()} from a file, Notion, or a bounded Google Sheets range</p></div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="flex flex-col gap-3">
|
||||
<div className="grid grid-cols-3 gap-1 rounded-xl bg-subtle p-1">
|
||||
{([
|
||||
['file', 'File'],
|
||||
['notion', 'Notion'],
|
||||
['google', 'Google Sheets'],
|
||||
] as const).map(([value, label]) => (
|
||||
<Button
|
||||
key={value}
|
||||
type="button"
|
||||
variant={source === value ? 'secondary' : 'ghost'}
|
||||
className="min-h-11 px-2"
|
||||
onClick={() => setSource(value)}
|
||||
>
|
||||
{label}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
{source === 'notion' ? <NotionImportSource disabled={!allowed} onTable={adoptTable} /> : null}
|
||||
{source === 'google' ? <GoogleSheetsSource onLoaded={adoptTable} /> : null}
|
||||
{source === 'file' ? (
|
||||
<div className="space-y-3">
|
||||
<Input
|
||||
type="file"
|
||||
accept=".csv,.xlsx,text/csv,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
disabled={!allowed || parse.isPending}
|
||||
onChange={(event) => {
|
||||
const file = event.target.files?.[0];
|
||||
if (file) parse.mutate(file);
|
||||
}}
|
||||
/>
|
||||
{parse.isPending ? <p className="flex items-center gap-2 text-sm text-muted"><LoaderCircle className="animate-spin" aria-hidden />Parsing untrusted cells safely…</p> : null}
|
||||
{fileError ? <ErrorNotice message={fileError} /> : null}
|
||||
</div>
|
||||
) : null}
|
||||
{parsed ? (
|
||||
<div className="flex flex-wrap items-center gap-2 text-sm">
|
||||
<Badge tone="positive"><CheckCircle2 aria-hidden />Parsed</Badge>
|
||||
<span className="font-medium">{parsed.fileName}</span>
|
||||
<span className="text-muted">{parsed.rows.length} rows · {parsed.headers.length} columns{parsed.sheetName ? ` · ${parsed.sheetName}` : ''}</span>
|
||||
</div>
|
||||
) : null}
|
||||
{parsed?.warnings.map((warning) => <p key={warning} className="text-xs text-warning">{warning}</p>)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{parsed ? (
|
||||
<Card>
|
||||
<CardHeader><CardTitle className="text-base">2. Map source columns</CardTitle><p className="text-xs text-muted">Only mapped fields are written. Blank optional cells clear nullable fields; required and non-null defaulted fields are left unchanged on updates.</p></CardHeader>
|
||||
<CardContent className="flex flex-col gap-5">
|
||||
<label className="flex flex-col gap-1.5 text-sm font-medium">
|
||||
Stable source key
|
||||
<Select value={keySourceColumn} onValueChange={(value) => { setKeySourceColumn(value); setPreview(null); }}>
|
||||
<SelectTrigger className="h-11"><SelectValue placeholder="Choose a unique source column" /></SelectTrigger>
|
||||
<SelectContent><SelectGroup>{parsed.headers.map((header) => <SelectItem key={header} value={header}>{header}</SelectItem>)}</SelectGroup></SelectContent>
|
||||
</Select>
|
||||
<span className="text-xs font-normal text-muted">Repeated imports update the same PIG record only when this source value and column name are unchanged.</span>
|
||||
</label>
|
||||
<Separator />
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
|
||||
{definition.fields.map((field) => (
|
||||
<label key={field.key} className="flex min-w-0 flex-col gap-1.5 text-sm font-medium">
|
||||
<span>{field.label}{field.required ? <span className="text-danger"> *</span> : null}</span>
|
||||
<Select value={mapping[field.key] ?? 'none'} onValueChange={(value) => {
|
||||
setMapping((current) => {
|
||||
const next = { ...current };
|
||||
if (value === 'none') delete next[field.key];
|
||||
else next[field.key] = value;
|
||||
return next;
|
||||
});
|
||||
setPreview(null);
|
||||
}}>
|
||||
<SelectTrigger className="h-11"><SelectValue /></SelectTrigger>
|
||||
<SelectContent><SelectGroup><SelectItem value="none">Do not import</SelectItem>{parsed.headers.map((header) => <SelectItem key={header} value={header}>{header}</SelectItem>)}</SelectGroup></SelectContent>
|
||||
</Select>
|
||||
{field.description ? <span className="text-xs font-normal text-muted">{field.description}</span> : null}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
<Button variant="primary" disabled={dryRun.isPending || !keySourceColumn} onClick={() => dryRun.mutate()}>
|
||||
{dryRun.isPending ? <LoaderCircle data-icon="inline-start" className="animate-spin" aria-hidden /> : <Upload data-icon="inline-start" aria-hidden />}
|
||||
{dryRun.isPending ? 'Validating rows…' : 'Run dry-run preview'}
|
||||
</Button>
|
||||
{dryRun.isError ? <ErrorNotice message={errorMessage(dryRun.error)} /> : null}
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : null}
|
||||
|
||||
{preview ? (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-base">3. Review the exact plan</CardTitle>
|
||||
<div className="flex flex-wrap gap-2 pt-1">
|
||||
<Badge tone="positive">{preview.counts.create} create</Badge>
|
||||
<Badge tone="info">{preview.counts.update} update</Badge>
|
||||
<Badge tone={preview.counts.error ? 'danger' : 'neutral'}>{preview.counts.error} errors</Badge>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="flex flex-col gap-4">
|
||||
<div className="scroll-x rounded-lg border border-border">
|
||||
<Table>
|
||||
<TableHeader><TableRow><TableHead>Row</TableHead><TableHead>Source key</TableHead><TableHead>Decision</TableHead><TableHead>Mapped values / errors</TableHead></TableRow></TableHeader>
|
||||
<TableBody>
|
||||
{preview.rows.slice(0, 100).map((row) => (
|
||||
<TableRow key={row.rowNumber}>
|
||||
<TableCell className="nums">{row.rowNumber}</TableCell>
|
||||
<TableCell className="max-w-48 truncate font-medium">{row.key || 'Blank'}</TableCell>
|
||||
<TableCell><Badge tone={row.action === 'create' ? 'positive' : row.action === 'update' ? 'info' : 'danger'}>{row.action}</Badge></TableCell>
|
||||
<TableCell className="min-w-72">
|
||||
{row.errors.length > 0 ? <ul className="flex flex-col gap-1 text-xs text-danger">{row.errors.map((error) => <li key={`${error.field}:${error.message}`}>{error.field ? `${error.field}: ` : ''}{error.message}</li>)}</ul> : <p className="text-xs text-muted">{Object.entries(row.values).slice(0, 4).map(([key, value]) => `${key}: ${String(value)}`).join(' · ')}</p>}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
{preview.rows.length > 100 ? <p className="text-xs text-muted">Showing the first 100 of {preview.rows.length} rows. All rows were validated and will be committed.</p> : null}
|
||||
{preview.counts.error > 0 ? <ErrorNotice message="Fix the source data or mapping, then run the dry run again. No rows can commit while any row has an error." /> : null}
|
||||
{commit.data ? <div role="status" className="rounded-lg border border-positive/30 bg-positive/10 p-4 text-sm text-positive">Committed {commit.data.total} rows: {commit.data.created} created and {commit.data.updated} updated.</div> : null}
|
||||
{commit.isError ? <ErrorNotice message={errorMessage(commit.error)} /> : null}
|
||||
<Button variant="primary" disabled={preview.counts.error > 0 || commit.isPending || Boolean(commit.data)} onClick={() => commit.mutate()}>
|
||||
{commit.isPending ? <LoaderCircle data-icon="inline-start" className="animate-spin" aria-hidden /> : <CheckCircle2 data-icon="inline-start" aria-hidden />}
|
||||
{commit.isPending ? 'Re-checking and committing…' : 'Commit reviewed import'}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ErrorNotice({ message }: { message: string }) {
|
||||
return <div role="alert" className="flex gap-3 rounded-lg border border-danger/30 bg-danger/10 p-3 text-sm text-danger"><AlertTriangle className="mt-0.5 size-4 shrink-0" aria-hidden /><p>{message}</p></div>;
|
||||
}
|
||||
|
||||
function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = '';
|
||||
for (let offset = 0; offset < bytes.length; offset += 0x8000) {
|
||||
binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function normalise(value: string): string {
|
||||
return value.toLocaleLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
if (error instanceof ApiError) return error.message;
|
||||
return error instanceof Error ? error.message : 'The import request failed.';
|
||||
}
|
||||
Reference in New Issue
Block a user