This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { schnorr } from '@noble/curves/secp256k1.js';
|
||||
import { finalizeEvent, getPublicKey, nip19, type Event, verifyEvent } from 'nostr-tools';
|
||||
import {
|
||||
NotificationDeliveryError,
|
||||
type Notification,
|
||||
type NotificationEnvelope,
|
||||
type NotificationReceipt,
|
||||
type Notifier,
|
||||
} from './notifier';
|
||||
|
||||
interface BuzzNotifierOptions {
|
||||
relayUrl: string;
|
||||
privateKey: string;
|
||||
authTag?: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
now?: () => Date;
|
||||
nonce?: () => string;
|
||||
}
|
||||
|
||||
interface BuzzRelayResponse {
|
||||
event_id?: string;
|
||||
accepted?: boolean;
|
||||
}
|
||||
|
||||
export class BuzzNotifier implements Notifier {
|
||||
readonly provider = 'buzz';
|
||||
readonly relayUrl: string;
|
||||
readonly workspaceId: string;
|
||||
private readonly eventsUrl: string;
|
||||
private readonly secretKey: Uint8Array;
|
||||
private readonly publicKey: string;
|
||||
private readonly authTag: string[] | null;
|
||||
private readonly authTagJson: string | null;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly now: () => Date;
|
||||
private readonly nonce: () => string;
|
||||
|
||||
constructor(options: BuzzNotifierOptions) {
|
||||
this.relayUrl = normaliseBuzzRelayUrl(options.relayUrl);
|
||||
this.workspaceId = new URL(this.relayUrl).host;
|
||||
this.eventsUrl = `${this.relayUrl}/events`;
|
||||
this.secretKey = parseBuzzPrivateKey(options.privateKey);
|
||||
this.publicKey = getPublicKey(this.secretKey);
|
||||
const parsedAuth = options.authTag
|
||||
? parseAndVerifyBuzzAuthTag(options.authTag, this.publicKey)
|
||||
: null;
|
||||
this.authTag = parsedAuth;
|
||||
this.authTagJson = parsedAuth ? JSON.stringify(parsedAuth) : null;
|
||||
this.fetchImpl = options.fetchImpl ?? fetch;
|
||||
this.now = options.now ?? (() => new Date());
|
||||
this.nonce = options.nonce ?? randomUUID;
|
||||
}
|
||||
|
||||
async send(
|
||||
envelope: NotificationEnvelope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NotificationReceipt> {
|
||||
if (envelope.workspaceId && envelope.workspaceId !== this.workspaceId) {
|
||||
throw new NotificationDeliveryError('buzz_workspace_mismatch', false);
|
||||
}
|
||||
if (!isUuid(envelope.destination)) {
|
||||
throw new NotificationDeliveryError('invalid_buzz_channel', false);
|
||||
}
|
||||
|
||||
const createdAt = notificationTimestamp(envelope.notification);
|
||||
if (this.authTag && !buzzAuthConditionsAllow(this.authTag[2]!, 9, createdAt)) {
|
||||
throw new NotificationDeliveryError('buzz_auth_tag_conditions', false);
|
||||
}
|
||||
const event = finalizeEvent(
|
||||
{
|
||||
kind: 9,
|
||||
created_at: createdAt,
|
||||
tags: [
|
||||
['h', envelope.destination],
|
||||
[
|
||||
'client',
|
||||
'PIG',
|
||||
createHash('sha256').update(envelope.idempotencyKey).digest('hex'),
|
||||
],
|
||||
...(this.authTag ? [this.authTag] : []),
|
||||
],
|
||||
content: formatBuzzNotification(envelope.notification),
|
||||
},
|
||||
this.secretKey,
|
||||
);
|
||||
const body = JSON.stringify(event);
|
||||
const authorization = this.createNip98Authorization(body);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await this.fetchImpl(this.eventsUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization,
|
||||
'content-type': 'application/json',
|
||||
...(this.authTagJson ? { 'x-auth-tag': this.authTagJson } : {}),
|
||||
},
|
||||
body,
|
||||
signal,
|
||||
});
|
||||
} catch {
|
||||
// The event id is content-addressed and remains stable on a retry, so an
|
||||
// ambiguous network outcome cannot create a second Buzz message.
|
||||
throw new NotificationDeliveryError('buzz_network_error', true);
|
||||
}
|
||||
|
||||
if (response.status === 429) {
|
||||
throw new NotificationDeliveryError(
|
||||
'buzz_rate_limited',
|
||||
true,
|
||||
retryAfterMs(response.headers.get('retry-after')),
|
||||
);
|
||||
}
|
||||
if (response.status === 408 || response.status >= 500) {
|
||||
throw new NotificationDeliveryError('buzz_unavailable', true);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new NotificationDeliveryError(`buzz_http_${response.status}`, false);
|
||||
}
|
||||
|
||||
let result: BuzzRelayResponse;
|
||||
try {
|
||||
result = (await response.json()) as BuzzRelayResponse;
|
||||
} catch {
|
||||
throw new NotificationDeliveryError('invalid_buzz_response', true);
|
||||
}
|
||||
if (result.accepted !== true || !isEventId(result.event_id)) {
|
||||
throw new NotificationDeliveryError('buzz_rejected', false);
|
||||
}
|
||||
if (result.event_id !== event.id) {
|
||||
throw new NotificationDeliveryError('buzz_event_id_mismatch', false);
|
||||
}
|
||||
return { externalId: event.id };
|
||||
}
|
||||
|
||||
private createNip98Authorization(body: string): string {
|
||||
const authEvent = finalizeEvent(
|
||||
{
|
||||
kind: 27235,
|
||||
created_at: Math.floor(this.now().getTime() / 1_000),
|
||||
tags: [
|
||||
['u', this.eventsUrl],
|
||||
['method', 'POST'],
|
||||
['nonce', this.nonce()],
|
||||
['payload', createHash('sha256').update(body).digest('hex')],
|
||||
],
|
||||
content: '',
|
||||
},
|
||||
this.secretKey,
|
||||
);
|
||||
return `Nostr ${Buffer.from(JSON.stringify(authEvent), 'utf8').toString('base64')}`;
|
||||
}
|
||||
}
|
||||
|
||||
export function normaliseBuzzRelayUrl(value: string): string {
|
||||
const trimmed = value.trim().replace(/^wss:/i, 'https:').replace(/^ws:/i, 'http:');
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(trimmed);
|
||||
} catch {
|
||||
throw new Error('BUZZ_RELAY_URL must be a valid http(s) or ws(s) URL.');
|
||||
}
|
||||
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) {
|
||||
throw new Error('BUZZ_RELAY_URL must be a credential-free http(s) or ws(s) URL.');
|
||||
}
|
||||
url.pathname = url.pathname.replace(/\/+$/, '');
|
||||
url.search = '';
|
||||
url.hash = '';
|
||||
return url.toString().replace(/\/$/, '');
|
||||
}
|
||||
|
||||
export function parseBuzzPrivateKey(value: string): Uint8Array {
|
||||
const trimmed = value.trim();
|
||||
let key: Uint8Array;
|
||||
try {
|
||||
if (trimmed.startsWith('nsec1')) {
|
||||
const decoded = nip19.decode(trimmed);
|
||||
if (decoded.type !== 'nsec') throw new Error('wrong key type');
|
||||
key = decoded.data;
|
||||
} else {
|
||||
if (!/^[a-f0-9]{64}$/i.test(trimmed)) throw new Error('invalid hex');
|
||||
key = Uint8Array.from(Buffer.from(trimmed, 'hex'));
|
||||
}
|
||||
getPublicKey(key);
|
||||
} catch {
|
||||
throw new Error('BUZZ_PRIVATE_KEY must be a valid 32-byte hex or nsec private key.');
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
export function parseAndVerifyBuzzAuthTag(value: string, agentPublicKey: string): string[] {
|
||||
let tag: unknown;
|
||||
try {
|
||||
tag = JSON.parse(value);
|
||||
} catch {
|
||||
throw new Error('BUZZ_AUTH_TAG must be a valid JSON array.');
|
||||
}
|
||||
if (
|
||||
!Array.isArray(tag) ||
|
||||
tag.length !== 4 ||
|
||||
tag.some((part) => typeof part !== 'string') ||
|
||||
tag[0] !== 'auth' ||
|
||||
!/^[a-f0-9]{64}$/.test(tag[1] as string) ||
|
||||
!/^[a-f0-9]{128}$/.test(tag[3] as string)
|
||||
) {
|
||||
throw new Error('BUZZ_AUTH_TAG has an invalid NIP-OA structure.');
|
||||
}
|
||||
|
||||
const parts = tag as string[];
|
||||
validateBuzzAuthConditions(parts[2]!);
|
||||
if (parts[1] === agentPublicKey) {
|
||||
throw new Error('BUZZ_AUTH_TAG must be signed by an owner distinct from the agent.');
|
||||
}
|
||||
const digest = createHash('sha256')
|
||||
.update(`nostr:agent-auth:${agentPublicKey}:${parts[2]}`)
|
||||
.digest();
|
||||
let verified = false;
|
||||
try {
|
||||
verified = schnorr.verify(hexBytes(parts[3]!), digest, hexBytes(parts[1]!));
|
||||
} catch {
|
||||
verified = false;
|
||||
}
|
||||
if (!verified) throw new Error('BUZZ_AUTH_TAG signature does not authorize this agent key.');
|
||||
return parts;
|
||||
}
|
||||
|
||||
function notificationTimestamp(notification: Notification): number {
|
||||
const value = notification.kind === 'stage_change' ? notification.changedAt : notification.observedAt;
|
||||
const milliseconds = Date.parse(value);
|
||||
if (!Number.isFinite(milliseconds)) {
|
||||
throw new NotificationDeliveryError('invalid_notification_timestamp', false);
|
||||
}
|
||||
return Math.floor(milliseconds / 1_000);
|
||||
}
|
||||
|
||||
function formatBuzzNotification(notification: Notification): string {
|
||||
if (notification.kind === 'stage_change') {
|
||||
return [
|
||||
`**${notification.dealName}** moved from \`${notification.fromStage}\` to \`${notification.toStage}\`.`,
|
||||
`${notification.dealSide === 'demand' ? 'Demand' : 'Supply'} pipeline stage changed in PIG.`,
|
||||
].join('\n');
|
||||
}
|
||||
return [
|
||||
`**Idle capacity: ${notification.commitmentName}**`,
|
||||
`${notification.gpuType} has ${formatNumber(notification.idleGpuHours)} unsold GPU-hours (${Math.round(notification.utilisation * 100)}% utilised).`,
|
||||
`Idle committed cost: ${formatMoney(notification.idleCostCents)}.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function formatNumber(value: number): string {
|
||||
return new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 }).format(value);
|
||||
}
|
||||
|
||||
function formatMoney(cents: number): string {
|
||||
return new Intl.NumberFormat('en-US', {
|
||||
style: 'currency',
|
||||
currency: 'USD',
|
||||
maximumFractionDigits: 0,
|
||||
}).format(cents / 100);
|
||||
}
|
||||
|
||||
function validateBuzzAuthConditions(conditions: string): void {
|
||||
if (conditions === '') return;
|
||||
for (const clause of conditions.split('&')) {
|
||||
const match = /^(kind=|created_at<|created_at>)(0|[1-9]\d*)$/.exec(clause);
|
||||
if (!match) throw new Error('BUZZ_AUTH_TAG contains invalid NIP-OA conditions.');
|
||||
const value = Number(match[2]);
|
||||
const maximum = match[1] === 'kind=' ? 65_535 : 4_294_967_295;
|
||||
if (!Number.isSafeInteger(value) || value > maximum) {
|
||||
throw new Error('BUZZ_AUTH_TAG contains out-of-range NIP-OA conditions.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buzzAuthConditionsAllow(conditions: string, kind: number, createdAt: number): boolean {
|
||||
if (conditions === '') return true;
|
||||
return conditions.split('&').every((clause) => {
|
||||
if (clause.startsWith('kind=')) return kind === Number(clause.slice(5));
|
||||
if (clause.startsWith('created_at<')) return createdAt < Number(clause.slice(11));
|
||||
if (clause.startsWith('created_at>')) return createdAt > Number(clause.slice(11));
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
function retryAfterMs(value: string | null): number | undefined {
|
||||
const seconds = Number(value);
|
||||
return Number.isFinite(seconds) && seconds > 0 ? Math.min(seconds, 3_600) * 1_000 : undefined;
|
||||
}
|
||||
|
||||
function hexBytes(value: string): Uint8Array {
|
||||
return Uint8Array.from(Buffer.from(value, 'hex'));
|
||||
}
|
||||
|
||||
function isUuid(value: string): boolean {
|
||||
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
|
||||
}
|
||||
|
||||
function isEventId(value: string | undefined): value is string {
|
||||
return Boolean(value && /^[a-f0-9]{64}$/.test(value));
|
||||
}
|
||||
|
||||
export function decodeBuzzAuthorization(value: string): Event | null {
|
||||
if (!value.startsWith('Nostr ')) return null;
|
||||
try {
|
||||
const event = JSON.parse(Buffer.from(value.slice(6), 'base64').toString('utf8')) as Event;
|
||||
return verifyEvent(event) ? event : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,672 @@
|
||||
import {
|
||||
ALLOCATION_STATUSES,
|
||||
CONSUMING_ALLOCATION_STATUSES,
|
||||
RESERVING_ALLOCATION_STATUSES,
|
||||
} from '@pig/core';
|
||||
import type {
|
||||
AllocationStatus,
|
||||
GuaranteeType,
|
||||
GpuSocket,
|
||||
InterconnectType,
|
||||
SecurityTier,
|
||||
} from '@pig/core';
|
||||
import type {
|
||||
Allocation,
|
||||
CapacityCommitment,
|
||||
Database,
|
||||
DemandDeal,
|
||||
NewCapacityCommitment,
|
||||
} from '@pig/db';
|
||||
import { allocations, capacityCommitments, demandDeals } from '@pig/db';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import type { Principal } from '../lib/auth';
|
||||
import { MutationError } from '../lib/mutation';
|
||||
import { quantityAt, type CommitmentShape } from './capacity';
|
||||
|
||||
export type CapacityWriteTransaction = Parameters<Parameters<Database['transaction']>[0]>[0];
|
||||
|
||||
export interface CommitmentCapacity {
|
||||
id: string;
|
||||
gpuCount: number;
|
||||
startsAt: Date;
|
||||
endsAt: Date;
|
||||
totalGpuHours: number;
|
||||
shape: CommitmentShape | null;
|
||||
oversubscriptionPct: number;
|
||||
terminatedAt: Date | null;
|
||||
}
|
||||
|
||||
export interface ReservationCapacity {
|
||||
id?: string;
|
||||
gpuHours: number;
|
||||
startsAt: Date;
|
||||
endsAt: Date;
|
||||
status: AllocationStatus;
|
||||
holdExpiresAt: Date | null;
|
||||
}
|
||||
|
||||
export interface CapacityViolation {
|
||||
code: 'outside_commitment_window' | 'total_capacity_exceeded' | 'shape_capacity_exceeded';
|
||||
message: string;
|
||||
at?: Date;
|
||||
reserved?: number;
|
||||
available?: number;
|
||||
}
|
||||
|
||||
const EPSILON = 1e-7;
|
||||
|
||||
function isLiveReservation(reservation: ReservationCapacity, now: Date): boolean {
|
||||
if (!(RESERVING_ALLOCATION_STATUSES as readonly string[]).includes(reservation.status)) {
|
||||
return false;
|
||||
}
|
||||
return !(
|
||||
reservation.status === 'planned' &&
|
||||
reservation.holdExpiresAt !== null &&
|
||||
reservation.holdExpiresAt <= now
|
||||
);
|
||||
}
|
||||
|
||||
function reservationRate(reservation: ReservationCapacity): number {
|
||||
const durationHours =
|
||||
(reservation.endsAt.getTime() - reservation.startsAt.getTime()) / 3_600_000;
|
||||
return durationHours > 0 ? reservation.gpuHours / durationHours : Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks both the contracted hour budget and every instantaneous shape interval.
|
||||
* The row lock used by the caller turns this pure check into a concurrency-safe
|
||||
* invariant rather than an optimistic preflight that two requests can both pass.
|
||||
*/
|
||||
export function findCapacityViolation(
|
||||
commitment: CommitmentCapacity,
|
||||
existing: readonly ReservationCapacity[],
|
||||
candidate: ReservationCapacity | null,
|
||||
now: Date,
|
||||
): CapacityViolation | null {
|
||||
const reservations = [...existing, ...(candidate ? [candidate] : [])].filter((reservation) =>
|
||||
isLiveReservation(reservation, now),
|
||||
);
|
||||
|
||||
for (const reservation of reservations) {
|
||||
if (
|
||||
reservation.startsAt < commitment.startsAt ||
|
||||
reservation.endsAt > commitment.endsAt ||
|
||||
reservation.endsAt <= reservation.startsAt
|
||||
) {
|
||||
return {
|
||||
code: 'outside_commitment_window',
|
||||
message: 'The allocation window must sit inside the commitment window.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const multiplier = 1 + commitment.oversubscriptionPct / 100;
|
||||
const allowedGpuHours = commitment.totalGpuHours * multiplier;
|
||||
const reservedGpuHours = reservations.reduce((sum, reservation) => sum + reservation.gpuHours, 0);
|
||||
if (reservedGpuHours - allowedGpuHours > EPSILON) {
|
||||
return {
|
||||
code: 'total_capacity_exceeded',
|
||||
message: 'The allocation would exceed the commitment GPU-hour budget.',
|
||||
reserved: reservedGpuHours,
|
||||
available: allowedGpuHours,
|
||||
};
|
||||
}
|
||||
|
||||
const boundaries = new Set<number>([
|
||||
commitment.startsAt.getTime(),
|
||||
commitment.endsAt.getTime(),
|
||||
...(commitment.shape?.intervals.map((boundary) => Date.parse(boundary)) ?? []),
|
||||
]);
|
||||
for (const reservation of reservations) {
|
||||
boundaries.add(reservation.startsAt.getTime());
|
||||
boundaries.add(reservation.endsAt.getTime());
|
||||
}
|
||||
|
||||
const ordered = [...boundaries]
|
||||
.filter(
|
||||
(boundary) =>
|
||||
Number.isFinite(boundary) &&
|
||||
boundary >= commitment.startsAt.getTime() &&
|
||||
boundary <= commitment.endsAt.getTime(),
|
||||
)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
for (let index = 0; index < ordered.length - 1; index++) {
|
||||
const intervalStart = ordered[index]!;
|
||||
const intervalEnd = ordered[index + 1]!;
|
||||
if (intervalEnd <= intervalStart) continue;
|
||||
|
||||
const overlapping = reservations.filter(
|
||||
(reservation) =>
|
||||
reservation.startsAt.getTime() < intervalEnd &&
|
||||
reservation.endsAt.getTime() > intervalStart,
|
||||
);
|
||||
if (overlapping.length === 0) continue;
|
||||
|
||||
const at = new Date(intervalStart + (intervalEnd - intervalStart) / 2);
|
||||
const reserved = overlapping.reduce(
|
||||
(sum, reservation) => sum + reservationRate(reservation),
|
||||
0,
|
||||
);
|
||||
const available = quantityAt(commitment.shape, commitment.gpuCount, at) * multiplier;
|
||||
if (reserved - available > EPSILON) {
|
||||
return {
|
||||
code: 'shape_capacity_exceeded',
|
||||
message: 'The allocation would exceed available GPUs during part of its window.',
|
||||
at,
|
||||
reserved,
|
||||
available,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function commitmentCapacityError(commitment: CommitmentCapacity): string | null {
|
||||
const durationHours =
|
||||
(commitment.endsAt.getTime() - commitment.startsAt.getTime()) / 3_600_000;
|
||||
if (durationHours <= 0) return 'Commitment end must be after its start.';
|
||||
if (commitment.oversubscriptionPct < 0) return 'Oversubscription cannot be negative.';
|
||||
|
||||
const shape = commitment.shape;
|
||||
if (!shape) {
|
||||
if (commitment.totalGpuHours - durationHours * commitment.gpuCount > EPSILON) {
|
||||
return 'Total GPU-hours cannot exceed the flat commitment envelope.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (shape.intervals.length < 2 || shape.quantities.length !== shape.intervals.length - 1) {
|
||||
return 'Shape quantities must have exactly one entry per interval.';
|
||||
}
|
||||
|
||||
const boundaries = shape.intervals.map((boundary) => Date.parse(boundary));
|
||||
if (boundaries.some((boundary) => !Number.isFinite(boundary))) {
|
||||
return 'Shape intervals must be valid ISO-8601 timestamps.';
|
||||
}
|
||||
if (
|
||||
boundaries[0] !== commitment.startsAt.getTime() ||
|
||||
boundaries[boundaries.length - 1] !== commitment.endsAt.getTime()
|
||||
) {
|
||||
return 'Shape intervals must cover the full commitment window.';
|
||||
}
|
||||
for (let index = 0; index < boundaries.length - 1; index++) {
|
||||
if (boundaries[index + 1]! <= boundaries[index]!) {
|
||||
return 'Shape intervals must be strictly ascending.';
|
||||
}
|
||||
}
|
||||
if (
|
||||
shape.quantities.some(
|
||||
(quantity) => !Number.isInteger(quantity) || quantity < 0 || quantity > commitment.gpuCount,
|
||||
)
|
||||
) {
|
||||
return 'Shape quantities must be whole GPUs within the commitment envelope.';
|
||||
}
|
||||
|
||||
const shapedGpuHours = shape.quantities.reduce(
|
||||
(sum, quantity, index) =>
|
||||
sum + ((boundaries[index + 1]! - boundaries[index]!) / 3_600_000) * quantity,
|
||||
0,
|
||||
);
|
||||
if (commitment.totalGpuHours - shapedGpuHours > EPSILON) {
|
||||
return 'Total GPU-hours cannot exceed the capacity described by the shape.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface CommitmentWriteInput {
|
||||
accountId: string;
|
||||
siteId?: string | null;
|
||||
supplyDealId?: string | null;
|
||||
name: string;
|
||||
gpuType: string;
|
||||
socket?: GpuSocket | null;
|
||||
gpuCount: number;
|
||||
interconnectType?: InterconnectType;
|
||||
securityTier?: SecurityTier;
|
||||
startsAt: string;
|
||||
endsAt: string;
|
||||
totalGpuHours: number;
|
||||
costPerGpuHourCents: number;
|
||||
currency?: string;
|
||||
shape?: CommitmentShape | null;
|
||||
colocateWith?: string[];
|
||||
isContiguous?: boolean;
|
||||
minimumSpendCents?: number | null;
|
||||
isAutoRenew?: boolean;
|
||||
noticeDays?: number | null;
|
||||
takeOrPayFloorPct?: number | null;
|
||||
prepaidPct?: number | null;
|
||||
prepaidAmountCents?: number | null;
|
||||
usefulLifeYears?: number | null;
|
||||
salvageValuePct?: number | null;
|
||||
depreciationStartAt?: string | null;
|
||||
costOfCapitalBps?: number | null;
|
||||
financingInstrument?: string | null;
|
||||
oversubscriptionPct?: number;
|
||||
notes?: string | null;
|
||||
terminatedAt?: string | null;
|
||||
}
|
||||
|
||||
export interface AllocationWriteInput {
|
||||
capacityCommitmentId: string;
|
||||
demandDealId: string;
|
||||
gpuHours: number;
|
||||
pricePerGpuHourCents: number;
|
||||
currency?: string;
|
||||
startsAt: string;
|
||||
endsAt: string;
|
||||
status: (typeof CONSUMING_ALLOCATION_STATUSES)[number];
|
||||
guaranteeType?: GuaranteeType;
|
||||
priority?: number;
|
||||
complianceDecisionId?: string | null;
|
||||
notes?: string | null;
|
||||
}
|
||||
|
||||
export interface HoldWriteInput
|
||||
extends Omit<AllocationWriteInput, 'status' | 'pricePerGpuHourCents'> {
|
||||
pricePerGpuHourCents?: number;
|
||||
holdExpiresAt: string;
|
||||
holdOpportunityCostCents?: number | null;
|
||||
}
|
||||
|
||||
export interface ReleaseWriteInput {
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface CommitmentWriteResult {
|
||||
commitment: CapacityCommitment;
|
||||
}
|
||||
|
||||
export interface AllocationWriteResult {
|
||||
allocation: Allocation;
|
||||
commitment: Pick<CapacityCommitment, 'id' | 'name'>;
|
||||
deal: Pick<DemandDeal, 'id' | 'accountId'>;
|
||||
}
|
||||
|
||||
export interface ReleaseWriteResult {
|
||||
allocation: Allocation;
|
||||
accountId?: string;
|
||||
previousStatus: AllocationStatus;
|
||||
}
|
||||
|
||||
function toCapacity(commitment: CapacityCommitment): CommitmentCapacity {
|
||||
return {
|
||||
id: commitment.id,
|
||||
gpuCount: commitment.gpuCount,
|
||||
startsAt: commitment.startsAt,
|
||||
endsAt: commitment.endsAt,
|
||||
totalGpuHours: Number(commitment.totalGpuHours),
|
||||
shape: commitment.shape,
|
||||
oversubscriptionPct: Number(commitment.oversubscriptionPct),
|
||||
terminatedAt: commitment.terminatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function toReservation(allocation: Allocation): ReservationCapacity {
|
||||
return {
|
||||
id: allocation.id,
|
||||
gpuHours: Number(allocation.gpuHours),
|
||||
startsAt: allocation.startsAt,
|
||||
endsAt: allocation.endsAt,
|
||||
status: allocation.status,
|
||||
holdExpiresAt: allocation.holdExpiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
function assertCommitmentValid(commitment: CommitmentCapacity): void {
|
||||
const error = commitmentCapacityError(commitment);
|
||||
if (error) throw new MutationError('invalid_commitment', error, 400);
|
||||
}
|
||||
|
||||
function assertCapacityAvailable(
|
||||
commitment: CommitmentCapacity,
|
||||
existing: readonly Allocation[],
|
||||
candidate: ReservationCapacity | null,
|
||||
now: Date,
|
||||
): void {
|
||||
const violation = findCapacityViolation(
|
||||
commitment,
|
||||
existing.map(toReservation),
|
||||
candidate,
|
||||
now,
|
||||
);
|
||||
if (!violation) return;
|
||||
throw new MutationError(violation.code, violation.message, 409);
|
||||
}
|
||||
|
||||
export class CapacityWriteService {
|
||||
constructor(private readonly tx: CapacityWriteTransaction) {}
|
||||
|
||||
private async lockCommitment(id: string): Promise<CapacityCommitment> {
|
||||
const [commitment] = await this.tx
|
||||
.select()
|
||||
.from(capacityCommitments)
|
||||
.where(eq(capacityCommitments.id, id))
|
||||
.limit(1)
|
||||
.for('update');
|
||||
if (!commitment) throw MutationError.notFound('Capacity commitment');
|
||||
return commitment;
|
||||
}
|
||||
|
||||
private async reservations(commitmentId: string): Promise<Allocation[]> {
|
||||
return this.tx
|
||||
.select()
|
||||
.from(allocations)
|
||||
.where(eq(allocations.capacityCommitmentId, commitmentId));
|
||||
}
|
||||
|
||||
private async demandDeal(id: string): Promise<Pick<DemandDeal, 'id' | 'accountId'>> {
|
||||
const [deal] = await this.tx
|
||||
.select({ id: demandDeals.id, accountId: demandDeals.accountId })
|
||||
.from(demandDeals)
|
||||
.where(eq(demandDeals.id, id))
|
||||
.limit(1);
|
||||
if (!deal) throw MutationError.notFound('Demand deal');
|
||||
return deal;
|
||||
}
|
||||
|
||||
async createCommitment(input: CommitmentWriteInput, now: Date): Promise<CommitmentWriteResult> {
|
||||
const capacity: CommitmentCapacity = {
|
||||
id: 'new',
|
||||
gpuCount: input.gpuCount,
|
||||
startsAt: new Date(input.startsAt),
|
||||
endsAt: new Date(input.endsAt),
|
||||
totalGpuHours: input.totalGpuHours,
|
||||
shape: input.shape ?? null,
|
||||
oversubscriptionPct: input.oversubscriptionPct ?? 0,
|
||||
terminatedAt: null,
|
||||
};
|
||||
assertCommitmentValid(capacity);
|
||||
|
||||
const [commitment] = await this.tx
|
||||
.insert(capacityCommitments)
|
||||
.values({
|
||||
accountId: input.accountId,
|
||||
siteId: input.siteId,
|
||||
supplyDealId: input.supplyDealId,
|
||||
name: input.name,
|
||||
gpuType: input.gpuType,
|
||||
socket: input.socket,
|
||||
gpuCount: input.gpuCount,
|
||||
interconnectType: input.interconnectType,
|
||||
securityTier: input.securityTier,
|
||||
startsAt: capacity.startsAt,
|
||||
endsAt: capacity.endsAt,
|
||||
totalGpuHours: String(input.totalGpuHours),
|
||||
costPerGpuHourCents: input.costPerGpuHourCents,
|
||||
currency: input.currency,
|
||||
shape: input.shape,
|
||||
colocateWith: input.colocateWith,
|
||||
isContiguous: input.isContiguous,
|
||||
minimumSpendCents: input.minimumSpendCents,
|
||||
isAutoRenew: input.isAutoRenew,
|
||||
noticeDays: input.noticeDays,
|
||||
takeOrPayFloorPct:
|
||||
input.takeOrPayFloorPct === undefined || input.takeOrPayFloorPct === null
|
||||
? input.takeOrPayFloorPct
|
||||
: String(input.takeOrPayFloorPct),
|
||||
prepaidPct:
|
||||
input.prepaidPct === undefined || input.prepaidPct === null
|
||||
? input.prepaidPct
|
||||
: String(input.prepaidPct),
|
||||
prepaidAmountCents: input.prepaidAmountCents,
|
||||
usefulLifeYears:
|
||||
input.usefulLifeYears === undefined || input.usefulLifeYears === null
|
||||
? input.usefulLifeYears
|
||||
: String(input.usefulLifeYears),
|
||||
salvageValuePct:
|
||||
input.salvageValuePct === undefined || input.salvageValuePct === null
|
||||
? input.salvageValuePct
|
||||
: String(input.salvageValuePct),
|
||||
depreciationStartAt:
|
||||
input.depreciationStartAt === undefined
|
||||
? undefined
|
||||
: input.depreciationStartAt === null
|
||||
? null
|
||||
: new Date(input.depreciationStartAt),
|
||||
costOfCapitalBps: input.costOfCapitalBps,
|
||||
financingInstrument: input.financingInstrument,
|
||||
oversubscriptionPct: String(input.oversubscriptionPct ?? 0),
|
||||
notes: input.notes,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning();
|
||||
if (!commitment) throw new MutationError('write_failed', 'Commitment was not created.', 409);
|
||||
return { commitment };
|
||||
}
|
||||
|
||||
async updateCommitment(
|
||||
id: string,
|
||||
input: Partial<CommitmentWriteInput>,
|
||||
now: Date,
|
||||
): Promise<CommitmentWriteResult> {
|
||||
const current = await this.lockCommitment(id);
|
||||
const existing = await this.reservations(id);
|
||||
const prospective: CommitmentCapacity = {
|
||||
id,
|
||||
gpuCount: input.gpuCount ?? current.gpuCount,
|
||||
startsAt: input.startsAt ? new Date(input.startsAt) : current.startsAt,
|
||||
endsAt: input.endsAt ? new Date(input.endsAt) : current.endsAt,
|
||||
totalGpuHours: input.totalGpuHours ?? Number(current.totalGpuHours),
|
||||
shape: input.shape === undefined ? current.shape : input.shape,
|
||||
oversubscriptionPct:
|
||||
input.oversubscriptionPct ?? Number(current.oversubscriptionPct),
|
||||
terminatedAt:
|
||||
input.terminatedAt === undefined
|
||||
? current.terminatedAt
|
||||
: input.terminatedAt
|
||||
? new Date(input.terminatedAt)
|
||||
: null,
|
||||
};
|
||||
assertCommitmentValid(prospective);
|
||||
assertCapacityAvailable(prospective, existing, null, now);
|
||||
if (
|
||||
prospective.terminatedAt &&
|
||||
existing.some((allocation) => isLiveReservation(toReservation(allocation), now))
|
||||
) {
|
||||
throw new MutationError(
|
||||
'commitment_in_use',
|
||||
'Release live allocations before terminating the commitment.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
|
||||
const changes: Partial<NewCapacityCommitment> = { updatedAt: now };
|
||||
const assign = <Key extends keyof CommitmentWriteInput>(
|
||||
key: Key,
|
||||
value: NewCapacityCommitment[keyof NewCapacityCommitment],
|
||||
) => {
|
||||
if (input[key] !== undefined) {
|
||||
(changes as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
};
|
||||
assign('accountId', input.accountId);
|
||||
assign('siteId', input.siteId);
|
||||
assign('supplyDealId', input.supplyDealId);
|
||||
assign('name', input.name);
|
||||
assign('gpuType', input.gpuType);
|
||||
assign('socket', input.socket);
|
||||
assign('gpuCount', input.gpuCount);
|
||||
assign('interconnectType', input.interconnectType);
|
||||
assign('securityTier', input.securityTier);
|
||||
assign('startsAt', prospective.startsAt);
|
||||
assign('endsAt', prospective.endsAt);
|
||||
assign('totalGpuHours', String(prospective.totalGpuHours));
|
||||
assign('costPerGpuHourCents', input.costPerGpuHourCents);
|
||||
assign('currency', input.currency);
|
||||
assign('shape', input.shape);
|
||||
assign('colocateWith', input.colocateWith);
|
||||
assign('isContiguous', input.isContiguous);
|
||||
assign('minimumSpendCents', input.minimumSpendCents);
|
||||
assign('isAutoRenew', input.isAutoRenew);
|
||||
assign('noticeDays', input.noticeDays);
|
||||
assign(
|
||||
'takeOrPayFloorPct',
|
||||
input.takeOrPayFloorPct === null ? null : String(input.takeOrPayFloorPct),
|
||||
);
|
||||
assign('prepaidPct', input.prepaidPct === null ? null : String(input.prepaidPct));
|
||||
assign('prepaidAmountCents', input.prepaidAmountCents);
|
||||
assign('usefulLifeYears', input.usefulLifeYears === null ? null : String(input.usefulLifeYears));
|
||||
assign('salvageValuePct', input.salvageValuePct === null ? null : String(input.salvageValuePct));
|
||||
assign(
|
||||
'depreciationStartAt',
|
||||
input.depreciationStartAt ? new Date(input.depreciationStartAt) : null,
|
||||
);
|
||||
assign('costOfCapitalBps', input.costOfCapitalBps);
|
||||
assign('financingInstrument', input.financingInstrument);
|
||||
assign('oversubscriptionPct', String(prospective.oversubscriptionPct));
|
||||
assign('notes', input.notes);
|
||||
assign('terminatedAt', prospective.terminatedAt);
|
||||
|
||||
const [commitment] = await this.tx
|
||||
.update(capacityCommitments)
|
||||
.set(changes)
|
||||
.where(eq(capacityCommitments.id, id))
|
||||
.returning();
|
||||
if (!commitment) throw MutationError.notFound('Capacity commitment');
|
||||
return { commitment };
|
||||
}
|
||||
|
||||
async createAllocation(
|
||||
input: AllocationWriteInput,
|
||||
principal: Principal,
|
||||
now: Date,
|
||||
): Promise<AllocationWriteResult> {
|
||||
const commitment = await this.lockCommitment(input.capacityCommitmentId);
|
||||
if (commitment.terminatedAt) {
|
||||
throw new MutationError('commitment_terminated', 'The commitment is terminated.', 409);
|
||||
}
|
||||
const deal = await this.demandDeal(input.demandDealId);
|
||||
const existing = await this.reservations(commitment.id);
|
||||
const candidate: ReservationCapacity = {
|
||||
gpuHours: input.gpuHours,
|
||||
startsAt: new Date(input.startsAt),
|
||||
endsAt: new Date(input.endsAt),
|
||||
status: input.status,
|
||||
holdExpiresAt: null,
|
||||
};
|
||||
assertCapacityAvailable(toCapacity(commitment), existing, candidate, now);
|
||||
|
||||
const [allocation] = await this.tx
|
||||
.insert(allocations)
|
||||
.values({
|
||||
capacityCommitmentId: commitment.id,
|
||||
demandDealId: deal.id,
|
||||
gpuHours: String(input.gpuHours),
|
||||
pricePerGpuHourCents: input.pricePerGpuHourCents,
|
||||
currency: input.currency,
|
||||
startsAt: candidate.startsAt,
|
||||
endsAt: candidate.endsAt,
|
||||
status: input.status,
|
||||
guaranteeType: input.guaranteeType,
|
||||
priority: input.priority,
|
||||
complianceDecisionId: input.complianceDecisionId,
|
||||
createdByUserId: principal.userId,
|
||||
notes: input.notes,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning();
|
||||
if (!allocation) throw new MutationError('write_failed', 'Allocation was not created.', 409);
|
||||
return { allocation, commitment, deal };
|
||||
}
|
||||
|
||||
async createHold(
|
||||
input: HoldWriteInput,
|
||||
principal: Principal,
|
||||
now: Date,
|
||||
): Promise<AllocationWriteResult> {
|
||||
const holdExpiresAt = new Date(input.holdExpiresAt);
|
||||
if (holdExpiresAt <= now) {
|
||||
throw new MutationError('hold_already_expired', 'A new hold must expire in the future.', 400);
|
||||
}
|
||||
const commitment = await this.lockCommitment(input.capacityCommitmentId);
|
||||
if (commitment.terminatedAt) {
|
||||
throw new MutationError('commitment_terminated', 'The commitment is terminated.', 409);
|
||||
}
|
||||
const deal = await this.demandDeal(input.demandDealId);
|
||||
const existing = await this.reservations(commitment.id);
|
||||
const candidate: ReservationCapacity = {
|
||||
gpuHours: input.gpuHours,
|
||||
startsAt: new Date(input.startsAt),
|
||||
endsAt: new Date(input.endsAt),
|
||||
status: 'planned',
|
||||
holdExpiresAt,
|
||||
};
|
||||
assertCapacityAvailable(toCapacity(commitment), existing, candidate, now);
|
||||
|
||||
const [allocation] = await this.tx
|
||||
.insert(allocations)
|
||||
.values({
|
||||
capacityCommitmentId: commitment.id,
|
||||
demandDealId: deal.id,
|
||||
gpuHours: String(input.gpuHours),
|
||||
pricePerGpuHourCents: input.pricePerGpuHourCents ?? 0,
|
||||
currency: input.currency,
|
||||
startsAt: candidate.startsAt,
|
||||
endsAt: candidate.endsAt,
|
||||
status: 'planned',
|
||||
holdExpiresAt,
|
||||
holdOpportunityCostCents: input.holdOpportunityCostCents,
|
||||
guaranteeType: input.guaranteeType,
|
||||
priority: input.priority,
|
||||
complianceDecisionId: input.complianceDecisionId,
|
||||
createdByUserId: principal.userId,
|
||||
notes: input.notes,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning();
|
||||
if (!allocation) throw new MutationError('write_failed', 'Hold was not created.', 409);
|
||||
return { allocation, commitment, deal };
|
||||
}
|
||||
|
||||
async releaseAllocation(
|
||||
id: string,
|
||||
_input: ReleaseWriteInput,
|
||||
now: Date,
|
||||
): Promise<ReleaseWriteResult> {
|
||||
const [reference] = await this.tx
|
||||
.select({ capacityCommitmentId: allocations.capacityCommitmentId })
|
||||
.from(allocations)
|
||||
.where(eq(allocations.id, id))
|
||||
.limit(1);
|
||||
if (!reference) throw MutationError.notFound('Allocation');
|
||||
await this.lockCommitment(reference.capacityCommitmentId);
|
||||
|
||||
const [current] = await this.tx
|
||||
.select()
|
||||
.from(allocations)
|
||||
.where(eq(allocations.id, id))
|
||||
.limit(1)
|
||||
.for('update');
|
||||
if (!current) throw MutationError.notFound('Allocation');
|
||||
if (current.status === 'released') {
|
||||
throw new MutationError('already_released', 'The allocation is already released.', 409);
|
||||
}
|
||||
if (current.status === 'completed') {
|
||||
throw new MutationError(
|
||||
'completed_allocation',
|
||||
'Completed billable usage cannot be released.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
|
||||
const [allocation] = await this.tx
|
||||
.update(allocations)
|
||||
.set({ status: 'released', releasedAt: now, holdExpiresAt: null, updatedAt: now })
|
||||
.where(eq(allocations.id, id))
|
||||
.returning();
|
||||
if (!allocation) throw MutationError.notFound('Allocation');
|
||||
|
||||
const deal = current.demandDealId
|
||||
? await this.demandDeal(current.demandDealId)
|
||||
: undefined;
|
||||
return { allocation, accountId: deal?.accountId, previousStatus: current.status };
|
||||
}
|
||||
}
|
||||
|
||||
export const VALID_ALLOCATION_STATUSES = ALLOCATION_STATUSES;
|
||||
@@ -13,12 +13,18 @@ import type { Database } from '@pig/db';
|
||||
import {
|
||||
allocations,
|
||||
capacityCommitments,
|
||||
CONSUMING_ALLOCATION_STATUSES,
|
||||
RESERVING_ALLOCATION_STATUSES,
|
||||
complianceDecisionAllowsMatch,
|
||||
inventoryListings,
|
||||
} from '@pig/db';
|
||||
import { computeMargin, breakEvenPricePerGpuHourCents } from '@pig/core';
|
||||
import {
|
||||
computeMargin,
|
||||
breakEvenPricePerGpuHourCents,
|
||||
CONSUMING_ALLOCATION_STATUSES,
|
||||
RESERVING_ALLOCATION_STATUSES,
|
||||
securityTierSatisfies,
|
||||
} from '@pig/core';
|
||||
import type { InterconnectType, SecurityTier } from '@pig/core';
|
||||
import type { ComplianceMatchDecision } from '@pig/db';
|
||||
|
||||
export interface CommitmentShape {
|
||||
intervals: string[];
|
||||
@@ -71,6 +77,7 @@ export function gpuHoursFromShape(shape: CommitmentShape): number {
|
||||
|
||||
export interface AvailabilityRow {
|
||||
commitmentId: string;
|
||||
accountId: string;
|
||||
name: string;
|
||||
gpuType: string;
|
||||
gpuCount: number;
|
||||
@@ -91,6 +98,48 @@ export interface AvailabilityRow {
|
||||
breakEvenPriceCents: number | null;
|
||||
}
|
||||
|
||||
export interface CapacityMatchRequirement {
|
||||
gpuType?: string;
|
||||
gpuTypeAlternatives?: string[];
|
||||
gpuCount: number;
|
||||
totalGpuHours?: number;
|
||||
requiresHighSpeedInterconnect?: boolean;
|
||||
minSecurityTier?: SecurityTier;
|
||||
startsAt?: Date;
|
||||
endsAt?: Date;
|
||||
maxPricePerGpuHourCents?: number;
|
||||
/** `null` means evaluated but missing, and therefore blocks the match. */
|
||||
complianceDecision?: ComplianceMatchDecision | null;
|
||||
}
|
||||
|
||||
export function capacityMeetsRequirement(
|
||||
row: AvailabilityRow,
|
||||
requirement: CapacityMatchRequirement,
|
||||
): boolean {
|
||||
const acceptableTypes = [
|
||||
...(requirement.gpuType ? [requirement.gpuType] : []),
|
||||
...(requirement.gpuTypeAlternatives ?? []),
|
||||
];
|
||||
if (acceptableTypes.length > 0 && !acceptableTypes.includes(row.gpuType)) return false;
|
||||
if (row.gpuCount < requirement.gpuCount) return false;
|
||||
|
||||
if (requirement.requiresHighSpeedInterconnect) {
|
||||
const fast: InterconnectType[] = ['Infiniband', 'RoCE', 'NVLink'];
|
||||
if (!fast.includes(row.interconnectType)) return false;
|
||||
}
|
||||
if (
|
||||
requirement.minSecurityTier &&
|
||||
!securityTierSatisfies(row.securityTier, requirement.minSecurityTier)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (!complianceDecisionAllowsMatch(requirement.complianceDecision)) return false;
|
||||
if (requirement.startsAt && row.startsAt > requirement.startsAt) return false;
|
||||
if (requirement.endsAt && row.endsAt < requirement.endsAt) return false;
|
||||
if (requirement.totalGpuHours && row.availableGpuHours < requirement.totalGpuHours) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export class CapacityService {
|
||||
constructor(private readonly db: Database) {}
|
||||
|
||||
@@ -161,6 +210,7 @@ export class CapacityService {
|
||||
|
||||
return {
|
||||
commitmentId: commitment.id,
|
||||
accountId: commitment.accountId,
|
||||
name: commitment.name,
|
||||
gpuType: commitment.gpuType,
|
||||
gpuCount: commitment.gpuCount,
|
||||
@@ -198,17 +248,7 @@ export class CapacityService {
|
||||
* requirement for high-speed fabric excludes `Ethernet` and `Unknown` alike.
|
||||
* `Unknown` is excluded deliberately: unverified is not the same as adequate.
|
||||
*/
|
||||
async match(requirement: {
|
||||
gpuType?: string;
|
||||
gpuTypeAlternatives?: string[];
|
||||
gpuCount: number;
|
||||
totalGpuHours?: number;
|
||||
requiresHighSpeedInterconnect?: boolean;
|
||||
minSecurityTier?: SecurityTier;
|
||||
startsAt?: Date;
|
||||
endsAt?: Date;
|
||||
maxPricePerGpuHourCents?: number;
|
||||
}): Promise<
|
||||
async match(requirement: CapacityMatchRequirement): Promise<
|
||||
(AvailabilityRow & {
|
||||
/** 0–1. Higher is a better fit. */
|
||||
score: number;
|
||||
@@ -224,24 +264,7 @@ export class CapacityService {
|
||||
const rows = await this.availability({ at: requirement.startsAt ?? new Date() });
|
||||
|
||||
const matches = rows
|
||||
.filter((row) => {
|
||||
if (acceptableTypes.length > 0 && !acceptableTypes.includes(row.gpuType)) return false;
|
||||
if (row.gpuCount < requirement.gpuCount) return false;
|
||||
|
||||
if (requirement.requiresHighSpeedInterconnect) {
|
||||
const fast: InterconnectType[] = ['Infiniband', 'RoCE', 'NVLink'];
|
||||
if (!fast.includes(row.interconnectType)) return false;
|
||||
}
|
||||
if (requirement.minSecurityTier === 'secure_cloud' && row.securityTier !== 'secure_cloud') {
|
||||
return false;
|
||||
}
|
||||
if (requirement.startsAt && row.startsAt > requirement.startsAt) return false;
|
||||
if (requirement.endsAt && row.endsAt < requirement.endsAt) return false;
|
||||
if (requirement.totalGpuHours && row.availableGpuHours < requirement.totalGpuHours) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.filter((row) => capacityMeetsRequirement(row, requirement))
|
||||
.map((row) => {
|
||||
const rationale: string[] = [];
|
||||
let score = 0.5;
|
||||
@@ -268,6 +291,12 @@ export class CapacityService {
|
||||
rationale.push(`${row.interconnectType} fabric meets the training requirement.`);
|
||||
}
|
||||
|
||||
if (requirement.minSecurityTier) {
|
||||
rationale.push(
|
||||
`${row.securityTier} capacity meets the ${requirement.minSecurityTier} security requirement.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Margin headroom: can this be sold above break-even, within the
|
||||
// customer's ceiling?
|
||||
if (requirement.maxPricePerGpuHourCents && row.breakEvenPriceCents != null) {
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { asc, desc, eq, inArray } from 'drizzle-orm';
|
||||
import type { Contract, Database, SlaTerm } from '@pig/db';
|
||||
import {
|
||||
accounts,
|
||||
contractObligations,
|
||||
contracts,
|
||||
slaMetricTargets,
|
||||
slaTerms,
|
||||
} from '@pig/db';
|
||||
|
||||
export interface EffectiveTerm<Value = unknown> {
|
||||
value: Value;
|
||||
sourceContractId: string;
|
||||
inherited: boolean;
|
||||
}
|
||||
|
||||
export type EffectiveTerms = Record<string, EffectiveTerm>;
|
||||
|
||||
const CONTRACT_TERM_FIELDS = [
|
||||
'contractingPartyName',
|
||||
'takeOrPayFloorPct',
|
||||
'prepaidPct',
|
||||
'terminationTier',
|
||||
'assignableOnDefault',
|
||||
'assignmentDeadlineBusinessDays',
|
||||
'effectiveAt',
|
||||
'expiresAt',
|
||||
'isAutoRenew',
|
||||
'noticeDays',
|
||||
'valueCents',
|
||||
'currency',
|
||||
'governingLaw',
|
||||
] as const satisfies readonly (keyof Contract)[];
|
||||
|
||||
const SLA_TERM_FIELDS = [
|
||||
'kind',
|
||||
'uptimeTargetPct',
|
||||
'nodeReplacementHours',
|
||||
'mttrHours',
|
||||
'supportResponseHours',
|
||||
'measurementWindow',
|
||||
'measurementUnit',
|
||||
'remedyType',
|
||||
'abatementTriggerValue',
|
||||
'abatementTriggerUnit',
|
||||
'claimDeadlineValue',
|
||||
'claimDeadlineUnit',
|
||||
'creditExpiryMonths',
|
||||
'isSoleRemedy',
|
||||
'sparePoolObligation',
|
||||
'sparePoolScope',
|
||||
'maintenanceClasses',
|
||||
'reasonableEndeavoursDaysPerYear',
|
||||
'rcaDeliveryHours',
|
||||
'creditSchedule',
|
||||
'creditCapPct',
|
||||
'exclusions',
|
||||
] as const satisfies readonly (keyof SlaTerm)[];
|
||||
|
||||
/**
|
||||
* Resolves explicit child terms before walking toward the master agreement.
|
||||
* Null means "not negotiated here"; false, zero and empty arrays are explicit
|
||||
* values and therefore must not accidentally fall through to a parent.
|
||||
*/
|
||||
export function resolveContractPrecedence(
|
||||
selectedId: string,
|
||||
contractRows: readonly Contract[],
|
||||
slaRows: readonly SlaTerm[],
|
||||
): { chain: Contract[]; contract: EffectiveTerms; sla: EffectiveTerms } {
|
||||
const byId = new Map(contractRows.map((row) => [row.id, row]));
|
||||
const slaByContract = new Map(slaRows.map((row) => [row.contractId, row]));
|
||||
const chain: Contract[] = [];
|
||||
const visited = new Set<string>();
|
||||
let cursor = byId.get(selectedId);
|
||||
|
||||
while (cursor && !visited.has(cursor.id)) {
|
||||
chain.push(cursor);
|
||||
visited.add(cursor.id);
|
||||
cursor = cursor.parentContractId ? byId.get(cursor.parentContractId) : undefined;
|
||||
}
|
||||
|
||||
const contract: EffectiveTerms = {};
|
||||
const sla: EffectiveTerms = {};
|
||||
for (const [index, row] of chain.entries()) {
|
||||
for (const field of CONTRACT_TERM_FIELDS) {
|
||||
const value = row[field];
|
||||
if (!(field in contract) && value !== null && value !== undefined) {
|
||||
contract[field] = { value, sourceContractId: row.id, inherited: index > 0 };
|
||||
}
|
||||
}
|
||||
|
||||
const serviceLevel = slaByContract.get(row.id);
|
||||
if (!serviceLevel) continue;
|
||||
for (const field of SLA_TERM_FIELDS) {
|
||||
const value = serviceLevel[field];
|
||||
if (!(field in sla) && value !== null && value !== undefined) {
|
||||
sla[field] = { value, sourceContractId: row.id, inherited: index > 0 };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { chain, contract, sla };
|
||||
}
|
||||
|
||||
export type RenewalState = 'not_applicable' | 'scheduled' | 'due' | 'expired';
|
||||
|
||||
export function renewalAlarm(
|
||||
contract: Pick<Contract, 'expiresAt' | 'isAutoRenew' | 'noticeDays'>,
|
||||
now = new Date(),
|
||||
): { renewalNoticeAt: Date | null; renewalState: RenewalState } {
|
||||
if (!contract.isAutoRenew || !contract.expiresAt || contract.noticeDays == null) {
|
||||
return { renewalNoticeAt: null, renewalState: 'not_applicable' };
|
||||
}
|
||||
const renewalNoticeAt = new Date(
|
||||
contract.expiresAt.getTime() - contract.noticeDays * 24 * 60 * 60 * 1000,
|
||||
);
|
||||
const renewalState =
|
||||
contract.expiresAt <= now ? 'expired' : renewalNoticeAt <= now ? 'due' : 'scheduled';
|
||||
return { renewalNoticeAt, renewalState };
|
||||
}
|
||||
|
||||
export function validateParentRelationship(
|
||||
child: Pick<Contract, 'id' | 'accountId' | 'side'>,
|
||||
parent: Pick<Contract, 'id' | 'accountId' | 'side'>,
|
||||
ancestors: readonly Pick<Contract, 'id'>[],
|
||||
): string | null {
|
||||
if (child.id === parent.id || ancestors.some((ancestor) => ancestor.id === child.id)) {
|
||||
return 'A contract cannot be its own ancestor.';
|
||||
}
|
||||
if (child.accountId !== parent.accountId) {
|
||||
return 'Parent and child contracts must belong to the same account.';
|
||||
}
|
||||
if (child.side !== parent.side) {
|
||||
return 'Parent and child contracts must govern the same market side.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export class ContractService {
|
||||
constructor(private readonly db: Database) {}
|
||||
|
||||
async list() {
|
||||
const rows = await this.db
|
||||
.select({ contract: contracts, accountName: accounts.name })
|
||||
.from(contracts)
|
||||
.leftJoin(accounts, eq(accounts.id, contracts.accountId))
|
||||
.orderBy(desc(contracts.updatedAt));
|
||||
|
||||
return rows.map((row) => ({
|
||||
...row,
|
||||
...renewalAlarm(row.contract),
|
||||
}));
|
||||
}
|
||||
|
||||
async detail(id: string) {
|
||||
const [selected] = await this.db
|
||||
.select({ contract: contracts, accountName: accounts.name })
|
||||
.from(contracts)
|
||||
.leftJoin(accounts, eq(accounts.id, contracts.accountId))
|
||||
.where(eq(contracts.id, id))
|
||||
.limit(1);
|
||||
if (!selected) return null;
|
||||
|
||||
const accountContracts = await this.db
|
||||
.select()
|
||||
.from(contracts)
|
||||
.where(eq(contracts.accountId, selected.contract.accountId))
|
||||
.orderBy(asc(contracts.createdAt));
|
||||
const contractIds = accountContracts.map((row) => row.id);
|
||||
const serviceLevels = contractIds.length
|
||||
? await this.db.select().from(slaTerms).where(inArray(slaTerms.contractId, contractIds))
|
||||
: [];
|
||||
const termIds = serviceLevels.map((row) => row.id);
|
||||
|
||||
const [metrics, obligations] = await Promise.all([
|
||||
termIds.length
|
||||
? this.db
|
||||
.select()
|
||||
.from(slaMetricTargets)
|
||||
.where(inArray(slaMetricTargets.slaTermId, termIds))
|
||||
: Promise.resolve([]),
|
||||
this.db
|
||||
.select()
|
||||
.from(contractObligations)
|
||||
.where(eq(contractObligations.contractId, id))
|
||||
.orderBy(asc(contractObligations.dueAt)),
|
||||
]);
|
||||
|
||||
const precedence = resolveContractPrecedence(id, accountContracts, serviceLevels);
|
||||
const selectedSla = serviceLevels.find((row) => row.contractId === id) ?? null;
|
||||
|
||||
return {
|
||||
...selected,
|
||||
...renewalAlarm(selected.contract),
|
||||
hierarchy: {
|
||||
chain: precedence.chain,
|
||||
children: accountContracts.filter((row) => row.parentContractId === id),
|
||||
},
|
||||
effectiveTerms: { contract: precedence.contract, sla: precedence.sla },
|
||||
sla: selectedSla,
|
||||
slaMetrics: selectedSla
|
||||
? metrics.filter((row) => row.slaTermId === selectedSla.id)
|
||||
: [],
|
||||
obligations,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,551 @@
|
||||
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import type { Database, GoogleConnection, GoogleOauthFlow } from '@pig/db';
|
||||
import { activities, googleConnections, googleOauthFlows } from '@pig/db';
|
||||
import { and, eq, gt, isNull } from 'drizzle-orm';
|
||||
import type { Principal } from '../lib/auth';
|
||||
import { decryptSecret, encryptionReady, encryptSecret } from '../lib/secrets';
|
||||
import { MutationError } from '../lib/mutation';
|
||||
import {
|
||||
MAX_IMPORT_CELL_CHARS,
|
||||
MAX_IMPORT_COLUMNS,
|
||||
MAX_IMPORT_ROWS,
|
||||
normaliseTabularRows,
|
||||
type ParsedTable,
|
||||
} from './tabular-import';
|
||||
|
||||
export const GOOGLE_OAUTH_SCOPES = [
|
||||
'https://www.googleapis.com/auth/drive.metadata.readonly',
|
||||
'https://www.googleapis.com/auth/spreadsheets.readonly',
|
||||
] as const;
|
||||
const GOOGLE_AUTHORIZATION_ENDPOINT = 'https://accounts.google.com/o/oauth2/v2/auth';
|
||||
const GOOGLE_TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token';
|
||||
const GOOGLE_REVOCATION_ENDPOINT = 'https://oauth2.googleapis.com/revoke';
|
||||
const GOOGLE_DRIVE_FILES_ENDPOINT = 'https://www.googleapis.com/drive/v3/files';
|
||||
const GOOGLE_SHEETS_ENDPOINT = 'https://sheets.googleapis.com/v4/spreadsheets';
|
||||
const OAUTH_FLOW_TTL_MS = 10 * 60 * 1_000;
|
||||
const ACCESS_TOKEN_SKEW_MS = 60 * 1_000;
|
||||
const DRIVE_PAGE_SIZE = 50;
|
||||
const PKCE_PURPOSE = 'google-oauth:pkce-verifier';
|
||||
const REFRESH_TOKEN_PURPOSE = 'google-oauth:refresh-token';
|
||||
const ACCESS_TOKEN_PURPOSE = 'google-oauth:access-token';
|
||||
|
||||
export interface GoogleSheetsConfig {
|
||||
clientId?: string;
|
||||
clientSecret?: string;
|
||||
redirectUri?: string;
|
||||
encryptionKey?: string;
|
||||
publicUrl: string;
|
||||
}
|
||||
|
||||
export interface GoogleConnectionMetadata {
|
||||
configured: boolean;
|
||||
connected: boolean;
|
||||
connectedAt: string | null;
|
||||
scopes: string[];
|
||||
}
|
||||
|
||||
export interface GoogleDriveFile {
|
||||
id: string;
|
||||
name: string;
|
||||
modifiedTime: string | null;
|
||||
}
|
||||
|
||||
export interface GoogleSheetMetadata {
|
||||
sheetId: number;
|
||||
title: string;
|
||||
rowCount: number;
|
||||
columnCount: number;
|
||||
}
|
||||
|
||||
interface GoogleTokenResponse {
|
||||
access_token?: string;
|
||||
expires_in?: number;
|
||||
refresh_token?: string;
|
||||
scope?: string;
|
||||
token_type?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface DriveListResponse {
|
||||
files?: { id?: string; name?: string; modifiedTime?: string }[];
|
||||
nextPageToken?: string;
|
||||
incompleteSearch?: boolean;
|
||||
}
|
||||
|
||||
interface SpreadsheetMetadataResponse {
|
||||
properties?: { title?: string };
|
||||
sheets?: {
|
||||
properties?: {
|
||||
sheetId?: number;
|
||||
title?: string;
|
||||
sheetType?: string;
|
||||
hidden?: boolean;
|
||||
gridProperties?: { rowCount?: number; columnCount?: number };
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
interface ValuesResponse {
|
||||
values?: unknown[][];
|
||||
}
|
||||
|
||||
export class GoogleApiError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly reconnectRequired = false,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'GoogleApiError';
|
||||
}
|
||||
}
|
||||
|
||||
export class GoogleSheetsService {
|
||||
constructor(
|
||||
private readonly db: Database,
|
||||
private readonly config: GoogleSheetsConfig,
|
||||
private readonly fetchImpl: typeof fetch = fetch,
|
||||
) {}
|
||||
|
||||
configured(): boolean {
|
||||
return Boolean(
|
||||
this.config.clientId &&
|
||||
this.config.clientSecret &&
|
||||
this.config.redirectUri &&
|
||||
encryptionReady(this.config.encryptionKey),
|
||||
);
|
||||
}
|
||||
|
||||
async connectionMetadata(userId: string): Promise<GoogleConnectionMetadata> {
|
||||
const [connection] = await this.db
|
||||
.select()
|
||||
.from(googleConnections)
|
||||
.where(eq(googleConnections.userId, userId))
|
||||
.limit(1);
|
||||
return googleConnectionMetadata(this.configured(), connection);
|
||||
}
|
||||
|
||||
async beginOAuth(principal: Principal, now = new Date()): Promise<{ authorizationUrl: string; browserBinding: string }> {
|
||||
this.requireConfigured();
|
||||
const state = randomBytes(32).toString('base64url');
|
||||
const browserBinding = randomBytes(32).toString('base64url');
|
||||
const verifier = randomBytes(32).toString('base64url');
|
||||
const challenge = createHash('sha256').update(verifier).digest('base64url');
|
||||
await this.db.insert(googleOauthFlows).values({
|
||||
userId: principal.userId,
|
||||
stateHash: oauthStateHash(state),
|
||||
browserBindingHash: oauthStateHash(browserBinding),
|
||||
pkceVerifierEncrypted: encryptSecret(verifier, this.config.encryptionKey, PKCE_PURPOSE),
|
||||
expiresAt: new Date(now.getTime() + OAUTH_FLOW_TTL_MS),
|
||||
createdAt: now,
|
||||
});
|
||||
return {
|
||||
authorizationUrl: buildGoogleAuthorizationUrl({
|
||||
clientId: this.config.clientId!,
|
||||
redirectUri: this.config.redirectUri!,
|
||||
state,
|
||||
challenge,
|
||||
}),
|
||||
browserBinding,
|
||||
};
|
||||
}
|
||||
|
||||
async completeOAuth(input: { state: string; code: string; browserBinding: string }, now = new Date()): Promise<void> {
|
||||
this.requireConfigured();
|
||||
const [flow] = await this.db
|
||||
.update(googleOauthFlows)
|
||||
.set({ consumedAt: now })
|
||||
.where(and(
|
||||
eq(googleOauthFlows.stateHash, oauthStateHash(input.state)),
|
||||
isNull(googleOauthFlows.consumedAt),
|
||||
gt(googleOauthFlows.expiresAt, now),
|
||||
))
|
||||
.returning();
|
||||
if (!flow || !oauthFlowMatches(flow, input.state, input.browserBinding, now)) {
|
||||
throw new MutationError('invalid_oauth_state', 'The Google authorization request is invalid or expired.', 400);
|
||||
}
|
||||
const verifier = decryptSecret(
|
||||
flow.pkceVerifierEncrypted,
|
||||
this.config.encryptionKey,
|
||||
PKCE_PURPOSE,
|
||||
);
|
||||
const token = await this.exchangeToken(new URLSearchParams({
|
||||
client_id: this.config.clientId!,
|
||||
client_secret: this.config.clientSecret!,
|
||||
code: input.code,
|
||||
code_verifier: verifier,
|
||||
grant_type: 'authorization_code',
|
||||
redirect_uri: this.config.redirectUri!,
|
||||
}));
|
||||
if (!token.access_token) {
|
||||
throw new GoogleApiError('Google did not return an access token. Connect again.', 502, true);
|
||||
}
|
||||
const [existing] = await this.db
|
||||
.select()
|
||||
.from(googleConnections)
|
||||
.where(eq(googleConnections.userId, flow.userId))
|
||||
.limit(1);
|
||||
const refreshTokenEncrypted = token.refresh_token
|
||||
? encryptSecret(token.refresh_token, this.config.encryptionKey, REFRESH_TOKEN_PURPOSE)
|
||||
: existing?.refreshTokenEncrypted;
|
||||
if (!refreshTokenEncrypted) {
|
||||
throw new GoogleApiError('Google did not grant offline access. Connect again and approve access.', 502, true);
|
||||
}
|
||||
const expiresAt = token.expires_in
|
||||
? new Date(now.getTime() + token.expires_in * 1_000)
|
||||
: null;
|
||||
const scopes = token.scope?.split(/\s+/).filter(Boolean) ?? [...GOOGLE_OAUTH_SCOPES];
|
||||
await this.db.transaction(async (tx) => {
|
||||
await tx.insert(googleConnections).values({
|
||||
userId: flow.userId,
|
||||
refreshTokenEncrypted,
|
||||
accessTokenEncrypted: encryptSecret(token.access_token!, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE),
|
||||
accessTokenExpiresAt: expiresAt,
|
||||
scopes,
|
||||
connectedAt: now,
|
||||
updatedAt: now,
|
||||
}).onConflictDoUpdate({
|
||||
target: googleConnections.userId,
|
||||
set: {
|
||||
refreshTokenEncrypted,
|
||||
accessTokenEncrypted: encryptSecret(token.access_token!, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE),
|
||||
accessTokenExpiresAt: expiresAt,
|
||||
scopes,
|
||||
connectedAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.insert(activities).values({
|
||||
type: 'note',
|
||||
subject: 'Connected Google Sheets import',
|
||||
actorUserId: flow.userId,
|
||||
source: 'manual',
|
||||
occurredAt: now,
|
||||
meta: { action: 'integration.connected', integration: 'google_sheets' },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async disconnect(principal: Principal, now = new Date()): Promise<void> {
|
||||
const [connection] = await this.db
|
||||
.select()
|
||||
.from(googleConnections)
|
||||
.where(eq(googleConnections.userId, principal.userId))
|
||||
.limit(1);
|
||||
if (!connection) return;
|
||||
const token = decryptSecret(
|
||||
connection.refreshTokenEncrypted,
|
||||
this.config.encryptionKey,
|
||||
REFRESH_TOKEN_PURPOSE,
|
||||
);
|
||||
const response = await this.fetchImpl(GOOGLE_REVOCATION_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/x-www-form-urlencoded' },
|
||||
body: new URLSearchParams({ token }),
|
||||
});
|
||||
if (!response.ok && response.status !== 400) {
|
||||
throw new GoogleApiError('Google access could not be revoked. Try again.', response.status);
|
||||
}
|
||||
await this.db.transaction(async (tx) => {
|
||||
await tx.delete(googleConnections).where(eq(googleConnections.userId, principal.userId));
|
||||
await tx.insert(activities).values({
|
||||
type: 'note',
|
||||
subject: 'Disconnected Google Sheets import',
|
||||
actorUserId: principal.userId,
|
||||
source: 'manual',
|
||||
occurredAt: now,
|
||||
meta: { action: 'integration.disconnected', integration: 'google_sheets' },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async listSpreadsheets(
|
||||
userId: string,
|
||||
options: { pageToken?: string; search?: string },
|
||||
): Promise<{ files: GoogleDriveFile[]; nextPageToken: string | null; incomplete: boolean }> {
|
||||
const token = await this.accessToken(userId);
|
||||
const query = [
|
||||
"mimeType='application/vnd.google-apps.spreadsheet'",
|
||||
'trashed=false',
|
||||
...(options.search?.trim()
|
||||
? [`name contains '${escapeDriveQuery(options.search.trim().slice(0, 100))}'`]
|
||||
: []),
|
||||
].join(' and ');
|
||||
const url = new URL(GOOGLE_DRIVE_FILES_ENDPOINT);
|
||||
url.searchParams.set('q', query);
|
||||
url.searchParams.set('spaces', 'drive');
|
||||
url.searchParams.set('orderBy', 'modifiedTime desc,name_natural');
|
||||
url.searchParams.set('pageSize', String(DRIVE_PAGE_SIZE));
|
||||
url.searchParams.set('fields', 'files(id,name,modifiedTime),nextPageToken,incompleteSearch');
|
||||
url.searchParams.set('supportsAllDrives', 'true');
|
||||
url.searchParams.set('includeItemsFromAllDrives', 'true');
|
||||
if (options.pageToken) url.searchParams.set('pageToken', options.pageToken);
|
||||
const body = await this.googleJson<DriveListResponse>(url, token, 'Google Drive could not list spreadsheets.');
|
||||
return {
|
||||
files: (body.files ?? []).flatMap((file) => file.id && file.name
|
||||
? [{ id: file.id, name: file.name, modifiedTime: file.modifiedTime ?? null }]
|
||||
: []),
|
||||
nextPageToken: body.nextPageToken ?? null,
|
||||
incomplete: body.incompleteSearch === true,
|
||||
};
|
||||
}
|
||||
|
||||
async spreadsheetMetadata(
|
||||
userId: string,
|
||||
spreadsheetId: string,
|
||||
): Promise<{ title: string; sheets: GoogleSheetMetadata[] }> {
|
||||
const token = await this.accessToken(userId);
|
||||
const url = new URL(`${GOOGLE_SHEETS_ENDPOINT}/${encodeURIComponent(spreadsheetId)}`);
|
||||
url.searchParams.set(
|
||||
'fields',
|
||||
'properties(title),sheets(properties(sheetId,title,index,sheetType,hidden,gridProperties(rowCount,columnCount)))',
|
||||
);
|
||||
const body = await this.googleJson<SpreadsheetMetadataResponse>(url, token, 'Google Sheets could not read spreadsheet metadata.');
|
||||
const sheets = (body.sheets ?? []).flatMap((sheet) => {
|
||||
const properties = sheet.properties;
|
||||
if (
|
||||
!properties ||
|
||||
properties.sheetType !== 'GRID' ||
|
||||
properties.hidden ||
|
||||
properties.sheetId == null ||
|
||||
!properties.title
|
||||
) return [];
|
||||
return [{
|
||||
sheetId: properties.sheetId,
|
||||
title: properties.title,
|
||||
rowCount: properties.gridProperties?.rowCount ?? 0,
|
||||
columnCount: properties.gridProperties?.columnCount ?? 0,
|
||||
}];
|
||||
});
|
||||
return { title: body.properties?.title ?? 'Google spreadsheet', sheets };
|
||||
}
|
||||
|
||||
async readTable(
|
||||
userId: string,
|
||||
input: { spreadsheetId: string; sheetId: number; range: string },
|
||||
): Promise<ParsedTable> {
|
||||
const metadata = await this.spreadsheetMetadata(userId, input.spreadsheetId);
|
||||
const sheet = metadata.sheets.find((candidate) => candidate.sheetId === input.sheetId);
|
||||
if (!sheet) throw new MutationError('google_sheet_not_found', 'The selected visible grid sheet no longer exists.', 404);
|
||||
const bounded = parseBoundedGoogleRange(input.range, {
|
||||
rowCount: sheet.rowCount,
|
||||
columnCount: sheet.columnCount,
|
||||
});
|
||||
const a1 = `'${sheet.title.replace(/'/g, "''")}'!${bounded.a1}`;
|
||||
const token = await this.accessToken(userId);
|
||||
const url = new URL(
|
||||
`${GOOGLE_SHEETS_ENDPOINT}/${encodeURIComponent(input.spreadsheetId)}/values/${encodeURIComponent(a1)}`,
|
||||
);
|
||||
url.searchParams.set('majorDimension', 'ROWS');
|
||||
// Formatted values expose cached/display results, never executable formula source.
|
||||
url.searchParams.set('valueRenderOption', 'FORMATTED_VALUE');
|
||||
const body = await this.googleJson<ValuesResponse>(url, token, 'Google Sheets could not read the selected range.');
|
||||
const table = normaliseGoogleValues(body.values ?? [], bounded.columns);
|
||||
return {
|
||||
fileName: `Google Sheets · ${metadata.title} · ${sheet.title} · ${bounded.a1}`,
|
||||
sheetName: sheet.title,
|
||||
...table,
|
||||
};
|
||||
}
|
||||
|
||||
private async accessToken(userId: string, now = new Date()): Promise<string> {
|
||||
const [connection] = await this.db
|
||||
.select()
|
||||
.from(googleConnections)
|
||||
.where(eq(googleConnections.userId, userId))
|
||||
.limit(1);
|
||||
if (!connection) throw new GoogleApiError('Connect Google Sheets before selecting a spreadsheet.', 409, true);
|
||||
if (
|
||||
connection.accessTokenEncrypted &&
|
||||
connection.accessTokenExpiresAt &&
|
||||
connection.accessTokenExpiresAt.getTime() > now.getTime() + ACCESS_TOKEN_SKEW_MS
|
||||
) {
|
||||
return decryptSecret(connection.accessTokenEncrypted, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE);
|
||||
}
|
||||
this.requireConfigured();
|
||||
const refreshToken = decryptSecret(
|
||||
connection.refreshTokenEncrypted,
|
||||
this.config.encryptionKey,
|
||||
REFRESH_TOKEN_PURPOSE,
|
||||
);
|
||||
const token = await this.exchangeToken(new URLSearchParams({
|
||||
client_id: this.config.clientId!,
|
||||
client_secret: this.config.clientSecret!,
|
||||
refresh_token: refreshToken,
|
||||
grant_type: 'refresh_token',
|
||||
}));
|
||||
if (!token.access_token) throw new GoogleApiError('Google access expired. Connect again.', 401, true);
|
||||
await this.db.update(googleConnections).set({
|
||||
accessTokenEncrypted: encryptSecret(token.access_token, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE),
|
||||
accessTokenExpiresAt: token.expires_in
|
||||
? new Date(now.getTime() + token.expires_in * 1_000)
|
||||
: null,
|
||||
updatedAt: now,
|
||||
}).where(eq(googleConnections.userId, userId));
|
||||
return token.access_token;
|
||||
}
|
||||
|
||||
private async exchangeToken(parameters: URLSearchParams): Promise<GoogleTokenResponse> {
|
||||
const response = await this.fetchImpl(GOOGLE_TOKEN_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/x-www-form-urlencoded' },
|
||||
body: parameters,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new GoogleApiError('Google authorization failed. Connect again.', response.status, true);
|
||||
}
|
||||
return await response.json() as GoogleTokenResponse;
|
||||
}
|
||||
|
||||
private async googleJson<T>(url: URL, token: string, message: string): Promise<T> {
|
||||
const response = await this.fetchImpl(url, {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new GoogleApiError(message, response.status, response.status === 401 || response.status === 403);
|
||||
}
|
||||
return await response.json() as T;
|
||||
}
|
||||
|
||||
private requireConfigured(): void {
|
||||
if (!this.configured()) {
|
||||
throw new MutationError(
|
||||
'google_not_configured',
|
||||
'Google Sheets import requires OAuth credentials and the settings encryption key.',
|
||||
503,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function oauthStateHash(state: string): string {
|
||||
return createHash('sha256').update(state).digest('hex');
|
||||
}
|
||||
|
||||
export function oauthFlowMatches(
|
||||
flow: Pick<GoogleOauthFlow, 'stateHash' | 'browserBindingHash' | 'expiresAt' | 'consumedAt'>,
|
||||
state: string,
|
||||
browserBinding: string,
|
||||
now: Date,
|
||||
): boolean {
|
||||
if (!flow.consumedAt || flow.consumedAt.getTime() !== now.getTime() || flow.expiresAt <= now) return false;
|
||||
const expected = Buffer.from(flow.stateHash, 'hex');
|
||||
const actual = Buffer.from(oauthStateHash(state), 'hex');
|
||||
const expectedBinding = Buffer.from(flow.browserBindingHash, 'hex');
|
||||
const actualBinding = Buffer.from(oauthStateHash(browserBinding), 'hex');
|
||||
return (
|
||||
expected.length === actual.length &&
|
||||
timingSafeEqual(expected, actual) &&
|
||||
expectedBinding.length === actualBinding.length &&
|
||||
timingSafeEqual(expectedBinding, actualBinding)
|
||||
);
|
||||
}
|
||||
|
||||
export function buildGoogleAuthorizationUrl(input: {
|
||||
clientId: string;
|
||||
redirectUri: string;
|
||||
state: string;
|
||||
challenge: string;
|
||||
}): string {
|
||||
const url = new URL(GOOGLE_AUTHORIZATION_ENDPOINT);
|
||||
url.searchParams.set('client_id', input.clientId);
|
||||
url.searchParams.set('redirect_uri', input.redirectUri);
|
||||
url.searchParams.set('response_type', 'code');
|
||||
url.searchParams.set('scope', GOOGLE_OAUTH_SCOPES.join(' '));
|
||||
url.searchParams.set('state', input.state);
|
||||
url.searchParams.set('code_challenge', input.challenge);
|
||||
url.searchParams.set('code_challenge_method', 'S256');
|
||||
url.searchParams.set('access_type', 'offline');
|
||||
url.searchParams.set('include_granted_scopes', 'true');
|
||||
url.searchParams.set('prompt', 'consent');
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export function googleConnectionMetadata(
|
||||
configured: boolean,
|
||||
connection: GoogleConnection | undefined,
|
||||
): GoogleConnectionMetadata {
|
||||
return {
|
||||
configured,
|
||||
connected: Boolean(connection),
|
||||
connectedAt: connection?.connectedAt.toISOString() ?? null,
|
||||
scopes: connection?.scopes ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
export interface BoundedGoogleRange {
|
||||
a1: string;
|
||||
rows: number;
|
||||
columns: number;
|
||||
}
|
||||
|
||||
export function parseBoundedGoogleRange(
|
||||
value: string,
|
||||
grid?: { rowCount: number; columnCount: number },
|
||||
): BoundedGoogleRange {
|
||||
const match = value.trim().match(/^([A-Za-z]{1,3})([1-9]\d*):([A-Za-z]{1,3})([1-9]\d*)$/);
|
||||
if (!match) {
|
||||
throw new MutationError('invalid_google_range', 'Use a rectangular A1 range such as A1:H500.', 400);
|
||||
}
|
||||
const startColumn = columnNumber(match[1]!);
|
||||
const endColumn = columnNumber(match[3]!);
|
||||
const startRow = Number(match[2]);
|
||||
const endRow = Number(match[4]);
|
||||
if (endColumn < startColumn || endRow < startRow) {
|
||||
throw new MutationError('invalid_google_range', 'The range end must follow its start.', 400);
|
||||
}
|
||||
const columns = endColumn - startColumn + 1;
|
||||
const rows = endRow - startRow + 1;
|
||||
if (columns > MAX_IMPORT_COLUMNS || rows > MAX_IMPORT_ROWS + 1) {
|
||||
throw new MutationError(
|
||||
'google_range_too_large',
|
||||
`Select at most ${MAX_IMPORT_COLUMNS} columns and ${MAX_IMPORT_ROWS + 1} rows including the header.`,
|
||||
400,
|
||||
);
|
||||
}
|
||||
if (grid && (endColumn > grid.columnCount || endRow > grid.rowCount)) {
|
||||
throw new MutationError('google_range_outside_sheet', 'The selected range extends beyond the sheet grid.', 400);
|
||||
}
|
||||
return { a1: `${match[1]!.toUpperCase()}${startRow}:${match[3]!.toUpperCase()}${endRow}`, rows, columns };
|
||||
}
|
||||
|
||||
export function normaliseGoogleValues(
|
||||
values: readonly (readonly unknown[])[],
|
||||
requestedColumns: number,
|
||||
): Omit<ParsedTable, 'fileName' | 'sheetName'> {
|
||||
const table = values.map((row) => {
|
||||
if (row.length > requestedColumns) {
|
||||
throw new MutationError('invalid_google_values', 'Google returned cells outside the requested range.', 502);
|
||||
}
|
||||
return row.map((value) => normaliseGoogleCell(value));
|
||||
});
|
||||
return normaliseTabularRows(table, [
|
||||
'Google formula source was not imported; formula cells use only their formatted cached result.',
|
||||
]);
|
||||
}
|
||||
|
||||
function normaliseGoogleCell(value: unknown): string {
|
||||
if (value == null) return '';
|
||||
if (typeof value === 'boolean') return value ? 'true' : 'false';
|
||||
if (typeof value === 'number') {
|
||||
if (!Number.isFinite(value)) throw new MutationError('invalid_google_values', 'Google returned a non-finite number.', 502);
|
||||
return String(value);
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
if (value.length > MAX_IMPORT_CELL_CHARS) {
|
||||
throw new MutationError('google_cell_too_large', `A Google Sheets cell exceeds ${MAX_IMPORT_CELL_CHARS} characters.`, 400);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
throw new MutationError('invalid_google_values', 'Google returned an unsupported cell value.', 502);
|
||||
}
|
||||
|
||||
function columnNumber(letters: string): number {
|
||||
let result = 0;
|
||||
for (const letter of letters.toUpperCase()) result = result * 26 + letter.charCodeAt(0) - 64;
|
||||
return result;
|
||||
}
|
||||
|
||||
function escapeDriveQuery(value: string): string {
|
||||
return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
IMPORT_ENTITY_DEFINITIONS,
|
||||
type ImportEntity,
|
||||
type ImportFieldDefinition,
|
||||
} from '@pig/core';
|
||||
import type { Database } from '@pig/db';
|
||||
import {
|
||||
accounts,
|
||||
contacts,
|
||||
demandDeals,
|
||||
importIdentities,
|
||||
supplyDeals,
|
||||
} from '@pig/db';
|
||||
import { and, eq, inArray, sql } from 'drizzle-orm';
|
||||
import type { Principal } from '../lib/auth';
|
||||
import { MutationError } from '../lib/mutation';
|
||||
import {
|
||||
MAX_IMPORT_CELL_CHARS,
|
||||
MAX_IMPORT_COLUMNS,
|
||||
MAX_IMPORT_ROWS,
|
||||
} from './tabular-import';
|
||||
|
||||
export type ImportTransaction = Parameters<Parameters<Database['transaction']>[0]>[0];
|
||||
|
||||
export interface ImportPlanInput {
|
||||
entity: ImportEntity;
|
||||
sourceName: string;
|
||||
headers: string[];
|
||||
rows: string[][];
|
||||
mapping: Record<string, string>;
|
||||
keySourceColumn: string;
|
||||
}
|
||||
|
||||
export interface ImportRowError {
|
||||
field: string | null;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface ImportPreviewRow {
|
||||
rowNumber: number;
|
||||
key: string;
|
||||
action: 'create' | 'update' | 'error';
|
||||
recordId: string | null;
|
||||
values: Record<string, unknown>;
|
||||
errors: ImportRowError[];
|
||||
}
|
||||
|
||||
export interface ImportPreview {
|
||||
digest: string;
|
||||
rows: ImportPreviewRow[];
|
||||
counts: { create: number; update: number; error: number };
|
||||
}
|
||||
|
||||
export interface ImportCommitResult {
|
||||
created: number;
|
||||
updated: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
type ImportDb = Database | ImportTransaction;
|
||||
|
||||
export class ImportService {
|
||||
constructor(private readonly db: ImportDb) {}
|
||||
|
||||
async preview(input: ImportPlanInput): Promise<ImportPreview> {
|
||||
assertPlanBounds(input);
|
||||
const definition = IMPORT_ENTITY_DEFINITIONS[input.entity];
|
||||
const knownFields = new Set(definition.fields.map((field) => field.key));
|
||||
const headerSet = new Set(input.headers);
|
||||
for (const [target, source] of Object.entries(input.mapping)) {
|
||||
if (!knownFields.has(target)) throw new MutationError('invalid_mapping', `Unknown target field: ${target}.`, 400);
|
||||
if (!headerSet.has(source)) throw new MutationError('invalid_mapping', `Unknown source column: ${source}.`, 400);
|
||||
}
|
||||
if (!headerSet.has(input.keySourceColumn)) {
|
||||
throw new MutationError('invalid_mapping', 'Select a source column as the stable import key.', 400);
|
||||
}
|
||||
|
||||
const keyIndex = input.headers.indexOf(input.keySourceColumn);
|
||||
const keys = input.rows.map((row) => (row[keyIndex] ?? '').trim());
|
||||
const nonemptyKeys = [...new Set(keys.filter(Boolean))];
|
||||
const identities = nonemptyKeys.length === 0
|
||||
? []
|
||||
: await this.db
|
||||
.select()
|
||||
.from(importIdentities)
|
||||
.where(and(
|
||||
eq(importIdentities.entity, input.entity),
|
||||
eq(importIdentities.keyColumn, input.keySourceColumn),
|
||||
inArray(importIdentities.keyValue, nonemptyKeys),
|
||||
));
|
||||
const identityByKey = new Map(identities.map((identity) => [identity.keyValue, identity]));
|
||||
const duplicateKeys = findDuplicateImportKeys(keys);
|
||||
|
||||
const rows = input.rows.map((row, index): ImportPreviewRow => {
|
||||
const key = keys[index]!;
|
||||
const identity = identityByKey.get(key);
|
||||
const errors: ImportRowError[] = [];
|
||||
if (!key) errors.push({ field: null, message: 'The selected source key is blank.' });
|
||||
if (duplicateKeys.has(key)) errors.push({ field: null, message: 'The source key is duplicated in this file.' });
|
||||
const converted = convertImportRow(
|
||||
input.entity,
|
||||
input.headers,
|
||||
row,
|
||||
input.mapping,
|
||||
!identity,
|
||||
);
|
||||
errors.push(...converted.errors);
|
||||
return {
|
||||
rowNumber: index + 2,
|
||||
key,
|
||||
action: errors.length > 0 ? 'error' : identity ? 'update' : 'create',
|
||||
recordId: identity?.recordId ?? null,
|
||||
values: converted.values,
|
||||
errors,
|
||||
};
|
||||
});
|
||||
|
||||
await this.validateRelationships(input.entity, rows);
|
||||
const counts = rows.reduce(
|
||||
(total, row) => ({ ...total, [row.action]: total[row.action] + 1 }),
|
||||
{ create: 0, update: 0, error: 0 },
|
||||
);
|
||||
const digest = planDigest(input, rows);
|
||||
return { digest, rows, counts };
|
||||
}
|
||||
|
||||
private async validateRelationships(
|
||||
entity: ImportEntity,
|
||||
previewRows: ImportPreviewRow[],
|
||||
): Promise<void> {
|
||||
if (entity !== 'contact' && entity !== 'demand_deal' && entity !== 'supply_deal') return;
|
||||
const accountIds = [...new Set(previewRows
|
||||
.map((row) => row.values.accountId)
|
||||
.filter((value): value is string => typeof value === 'string'))];
|
||||
if (accountIds.length === 0) return;
|
||||
const found = await this.db
|
||||
.select({ id: accounts.id, side: accounts.side })
|
||||
.from(accounts)
|
||||
.where(inArray(accounts.id, accountIds));
|
||||
const accountsById = new Map(found.map((account) => [account.id, account]));
|
||||
for (const row of previewRows) {
|
||||
const accountId = row.values.accountId;
|
||||
if (typeof accountId !== 'string') continue;
|
||||
const account = accountsById.get(accountId);
|
||||
if (!account) row.errors.push({ field: 'accountId', message: 'The account does not exist.' });
|
||||
else if (entity === 'demand_deal' && account.side === 'supply') {
|
||||
row.errors.push({ field: 'accountId', message: 'A demand deal needs a demand or dual-sided account.' });
|
||||
} else if (entity === 'supply_deal' && account.side === 'demand') {
|
||||
row.errors.push({ field: 'accountId', message: 'A supply deal needs a supply or dual-sided account.' });
|
||||
}
|
||||
if (row.errors.length > 0) row.action = 'error';
|
||||
}
|
||||
}
|
||||
|
||||
async commit(
|
||||
input: ImportPlanInput & { previewDigest: string },
|
||||
principal: Principal,
|
||||
now: Date,
|
||||
): Promise<ImportCommitResult> {
|
||||
if (!('execute' in this.db)) throw new Error('Import commit requires a database transaction.');
|
||||
await this.db.execute(
|
||||
sql`select pg_advisory_xact_lock(hashtextextended(${`pig:import:${input.entity}`}, 0))`,
|
||||
);
|
||||
const preview = await this.preview(input);
|
||||
if (preview.digest !== input.previewDigest) {
|
||||
throw new MutationError(
|
||||
'stale_import_preview',
|
||||
'The import plan changed after preview. Run the dry run again before committing.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
if (preview.counts.error > 0) {
|
||||
throw new MutationError(
|
||||
'invalid_import_rows',
|
||||
'Fix every row error and run the dry run again before committing.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
for (const row of preview.rows) {
|
||||
const recordId = await this.writeRecord(input.entity, row, principal, now);
|
||||
if (row.action === 'create') {
|
||||
created += 1;
|
||||
await this.db.insert(importIdentities).values({
|
||||
entity: input.entity,
|
||||
keyColumn: input.keySourceColumn,
|
||||
keyValue: row.key,
|
||||
recordId,
|
||||
importedByUserId: principal.userId,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
} else {
|
||||
updated += 1;
|
||||
await this.db
|
||||
.update(importIdentities)
|
||||
.set({ importedByUserId: principal.userId, updatedAt: now })
|
||||
.where(and(
|
||||
eq(importIdentities.entity, input.entity),
|
||||
eq(importIdentities.keyColumn, input.keySourceColumn),
|
||||
eq(importIdentities.keyValue, row.key),
|
||||
));
|
||||
}
|
||||
}
|
||||
return { created, updated, total: preview.rows.length };
|
||||
}
|
||||
|
||||
private async writeRecord(
|
||||
entity: ImportEntity,
|
||||
row: ImportPreviewRow,
|
||||
principal: Principal,
|
||||
now: Date,
|
||||
): Promise<string> {
|
||||
const values = row.values;
|
||||
if (row.action === 'create') {
|
||||
if (entity === 'account') {
|
||||
const [record] = await this.db.insert(accounts).values({
|
||||
...values,
|
||||
ownerUserId: principal.userId,
|
||||
source: 'import',
|
||||
confidence: 'unverified',
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
} as typeof accounts.$inferInsert).returning({ id: accounts.id });
|
||||
if (record) return record.id;
|
||||
} else if (entity === 'contact') {
|
||||
const [record] = await this.db.insert(contacts).values({
|
||||
...values,
|
||||
ownerUserId: principal.userId,
|
||||
source: 'import',
|
||||
confidence: 'unverified',
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
} as typeof contacts.$inferInsert).returning({ id: contacts.id });
|
||||
if (record) return record.id;
|
||||
} else if (entity === 'demand_deal') {
|
||||
const [record] = await this.db.insert(demandDeals).values({
|
||||
...values,
|
||||
ownerUserId: principal.userId,
|
||||
stageChangedAt: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
} as typeof demandDeals.$inferInsert).returning({ id: demandDeals.id });
|
||||
if (record) return record.id;
|
||||
} else {
|
||||
const [record] = await this.db.insert(supplyDeals).values({
|
||||
...values,
|
||||
ownerUserId: principal.userId,
|
||||
stageChangedAt: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
} as typeof supplyDeals.$inferInsert).returning({ id: supplyDeals.id });
|
||||
if (record) return record.id;
|
||||
}
|
||||
throw new MutationError('write_failed', `Import row ${row.rowNumber} was not created.`, 409);
|
||||
}
|
||||
|
||||
const recordId = row.recordId;
|
||||
if (!recordId) throw new MutationError('missing_import_identity', 'The import identity is incomplete.', 409);
|
||||
if (entity === 'account') {
|
||||
const [record] = await this.db.update(accounts).set({
|
||||
...values,
|
||||
source: 'import',
|
||||
updatedAt: now,
|
||||
} as Partial<typeof accounts.$inferInsert>).where(eq(accounts.id, recordId)).returning({ id: accounts.id });
|
||||
if (record) return record.id;
|
||||
} else if (entity === 'contact') {
|
||||
const [record] = await this.db.update(contacts).set({
|
||||
...values,
|
||||
source: 'import',
|
||||
updatedAt: now,
|
||||
} as Partial<typeof contacts.$inferInsert>).where(eq(contacts.id, recordId)).returning({ id: contacts.id });
|
||||
if (record) return record.id;
|
||||
} else if (entity === 'demand_deal') {
|
||||
const [record] = await this.db.update(demandDeals).set({
|
||||
...values,
|
||||
updatedAt: now,
|
||||
} as Partial<typeof demandDeals.$inferInsert>).where(eq(demandDeals.id, recordId)).returning({ id: demandDeals.id });
|
||||
if (record) return record.id;
|
||||
} else {
|
||||
const [record] = await this.db.update(supplyDeals).set({
|
||||
...values,
|
||||
updatedAt: now,
|
||||
} as Partial<typeof supplyDeals.$inferInsert>).where(eq(supplyDeals.id, recordId)).returning({ id: supplyDeals.id });
|
||||
if (record) return record.id;
|
||||
}
|
||||
throw new MutationError('missing_import_record', `The record for import row ${row.rowNumber} no longer exists.`, 409);
|
||||
}
|
||||
}
|
||||
|
||||
export function convertImportRow(
|
||||
entity: ImportEntity,
|
||||
headers: readonly string[],
|
||||
row: readonly string[],
|
||||
mapping: Readonly<Record<string, string>>,
|
||||
isCreate: boolean,
|
||||
): { values: Record<string, unknown>; errors: ImportRowError[] } {
|
||||
const values: Record<string, unknown> = {};
|
||||
const errors: ImportRowError[] = [];
|
||||
for (const field of IMPORT_ENTITY_DEFINITIONS[entity].fields) {
|
||||
const sourceColumn = mapping[field.key];
|
||||
if (!sourceColumn) {
|
||||
if (isCreate && field.required) errors.push({ field: field.key, message: `${field.label} must be mapped.` });
|
||||
continue;
|
||||
}
|
||||
const sourceIndex = headers.indexOf(sourceColumn);
|
||||
const raw = sourceIndex < 0 ? '' : (row[sourceIndex] ?? '').trim();
|
||||
if (!raw) {
|
||||
if (field.required) errors.push({ field: field.key, message: `${field.label} is required.` });
|
||||
else if (field.clearable !== false) values[field.key] = null;
|
||||
continue;
|
||||
}
|
||||
const converted = convertCell(field, raw);
|
||||
if (converted.error) errors.push({ field: field.key, message: converted.error });
|
||||
else values[field.key] = converted.value;
|
||||
}
|
||||
if (entity === 'account') {
|
||||
const side = values.side;
|
||||
if (values.supplierType && side === 'demand') {
|
||||
errors.push({ field: 'supplierType', message: 'Supplier type requires a supply or dual-sided account.' });
|
||||
}
|
||||
if (values.customerSegment && side === 'supply') {
|
||||
errors.push({ field: 'customerSegment', message: 'Customer segment requires a demand or dual-sided account.' });
|
||||
}
|
||||
}
|
||||
return { values, errors };
|
||||
}
|
||||
|
||||
function convertCell(
|
||||
field: ImportFieldDefinition,
|
||||
raw: string,
|
||||
): { value?: unknown; error?: string } {
|
||||
if (raw.length > (field.maxLength ?? MAX_IMPORT_CELL_CHARS)) {
|
||||
return { error: `${field.label} exceeds ${field.maxLength ?? MAX_IMPORT_CELL_CHARS} characters.` };
|
||||
}
|
||||
if (field.kind === 'text') return { value: raw };
|
||||
if (field.kind === 'email') {
|
||||
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(raw)
|
||||
? { value: raw.toLocaleLowerCase() }
|
||||
: { error: `${field.label} is not a complete email address.` };
|
||||
}
|
||||
if (field.kind === 'url') {
|
||||
try {
|
||||
const url = new URL(raw);
|
||||
return ['http:', 'https:'].includes(url.protocol)
|
||||
? { value: url.toString() }
|
||||
: { error: `${field.label} must use http or https.` };
|
||||
} catch {
|
||||
return { error: `${field.label} is not a valid URL.` };
|
||||
}
|
||||
}
|
||||
if (field.kind === 'uuid') {
|
||||
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(raw)
|
||||
? { value: raw.toLocaleLowerCase() }
|
||||
: { error: `${field.label} must be a PIG UUID.` };
|
||||
}
|
||||
if (field.kind === 'integer') {
|
||||
if (!/^-?\d+$/.test(raw)) return { error: `${field.label} must be a whole number.` };
|
||||
const value = Number(raw);
|
||||
if (!Number.isSafeInteger(value)) return { error: `${field.label} is outside the supported range.` };
|
||||
if (field.min != null && value < field.min) return { error: `${field.label} must be at least ${field.min}.` };
|
||||
if (field.max != null && value > field.max) return { error: `${field.label} must be at most ${field.max}.` };
|
||||
return { value };
|
||||
}
|
||||
if (field.kind === 'decimal') {
|
||||
const value = Number(raw);
|
||||
if (!Number.isFinite(value)) return { error: `${field.label} must be a number.` };
|
||||
if (field.min != null && value < field.min) return { error: `${field.label} must be at least ${field.min}.` };
|
||||
if (field.max != null && value > field.max) return { error: `${field.label} must be at most ${field.max}.` };
|
||||
return { value };
|
||||
}
|
||||
if (field.kind === 'boolean') {
|
||||
const value = raw.toLocaleLowerCase();
|
||||
if (['true', 'yes', '1'].includes(value)) return { value: true };
|
||||
if (['false', 'no', '0'].includes(value)) return { value: false };
|
||||
return { error: `${field.label} must be true/false, yes/no, or 1/0.` };
|
||||
}
|
||||
if (field.kind === 'date') {
|
||||
const value = new Date(raw);
|
||||
return Number.isNaN(value.getTime())
|
||||
? { error: `${field.label} is not a valid date.` }
|
||||
: { value };
|
||||
}
|
||||
if (field.kind === 'currency') {
|
||||
return /^[A-Za-z]{3}$/.test(raw)
|
||||
? { value: raw.toUpperCase() }
|
||||
: { error: `${field.label} must be a three-letter currency code.` };
|
||||
}
|
||||
return field.options?.includes(raw)
|
||||
? { value: raw }
|
||||
: { error: `${field.label} must be one of: ${field.options?.join(', ')}.` };
|
||||
}
|
||||
|
||||
export function findDuplicateImportKeys(keys: readonly string[]): Set<string> {
|
||||
const once = new Set<string>();
|
||||
const duplicates = new Set<string>();
|
||||
for (const key of keys) {
|
||||
if (!key) continue;
|
||||
if (once.has(key)) duplicates.add(key);
|
||||
else once.add(key);
|
||||
}
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
function assertPlanBounds(input: ImportPlanInput): void {
|
||||
if (!input.sourceName.trim() || input.sourceName.length > 255) {
|
||||
throw new MutationError('invalid_import', 'The source file name is invalid.', 400);
|
||||
}
|
||||
if (input.headers.length === 0 || input.headers.length > MAX_IMPORT_COLUMNS) {
|
||||
throw new MutationError('invalid_import', 'Imports need 1–100 columns.', 400);
|
||||
}
|
||||
if (input.rows.length === 0 || input.rows.length > MAX_IMPORT_ROWS) {
|
||||
throw new MutationError('invalid_import', 'Imports need 1–2,000 data rows.', 400);
|
||||
}
|
||||
if (input.headers.some((header) => !header || header.length > 255)) {
|
||||
throw new MutationError('invalid_import', 'Source headers must be non-empty and at most 255 characters.', 400);
|
||||
}
|
||||
if (input.rows.some((row) => row.length > MAX_IMPORT_COLUMNS || row.some((cell) => cell.length > MAX_IMPORT_CELL_CHARS))) {
|
||||
throw new MutationError('invalid_import', 'The imported table exceeds the row, column, or cell limits.', 400);
|
||||
}
|
||||
}
|
||||
|
||||
function planDigest(input: ImportPlanInput, rows: readonly ImportPreviewRow[]): string {
|
||||
const mapping = Object.fromEntries(Object.entries(input.mapping).sort(([left], [right]) => left.localeCompare(right)));
|
||||
return createHash('sha256').update(JSON.stringify({
|
||||
entity: input.entity,
|
||||
sourceName: input.sourceName,
|
||||
headers: input.headers,
|
||||
rows: input.rows,
|
||||
mapping,
|
||||
keySourceColumn: input.keySourceColumn,
|
||||
decisions: rows.map((row) => ({
|
||||
rowNumber: row.rowNumber,
|
||||
key: row.key,
|
||||
action: row.action,
|
||||
recordId: row.recordId,
|
||||
errors: row.errors,
|
||||
})),
|
||||
})).digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { Database } from '@pig/db';
|
||||
import { channelLinks, notificationOutbox, type NotificationOutboxItem } from '@pig/db';
|
||||
import { and, asc, eq, inArray, isNull, lt, lte, or, sql } from 'drizzle-orm';
|
||||
import { z } from 'zod';
|
||||
import type { AvailabilityRow } from './capacity';
|
||||
import {
|
||||
NotificationDeliveryError,
|
||||
NOTIFIER_PROVIDERS,
|
||||
isNotifierProvider,
|
||||
type IdleCapacityNotification,
|
||||
type Notification,
|
||||
type Notifier,
|
||||
type StageChangeNotification,
|
||||
} from './notifier';
|
||||
|
||||
type Transaction = Parameters<Parameters<Database['transaction']>[0]>[0];
|
||||
|
||||
const notificationSchema = z.discriminatedUnion('kind', [
|
||||
z.object({
|
||||
kind: z.literal('stage_change'),
|
||||
accountId: z.string().uuid(),
|
||||
dealId: z.string().uuid(),
|
||||
dealSide: z.enum(['demand', 'supply']),
|
||||
dealName: z.string(),
|
||||
fromStage: z.string(),
|
||||
toStage: z.string(),
|
||||
changedAt: z.string().datetime(),
|
||||
}),
|
||||
z.object({
|
||||
kind: z.literal('idle_capacity'),
|
||||
accountId: z.string().uuid(),
|
||||
commitmentId: z.string().uuid(),
|
||||
commitmentName: z.string(),
|
||||
gpuType: z.string(),
|
||||
idleGpuHours: z.number().nonnegative(),
|
||||
idleCostCents: z.number().int().nonnegative(),
|
||||
utilisation: z.number().min(0).max(1),
|
||||
observedAt: z.string().datetime(),
|
||||
}),
|
||||
]);
|
||||
|
||||
export interface StageChangeEvent extends Omit<StageChangeNotification, 'kind'> {
|
||||
requestedByUserId?: string;
|
||||
}
|
||||
|
||||
export interface IdleCapacityRow extends AvailabilityRow {
|
||||
idleGpuHours: number;
|
||||
idleCostCents: number;
|
||||
}
|
||||
|
||||
export class NotificationOutbox {
|
||||
constructor(private readonly db: Database) {}
|
||||
|
||||
async enqueueStageChange(tx: Transaction, event: StageChangeEvent): Promise<number> {
|
||||
if (event.fromStage === event.toStage) return 0;
|
||||
const links = await tx
|
||||
.select()
|
||||
.from(channelLinks)
|
||||
.where(
|
||||
and(
|
||||
inArray(channelLinks.platform, [...NOTIFIER_PROVIDERS]),
|
||||
eq(channelLinks.accountId, event.accountId),
|
||||
),
|
||||
);
|
||||
const subscribedLinks = links.filter(
|
||||
(link) => isNotifierProvider(link.platform) && link.notifyOn.includes('stage_change'),
|
||||
);
|
||||
if (subscribedLinks.length === 0) return 0;
|
||||
|
||||
const notification: StageChangeNotification = { kind: 'stage_change', ...event };
|
||||
const eventKey = hashKey([
|
||||
notification.kind,
|
||||
notification.dealSide,
|
||||
notification.dealId,
|
||||
notification.changedAt,
|
||||
notification.fromStage,
|
||||
notification.toStage,
|
||||
]);
|
||||
const inserted = await tx
|
||||
.insert(notificationOutbox)
|
||||
.values(
|
||||
subscribedLinks.map((link) => ({
|
||||
provider: link.platform,
|
||||
kind: notification.kind,
|
||||
linkId: link.id,
|
||||
workspaceId: link.workspaceId,
|
||||
destination: link.channelId,
|
||||
payload: { ...notification },
|
||||
idempotencyKey: `${link.platform}:${link.id}:${eventKey}`,
|
||||
requestedByUserId: event.requestedByUserId,
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing({ target: notificationOutbox.idempotencyKey })
|
||||
.returning({ id: notificationOutbox.id });
|
||||
return inserted.length;
|
||||
}
|
||||
|
||||
async enqueueIdleCapacity(rows: IdleCapacityRow[], observedAt = new Date()): Promise<number> {
|
||||
const day = observedAt.toISOString().slice(0, 10);
|
||||
return this.db.transaction(async (tx) => {
|
||||
let count = 0;
|
||||
for (const row of rows) {
|
||||
const links = await tx
|
||||
.select()
|
||||
.from(channelLinks)
|
||||
.where(
|
||||
and(
|
||||
inArray(channelLinks.platform, [...NOTIFIER_PROVIDERS]),
|
||||
eq(channelLinks.accountId, row.accountId),
|
||||
),
|
||||
);
|
||||
const subscribedLinks = links.filter(
|
||||
(link) => isNotifierProvider(link.platform) && link.notifyOn.includes('idle_capacity'),
|
||||
);
|
||||
if (subscribedLinks.length === 0) continue;
|
||||
|
||||
const notification: IdleCapacityNotification = {
|
||||
kind: 'idle_capacity',
|
||||
accountId: row.accountId,
|
||||
commitmentId: row.commitmentId,
|
||||
commitmentName: row.name,
|
||||
gpuType: row.gpuType,
|
||||
idleGpuHours: row.idleGpuHours,
|
||||
idleCostCents: row.idleCostCents,
|
||||
utilisation: row.utilisation,
|
||||
observedAt: observedAt.toISOString(),
|
||||
};
|
||||
const inserted = await tx
|
||||
.insert(notificationOutbox)
|
||||
.values(
|
||||
subscribedLinks.map((link) => ({
|
||||
provider: link.platform,
|
||||
kind: notification.kind,
|
||||
linkId: link.id,
|
||||
workspaceId: link.workspaceId,
|
||||
destination: link.channelId,
|
||||
payload: { ...notification },
|
||||
idempotencyKey: `${link.platform}:${link.id}:idle:${row.commitmentId}:${day}`,
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing({ target: notificationOutbox.idempotencyKey })
|
||||
.returning({ id: notificationOutbox.id });
|
||||
count += inserted.length;
|
||||
}
|
||||
return count;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface NotificationWorkerOptions {
|
||||
workerId?: string;
|
||||
leaseSeconds?: number;
|
||||
pollIntervalMs?: number;
|
||||
}
|
||||
|
||||
export class NotificationWorker {
|
||||
private readonly workerId: string;
|
||||
private readonly leaseSeconds: number;
|
||||
private readonly pollIntervalMs: number;
|
||||
|
||||
constructor(
|
||||
private readonly db: Database,
|
||||
private readonly notifier: Notifier,
|
||||
options: NotificationWorkerOptions = {},
|
||||
) {
|
||||
this.workerId = options.workerId ?? `notification-${process.pid}-${randomUUID()}`;
|
||||
this.leaseSeconds = options.leaseSeconds ?? 30;
|
||||
this.pollIntervalMs = options.pollIntervalMs ?? 2_000;
|
||||
}
|
||||
|
||||
start(): () => void {
|
||||
let stopped = false;
|
||||
let working = false;
|
||||
const tick = () => {
|
||||
if (stopped || working) return;
|
||||
working = true;
|
||||
void this.processNext()
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
working = false;
|
||||
});
|
||||
};
|
||||
const timer = setInterval(tick, this.pollIntervalMs);
|
||||
timer.unref();
|
||||
tick();
|
||||
return () => {
|
||||
stopped = true;
|
||||
clearInterval(timer);
|
||||
};
|
||||
}
|
||||
|
||||
async processNext(now = new Date()): Promise<boolean> {
|
||||
const item = await this.claimNext(now);
|
||||
if (!item) return false;
|
||||
|
||||
const parsed = notificationSchema.safeParse(item.payload);
|
||||
if (!parsed.success) {
|
||||
await this.finishFailure(item, new NotificationDeliveryError('invalid_payload', false), now);
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const receipt = await this.notifier.send({
|
||||
idempotencyKey: item.idempotencyKey,
|
||||
destination: item.destination,
|
||||
workspaceId: item.workspaceId,
|
||||
notification: parsed.data as Notification,
|
||||
});
|
||||
await this.db
|
||||
.update(notificationOutbox)
|
||||
.set({
|
||||
status: 'delivered',
|
||||
deliveredAt: new Date(),
|
||||
externalId: receipt.externalId,
|
||||
leasedBy: null,
|
||||
leasedUntil: null,
|
||||
error: null,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(notificationOutbox.id, item.id),
|
||||
eq(notificationOutbox.leasedBy, this.workerId),
|
||||
eq(notificationOutbox.status, 'leased'),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
const deliveryError =
|
||||
error instanceof NotificationDeliveryError
|
||||
? error
|
||||
: new NotificationDeliveryError('provider_error', true);
|
||||
await this.finishFailure(item, deliveryError, new Date());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private async claimNext(now: Date): Promise<NotificationOutboxItem | null> {
|
||||
return this.db.transaction(async (tx) => {
|
||||
const [candidate] = await tx
|
||||
.select()
|
||||
.from(notificationOutbox)
|
||||
.where(
|
||||
and(
|
||||
eq(notificationOutbox.provider, this.notifier.provider),
|
||||
or(
|
||||
eq(notificationOutbox.status, 'pending'),
|
||||
and(
|
||||
eq(notificationOutbox.status, 'leased'),
|
||||
or(isNull(notificationOutbox.leasedUntil), lt(notificationOutbox.leasedUntil, now)),
|
||||
),
|
||||
),
|
||||
lte(notificationOutbox.dueAt, now),
|
||||
sql`${notificationOutbox.attempts} < ${notificationOutbox.maxAttempts}`,
|
||||
),
|
||||
)
|
||||
.orderBy(asc(notificationOutbox.dueAt), asc(notificationOutbox.createdAt))
|
||||
.limit(1)
|
||||
.for('update', { skipLocked: true });
|
||||
if (!candidate) return null;
|
||||
|
||||
const [claimed] = await tx
|
||||
.update(notificationOutbox)
|
||||
.set({
|
||||
status: 'leased',
|
||||
leasedBy: this.workerId,
|
||||
leasedUntil: new Date(now.getTime() + this.leaseSeconds * 1_000),
|
||||
attempts: sql`${notificationOutbox.attempts} + 1`,
|
||||
error: null,
|
||||
})
|
||||
.where(eq(notificationOutbox.id, candidate.id))
|
||||
.returning();
|
||||
return claimed ?? null;
|
||||
});
|
||||
}
|
||||
|
||||
private async finishFailure(
|
||||
item: NotificationOutboxItem,
|
||||
error: NotificationDeliveryError,
|
||||
now: Date,
|
||||
): Promise<void> {
|
||||
const retry = error.retryable && item.attempts < item.maxAttempts;
|
||||
const backoff = error.retryAfterMs ?? retryBackoffMs(item.attempts);
|
||||
await this.db
|
||||
.update(notificationOutbox)
|
||||
.set({
|
||||
status: retry ? 'pending' : 'failed',
|
||||
dueAt: retry ? new Date(now.getTime() + backoff) : item.dueAt,
|
||||
leasedBy: null,
|
||||
leasedUntil: null,
|
||||
error: error.code,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(notificationOutbox.id, item.id),
|
||||
eq(notificationOutbox.leasedBy, this.workerId),
|
||||
eq(notificationOutbox.status, 'leased'),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function retryBackoffMs(attempts: number): number {
|
||||
return Math.min(60 * 60_000, 30_000 * 2 ** Math.max(0, attempts - 1));
|
||||
}
|
||||
|
||||
function hashKey(parts: string[]): string {
|
||||
return createHash('sha256').update(JSON.stringify(parts)).digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
export interface StageChangeNotification {
|
||||
kind: 'stage_change';
|
||||
accountId: string;
|
||||
dealId: string;
|
||||
dealSide: 'demand' | 'supply';
|
||||
dealName: string;
|
||||
fromStage: string;
|
||||
toStage: string;
|
||||
changedAt: string;
|
||||
}
|
||||
|
||||
export interface IdleCapacityNotification {
|
||||
kind: 'idle_capacity';
|
||||
accountId: string;
|
||||
commitmentId: string;
|
||||
commitmentName: string;
|
||||
gpuType: string;
|
||||
idleGpuHours: number;
|
||||
idleCostCents: number;
|
||||
utilisation: number;
|
||||
observedAt: string;
|
||||
}
|
||||
|
||||
export type Notification = StageChangeNotification | IdleCapacityNotification;
|
||||
|
||||
export const NOTIFIER_PROVIDERS = ['slack', 'buzz'] as const;
|
||||
export type NotifierProvider = (typeof NOTIFIER_PROVIDERS)[number];
|
||||
|
||||
export function isNotifierProvider(value: string): value is NotifierProvider {
|
||||
return (NOTIFIER_PROVIDERS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export interface NotificationEnvelope {
|
||||
idempotencyKey: string;
|
||||
destination: string;
|
||||
workspaceId?: string | null;
|
||||
notification: Notification;
|
||||
}
|
||||
|
||||
export interface NotificationReceipt {
|
||||
externalId: string | null;
|
||||
}
|
||||
|
||||
/** Provider adapters perform one attempt; the durable worker owns retries. */
|
||||
export interface Notifier {
|
||||
readonly provider: string;
|
||||
send(envelope: NotificationEnvelope, signal?: AbortSignal): Promise<NotificationReceipt>;
|
||||
}
|
||||
|
||||
export class NotificationDeliveryError extends Error {
|
||||
constructor(
|
||||
readonly code: string,
|
||||
readonly retryable: boolean,
|
||||
readonly retryAfterMs?: number,
|
||||
) {
|
||||
super(`Notification delivery failed (${code}).`);
|
||||
this.name = 'NotificationDeliveryError';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import {
|
||||
MAX_IMPORT_CELL_CHARS,
|
||||
MAX_IMPORT_COLUMNS,
|
||||
MAX_IMPORT_ROWS,
|
||||
} from './tabular-import';
|
||||
|
||||
export const NOTION_API_VERSION = '2025-09-03';
|
||||
export const NOTION_OAUTH_TTL_MS = 10 * 60 * 1_000;
|
||||
const NOTION_API_BASE = 'https://api.notion.com/v1';
|
||||
|
||||
export interface NotionOAuthAttempt {
|
||||
state: string;
|
||||
stateHash: string;
|
||||
verifier: string;
|
||||
verifierHash: string;
|
||||
expiresAt: Date;
|
||||
}
|
||||
|
||||
export interface NotionConnectionMetadata {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
workspaceName: string | null;
|
||||
workspaceIcon: string | null;
|
||||
connectedAt: string;
|
||||
}
|
||||
|
||||
export interface NotionCredentials {
|
||||
accessToken: string;
|
||||
refreshToken?: string;
|
||||
expiresAt?: string;
|
||||
}
|
||||
|
||||
export interface NotionDataSource {
|
||||
id: string;
|
||||
databaseId: string | null;
|
||||
name: string;
|
||||
url: string | null;
|
||||
icon: string | null;
|
||||
}
|
||||
|
||||
export interface UnsupportedNotionProperty {
|
||||
name: string;
|
||||
type: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface NotionCellError {
|
||||
pageId: string;
|
||||
property: string;
|
||||
type: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface NotionMaterializedTable {
|
||||
fileName: string;
|
||||
sheetName: null;
|
||||
headers: string[];
|
||||
rows: string[][];
|
||||
warnings: string[];
|
||||
unsupportedProperties: UnsupportedNotionProperty[];
|
||||
cellErrors: NotionCellError[];
|
||||
}
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
type FetchLike = typeof fetch;
|
||||
|
||||
export class NotionApiError extends Error {
|
||||
constructor(readonly status: number, message = 'Notion rejected the request.') {
|
||||
super(message);
|
||||
this.name = 'NotionApiError';
|
||||
}
|
||||
}
|
||||
|
||||
export function hashOAuthValue(value: string): string {
|
||||
return createHash('sha256').update(value).digest('hex');
|
||||
}
|
||||
|
||||
export function createNotionOAuthAttempt(
|
||||
now = new Date(),
|
||||
random: (size: number) => Buffer = randomBytes,
|
||||
): NotionOAuthAttempt {
|
||||
const state = random(32).toString('base64url');
|
||||
const verifier = random(32).toString('base64url');
|
||||
return {
|
||||
state,
|
||||
stateHash: hashOAuthValue(state),
|
||||
verifier,
|
||||
verifierHash: hashOAuthValue(verifier),
|
||||
expiresAt: new Date(now.getTime() + NOTION_OAUTH_TTL_MS),
|
||||
};
|
||||
}
|
||||
|
||||
export function verifyNotionOAuthAttempt(
|
||||
verifier: string | undefined,
|
||||
expectedHash: string,
|
||||
expiresAt: Date,
|
||||
now = new Date(),
|
||||
): boolean {
|
||||
if (!verifier || expiresAt.getTime() <= now.getTime()) return false;
|
||||
const actual = Buffer.from(hashOAuthValue(verifier), 'hex');
|
||||
const expected = Buffer.from(expectedHash, 'hex');
|
||||
return actual.length === expected.length && timingSafeEqual(actual, expected);
|
||||
}
|
||||
|
||||
export function notionAuthorizationUrl(input: {
|
||||
clientId: string;
|
||||
redirectUri: string;
|
||||
state: string;
|
||||
}): string {
|
||||
const url = new URL('https://api.notion.com/v1/oauth/authorize');
|
||||
url.searchParams.set('client_id', input.clientId);
|
||||
url.searchParams.set('redirect_uri', input.redirectUri);
|
||||
url.searchParams.set('response_type', 'code');
|
||||
url.searchParams.set('owner', 'user');
|
||||
url.searchParams.set('state', input.state);
|
||||
// Notion does not document RFC 7636 parameters. The server-side verifier
|
||||
// provides browser binding without sending fields the provider may reject.
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export function notionConnectionMetadata(connection: {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
workspaceName: string | null;
|
||||
workspaceIcon: string | null;
|
||||
createdAt: Date;
|
||||
}): NotionConnectionMetadata {
|
||||
return {
|
||||
id: connection.id,
|
||||
workspaceId: connection.workspaceId,
|
||||
workspaceName: connection.workspaceName,
|
||||
workspaceIcon: connection.workspaceIcon,
|
||||
connectedAt: connection.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export class NotionClient {
|
||||
constructor(private readonly request: FetchLike = fetch) {}
|
||||
|
||||
async exchangeAuthorizationCode(input: {
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
code: string;
|
||||
redirectUri: string;
|
||||
}): Promise<JsonRecord> {
|
||||
return this.fetchJson('/oauth/token', undefined, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Basic ${Buffer.from(`${input.clientId}:${input.clientSecret}`).toString('base64')}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
grant_type: 'authorization_code',
|
||||
code: input.code,
|
||||
redirect_uri: input.redirectUri,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
async searchDataSources(accessToken: string): Promise<NotionDataSource[]> {
|
||||
const results = await collectNotionPages<JsonRecord>(async (cursor) => {
|
||||
const body: JsonRecord = {
|
||||
filter: { property: 'object', value: 'data_source' },
|
||||
sort: { direction: 'descending', timestamp: 'last_edited_time' },
|
||||
page_size: 100,
|
||||
};
|
||||
if (cursor) body.start_cursor = cursor;
|
||||
return this.fetchJson('/search', accessToken, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}, 2_000);
|
||||
return results.map(toDataSource).filter((value): value is NotionDataSource => value !== null);
|
||||
}
|
||||
|
||||
async retrieveDataSource(accessToken: string, dataSourceId: string): Promise<JsonRecord> {
|
||||
return this.fetchJson(`/data_sources/${encodeURIComponent(dataSourceId)}`, accessToken);
|
||||
}
|
||||
|
||||
async queryDataSource(accessToken: string, dataSourceId: string): Promise<JsonRecord[]> {
|
||||
return collectNotionPages<JsonRecord>(async (cursor) => {
|
||||
const body: JsonRecord = { page_size: 100 };
|
||||
if (cursor) body.start_cursor = cursor;
|
||||
return this.fetchJson(`/data_sources/${encodeURIComponent(dataSourceId)}/query`, accessToken, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}, MAX_IMPORT_ROWS + 1);
|
||||
}
|
||||
|
||||
async retrievePropertyItems(
|
||||
accessToken: string,
|
||||
pageId: string,
|
||||
propertyId: string,
|
||||
): Promise<JsonRecord[]> {
|
||||
return collectNotionPages<JsonRecord>(async (cursor) => {
|
||||
const query = new URLSearchParams({ page_size: '100' });
|
||||
if (cursor) query.set('start_cursor', cursor);
|
||||
return this.fetchJson(
|
||||
`/pages/${encodeURIComponent(pageId)}/properties/${encodeURIComponent(propertyId)}?${query}`,
|
||||
accessToken,
|
||||
);
|
||||
}, 10_000);
|
||||
}
|
||||
|
||||
private async fetchJson(
|
||||
path: string,
|
||||
accessToken?: string,
|
||||
init: RequestInit = {},
|
||||
): Promise<JsonRecord> {
|
||||
const headers = new Headers(init.headers);
|
||||
headers.set('Accept', 'application/json');
|
||||
headers.set('Notion-Version', NOTION_API_VERSION);
|
||||
if (accessToken) headers.set('Authorization', `Bearer ${accessToken}`);
|
||||
if (init.body && !headers.has('Content-Type')) headers.set('Content-Type', 'application/json');
|
||||
const response = await this.request(`${NOTION_API_BASE}${path}`, { ...init, headers });
|
||||
if (!response.ok) {
|
||||
const message = response.status === 401
|
||||
? 'The Notion connection is no longer authorized. Reconnect it and try again.'
|
||||
: `Notion request failed with status ${response.status}.`;
|
||||
throw new NotionApiError(response.status, message);
|
||||
}
|
||||
const value: unknown = await response.json();
|
||||
if (!isRecord(value)) throw new NotionApiError(502, 'Notion returned an invalid response.');
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
export async function collectNotionPages<T extends JsonRecord>(
|
||||
fetchPage: (cursor?: string) => Promise<JsonRecord>,
|
||||
maxItems: number,
|
||||
): Promise<T[]> {
|
||||
const collected: T[] = [];
|
||||
let cursor: string | undefined;
|
||||
for (let page = 0; page < 100; page += 1) {
|
||||
const response = await fetchPage(cursor);
|
||||
const results = Array.isArray(response.results)
|
||||
? response.results.filter(isRecord) as T[]
|
||||
: [];
|
||||
collected.push(...results);
|
||||
if (collected.length >= maxItems) return collected.slice(0, maxItems);
|
||||
if (response.has_more !== true || typeof response.next_cursor !== 'string') return collected;
|
||||
cursor = response.next_cursor;
|
||||
}
|
||||
throw new NotionApiError(422, 'Notion pagination exceeded the safety limit.');
|
||||
}
|
||||
|
||||
export async function materializeNotionDataSource(
|
||||
client: NotionClient,
|
||||
accessToken: string,
|
||||
dataSourceId: string,
|
||||
): Promise<NotionMaterializedTable> {
|
||||
const [source, pages] = await Promise.all([
|
||||
client.retrieveDataSource(accessToken, dataSourceId),
|
||||
client.queryDataSource(accessToken, dataSourceId),
|
||||
]);
|
||||
if (pages.length > MAX_IMPORT_ROWS) {
|
||||
throw new NotionApiError(422, `Notion imports may not exceed ${MAX_IMPORT_ROWS} rows.`);
|
||||
}
|
||||
|
||||
const schema = isRecord(source.properties) ? source.properties : {};
|
||||
const unsupportedProperties: UnsupportedNotionProperty[] = [];
|
||||
const supportedHeaders: string[] = [];
|
||||
for (const [name, definition] of Object.entries(schema)) {
|
||||
const type = isRecord(definition) && typeof definition.type === 'string'
|
||||
? definition.type
|
||||
: 'unknown';
|
||||
const reason = unsupportedReason(type);
|
||||
if (reason) unsupportedProperties.push({ name, type, reason });
|
||||
else if (supportedHeaders.length < MAX_IMPORT_COLUMNS - 2) supportedHeaders.push(name);
|
||||
else unsupportedProperties.push({
|
||||
name,
|
||||
type,
|
||||
reason: `A15 is limited to ${MAX_IMPORT_COLUMNS} columns so A14 can validate the plan safely.`,
|
||||
});
|
||||
}
|
||||
|
||||
const cellErrors: NotionCellError[] = [];
|
||||
const rows: string[][] = [];
|
||||
for (const page of pages) {
|
||||
const pageId = typeof page.id === 'string' ? page.id : '';
|
||||
const properties = isRecord(page.properties) ? page.properties : {};
|
||||
const row = [pageId, typeof page.url === 'string' ? page.url : ''];
|
||||
for (const header of supportedHeaders) {
|
||||
const property = properties[header];
|
||||
if (!isRecord(property)) {
|
||||
row.push('');
|
||||
cellErrors.push({ pageId, property: header, type: 'unknown', message: 'The page omitted this property.' });
|
||||
continue;
|
||||
}
|
||||
const completed = await completePaginatedProperty(client, accessToken, pageId, property);
|
||||
const flattened = flattenNotionProperty(completed);
|
||||
if (flattened.error) {
|
||||
cellErrors.push({
|
||||
pageId,
|
||||
property: header,
|
||||
type: typeof property.type === 'string' ? property.type : 'unknown',
|
||||
message: flattened.error,
|
||||
});
|
||||
row.push('');
|
||||
} else if ((flattened.value ?? '').length > MAX_IMPORT_CELL_CHARS) {
|
||||
cellErrors.push({
|
||||
pageId,
|
||||
property: header,
|
||||
type: typeof property.type === 'string' ? property.type : 'unknown',
|
||||
message: `The flattened value exceeds ${MAX_IMPORT_CELL_CHARS} characters.`,
|
||||
});
|
||||
row.push('');
|
||||
} else {
|
||||
row.push(flattened.value ?? '');
|
||||
}
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
const warnings = [
|
||||
...unsupportedProperties.map(({ name, type }) => `${name} (${type}) was not imported.`),
|
||||
...(cellErrors.length > 0 ? [`${cellErrors.length} Notion cell values could not be flattened and were left blank.`] : []),
|
||||
];
|
||||
return {
|
||||
fileName: `Notion - ${notionTitle(source)}`,
|
||||
sheetName: null,
|
||||
headers: ['Notion page ID', 'Notion page URL', ...supportedHeaders],
|
||||
rows,
|
||||
warnings,
|
||||
unsupportedProperties,
|
||||
cellErrors,
|
||||
};
|
||||
}
|
||||
|
||||
export function flattenNotionProperty(property: JsonRecord): { value?: string; error?: string } {
|
||||
const type = typeof property.type === 'string' ? property.type : 'unknown';
|
||||
const value = property[type];
|
||||
if (['title', 'rich_text'].includes(type)) return { value: plainText(value) };
|
||||
if (type === 'number') return { value: value == null ? '' : String(value) };
|
||||
if (type === 'checkbox') return typeof value === 'boolean'
|
||||
? { value: value ? 'true' : 'false' }
|
||||
: { error: 'Notion returned a non-boolean checkbox.' };
|
||||
if (['url', 'email', 'phone_number', 'created_time', 'last_edited_time'].includes(type)) {
|
||||
return { value: value == null ? '' : String(value) };
|
||||
}
|
||||
if (['select', 'status'].includes(type)) {
|
||||
return { value: isRecord(value) && typeof value.name === 'string' ? value.name : '' };
|
||||
}
|
||||
if (type === 'multi_select') return { value: names(value).join(', ') };
|
||||
if (type === 'date') {
|
||||
if (!isRecord(value)) return { value: '' };
|
||||
const start = typeof value.start === 'string' ? value.start : '';
|
||||
const end = typeof value.end === 'string' ? value.end : '';
|
||||
return { value: end ? `${start}/${end}` : start };
|
||||
}
|
||||
if (type === 'people') return { value: people(value).join(', ') };
|
||||
if (type === 'files') return { value: fileUrls(value).join(', ') };
|
||||
if (type === 'relation') return { value: ids(value).join(', ') };
|
||||
if (['created_by', 'last_edited_by'].includes(type)) {
|
||||
if (!isRecord(value)) return { value: '' };
|
||||
return { value: typeof value.name === 'string' ? value.name : typeof value.id === 'string' ? value.id : '' };
|
||||
}
|
||||
if (type === 'unique_id') {
|
||||
if (!isRecord(value) || typeof value.number !== 'number') return { value: '' };
|
||||
return { value: `${typeof value.prefix === 'string' ? value.prefix : ''}${value.number}` };
|
||||
}
|
||||
if (type === 'formula' || type === 'rollup') {
|
||||
if (!isRecord(value) || typeof value.type !== 'string') return { error: `Notion returned an incomplete ${type}.` };
|
||||
if (value.type === 'array') {
|
||||
if (!Array.isArray(value.array)) return { error: `Notion returned an invalid ${type} array.` };
|
||||
const flattened = value.array.filter(isRecord).map(flattenNotionProperty);
|
||||
const error = flattened.find((item) => item.error)?.error;
|
||||
return error ? { error } : { value: flattened.map((item) => item.value ?? '').filter(Boolean).join(', ') };
|
||||
}
|
||||
return flattenNotionProperty({ type: value.type, [value.type]: value[value.type] });
|
||||
}
|
||||
return { error: unsupportedReason(type) ?? `Unsupported Notion property type: ${type}.` };
|
||||
}
|
||||
|
||||
async function completePaginatedProperty(
|
||||
client: NotionClient,
|
||||
accessToken: string,
|
||||
pageId: string,
|
||||
property: JsonRecord,
|
||||
): Promise<JsonRecord> {
|
||||
const type = typeof property.type === 'string' ? property.type : '';
|
||||
const value = property[type];
|
||||
const needsCompletion = type === 'relation'
|
||||
? property.has_more === true
|
||||
: ['title', 'rich_text', 'people'].includes(type) && Array.isArray(value) && value.length >= 25;
|
||||
if (!needsCompletion || typeof property.id !== 'string' || !pageId) return property;
|
||||
const items = await client.retrievePropertyItems(accessToken, pageId, property.id);
|
||||
const completedValues = items.map((item) => item[type]).filter((item) => item != null);
|
||||
return { ...property, [type]: completedValues, has_more: false };
|
||||
}
|
||||
|
||||
function unsupportedReason(type: string): string | null {
|
||||
if (['button', 'verification', 'place'].includes(type)) {
|
||||
return `Notion ${type} values do not have a stable tabular representation.`;
|
||||
}
|
||||
const supported = [
|
||||
'title', 'rich_text', 'number', 'checkbox', 'url', 'email', 'phone_number',
|
||||
'created_time', 'last_edited_time', 'select', 'status', 'multi_select', 'date',
|
||||
'people', 'files', 'relation', 'created_by', 'last_edited_by', 'unique_id',
|
||||
'formula', 'rollup',
|
||||
];
|
||||
return supported.includes(type) ? null : `Notion property type ${type} is not supported.`;
|
||||
}
|
||||
|
||||
function toDataSource(value: JsonRecord): NotionDataSource | null {
|
||||
if (typeof value.id !== 'string') return null;
|
||||
const parent = isRecord(value.parent) ? value.parent : {};
|
||||
return {
|
||||
id: value.id,
|
||||
databaseId: typeof parent.database_id === 'string' ? parent.database_id : null,
|
||||
name: notionTitle(value),
|
||||
url: typeof value.url === 'string' ? value.url : null,
|
||||
icon: isRecord(value.icon) && value.icon.type === 'emoji' && typeof value.icon.emoji === 'string'
|
||||
? value.icon.emoji
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function notionTitle(value: JsonRecord): string {
|
||||
const title = plainText(value.title);
|
||||
return title || (typeof value.name === 'string' ? value.name : 'Untitled data source');
|
||||
}
|
||||
|
||||
function plainText(value: unknown): string {
|
||||
if (!Array.isArray(value)) return '';
|
||||
return value.filter(isRecord).map((item) => {
|
||||
if (typeof item.plain_text === 'string') return item.plain_text;
|
||||
const nested = isRecord(item.text) ? item.text : null;
|
||||
return nested && typeof nested.content === 'string' ? nested.content : '';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function names(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value.filter(isRecord).flatMap((item) => typeof item.name === 'string' ? [item.name] : [])
|
||||
: [];
|
||||
}
|
||||
|
||||
function people(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter(isRecord).map((person) => {
|
||||
if (typeof person.name === 'string') return person.name;
|
||||
return typeof person.id === 'string' ? person.id : '';
|
||||
}).filter(Boolean) : [];
|
||||
}
|
||||
|
||||
function fileUrls(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter(isRecord).flatMap((item) => {
|
||||
const file = isRecord(item.file) ? item.file : isRecord(item.external) ? item.external : null;
|
||||
return file && typeof file.url === 'string' ? [file.url] : [];
|
||||
}) : [];
|
||||
}
|
||||
|
||||
function ids(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value.filter(isRecord).flatMap((item) => typeof item.id === 'string' ? [item.id] : [])
|
||||
: [];
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is JsonRecord {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
NotificationDeliveryError,
|
||||
type Notification,
|
||||
type NotificationEnvelope,
|
||||
type NotificationReceipt,
|
||||
type Notifier,
|
||||
} from './notifier';
|
||||
|
||||
interface SlackNotifierOptions {
|
||||
botToken: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
apiBase?: string;
|
||||
}
|
||||
|
||||
interface SlackResponse {
|
||||
ok?: boolean;
|
||||
error?: string;
|
||||
ts?: string;
|
||||
}
|
||||
|
||||
export class SlackNotifier implements Notifier {
|
||||
readonly provider = 'slack';
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly apiBase: string;
|
||||
|
||||
constructor(private readonly options: SlackNotifierOptions) {
|
||||
this.fetchImpl = options.fetchImpl ?? fetch;
|
||||
this.apiBase = options.apiBase ?? 'https://slack.com/api';
|
||||
}
|
||||
|
||||
async send(
|
||||
envelope: NotificationEnvelope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NotificationReceipt> {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await this.fetchImpl(`${this.apiBase}/chat.postMessage`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${this.options.botToken}`,
|
||||
'content-type': 'application/json; charset=utf-8',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel: envelope.destination,
|
||||
text: formatSlackNotification(envelope.notification),
|
||||
client_msg_id: slackClientMessageId(envelope.idempotencyKey),
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
} catch {
|
||||
throw new NotificationDeliveryError('network_error', true);
|
||||
}
|
||||
|
||||
if (response.status === 429) {
|
||||
const retrySeconds = Number(response.headers.get('retry-after'));
|
||||
throw new NotificationDeliveryError(
|
||||
'rate_limited',
|
||||
true,
|
||||
Number.isFinite(retrySeconds) && retrySeconds > 0 ? retrySeconds * 1_000 : undefined,
|
||||
);
|
||||
}
|
||||
if (response.status >= 500) {
|
||||
throw new NotificationDeliveryError('slack_unavailable', true);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new NotificationDeliveryError('slack_http_error', false);
|
||||
}
|
||||
|
||||
let result: SlackResponse;
|
||||
try {
|
||||
result = (await response.json()) as SlackResponse;
|
||||
} catch {
|
||||
throw new NotificationDeliveryError('invalid_slack_response', true);
|
||||
}
|
||||
if (!result.ok) {
|
||||
const code = normaliseSlackError(result.error);
|
||||
throw new NotificationDeliveryError(code, isRetryableSlackError(code));
|
||||
}
|
||||
return { externalId: result.ts ?? null };
|
||||
}
|
||||
}
|
||||
|
||||
export function slackClientMessageId(idempotencyKey: string): string {
|
||||
const digest = createHash('sha256').update(idempotencyKey).digest('hex');
|
||||
return `${digest.slice(0, 8)}-${digest.slice(8, 12)}-4${digest.slice(13, 16)}-a${digest.slice(17, 20)}-${digest.slice(20, 32)}`;
|
||||
}
|
||||
|
||||
function formatSlackNotification(notification: Notification): string {
|
||||
if (notification.kind === 'stage_change') {
|
||||
return [
|
||||
`*${notification.dealName}* moved from \`${notification.fromStage}\` to \`${notification.toStage}\`.`,
|
||||
`${notification.dealSide === 'demand' ? 'Demand' : 'Supply'} pipeline stage changed in PIG.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
return [
|
||||
`*Idle capacity: ${notification.commitmentName}*`,
|
||||
`${notification.gpuType} has ${formatNumber(notification.idleGpuHours)} unsold GPU-hours (${Math.round(notification.utilisation * 100)}% utilised).`,
|
||||
`Idle committed cost: ${formatMoney(notification.idleCostCents)}.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function formatNumber(value: number): string {
|
||||
return new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 }).format(value);
|
||||
}
|
||||
|
||||
function formatMoney(cents: number): string {
|
||||
return new Intl.NumberFormat('en-US', {
|
||||
style: 'currency',
|
||||
currency: 'USD',
|
||||
maximumFractionDigits: 0,
|
||||
}).format(cents / 100);
|
||||
}
|
||||
|
||||
function normaliseSlackError(error: string | undefined): string {
|
||||
return error && /^[a-z0-9_]+$/i.test(error) ? error : 'unknown_slack_error';
|
||||
}
|
||||
|
||||
function isRetryableSlackError(code: string): boolean {
|
||||
return ['ratelimited', 'internal_error', 'fatal_error', 'request_timeout'].includes(code);
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
import { inflateRawSync } from 'node:zlib';
|
||||
|
||||
export const MAX_IMPORT_FILE_BYTES = 5 * 1024 * 1024;
|
||||
export const MAX_IMPORT_ROWS = 2_000;
|
||||
export const MAX_IMPORT_COLUMNS = 100;
|
||||
export const MAX_IMPORT_CELL_CHARS = 10_000;
|
||||
const MAX_XLSX_ENTRIES = 256;
|
||||
const MAX_XLSX_UNCOMPRESSED_BYTES = 20 * 1024 * 1024;
|
||||
|
||||
export interface ParsedTable {
|
||||
fileName: string;
|
||||
sheetName: string | null;
|
||||
headers: string[];
|
||||
rows: string[][];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export function parseTabularFile(input: {
|
||||
fileName: string;
|
||||
mimeType?: string;
|
||||
bytes: Uint8Array;
|
||||
}): ParsedTable {
|
||||
if (input.bytes.byteLength === 0) throw new Error('The selected file is empty.');
|
||||
if (input.bytes.byteLength > MAX_IMPORT_FILE_BYTES) {
|
||||
throw new Error('Import files may not exceed 5 MB.');
|
||||
}
|
||||
const fileName = input.fileName.trim().slice(0, 255);
|
||||
const lowerName = fileName.toLowerCase();
|
||||
if (lowerName.endsWith('.csv') || input.mimeType === 'text/csv') {
|
||||
const text = new TextDecoder('utf-8', { fatal: true }).decode(input.bytes);
|
||||
const { headers, rows, warnings } = parseCsv(text);
|
||||
return { fileName, sheetName: null, headers, rows, warnings };
|
||||
}
|
||||
if (
|
||||
lowerName.endsWith('.xlsx') ||
|
||||
input.mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||||
) {
|
||||
const { sheetName, headers, rows, warnings } = parseXlsx(input.bytes);
|
||||
return { fileName, sheetName, headers, rows, warnings };
|
||||
}
|
||||
throw new Error('Use a UTF-8 CSV or .xlsx workbook. Legacy .xls files are not supported.');
|
||||
}
|
||||
|
||||
export function parseCsv(source: string): Omit<ParsedTable, 'fileName' | 'sheetName'> {
|
||||
if (source.includes('\0')) throw new Error('The CSV contains invalid null bytes.');
|
||||
const table: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
let quoted = false;
|
||||
|
||||
const pushField = () => {
|
||||
if (field.length > MAX_IMPORT_CELL_CHARS) throw new Error('A cell exceeds 10,000 characters.');
|
||||
row.push(field);
|
||||
field = '';
|
||||
if (row.length > MAX_IMPORT_COLUMNS) throw new Error('Imports may not exceed 100 columns.');
|
||||
};
|
||||
const pushRow = () => {
|
||||
pushField();
|
||||
table.push(row);
|
||||
row = [];
|
||||
if (table.length > MAX_IMPORT_ROWS + 1) throw new Error('Imports may not exceed 2,000 data rows.');
|
||||
};
|
||||
|
||||
const text = source.charCodeAt(0) === 0xfeff ? source.slice(1) : source;
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const character = text[index]!;
|
||||
if (quoted) {
|
||||
if (character === '"') {
|
||||
if (text[index + 1] === '"') {
|
||||
field += '"';
|
||||
index += 1;
|
||||
} else {
|
||||
quoted = false;
|
||||
}
|
||||
} else {
|
||||
field += character;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (character === '"' && field.length === 0) quoted = true;
|
||||
else if (character === ',') pushField();
|
||||
else if (character === '\n') pushRow();
|
||||
else if (character === '\r' && text[index + 1] === '\n') continue;
|
||||
else if (character === '\r') pushRow();
|
||||
else field += character;
|
||||
}
|
||||
if (quoted) throw new Error('The CSV ends inside a quoted cell.');
|
||||
if (field.length > 0 || row.length > 0) pushRow();
|
||||
|
||||
return normaliseTabularRows(table, []);
|
||||
}
|
||||
|
||||
function parseXlsx(bytes: Uint8Array): {
|
||||
sheetName: string;
|
||||
headers: string[];
|
||||
rows: string[][];
|
||||
warnings: string[];
|
||||
} {
|
||||
const entries = readZip(bytes);
|
||||
const workbook = readXml(entries, 'xl/workbook.xml');
|
||||
const relationships = readXml(entries, 'xl/_rels/workbook.xml.rels');
|
||||
rejectActiveXml(workbook);
|
||||
rejectActiveXml(relationships);
|
||||
|
||||
const sheets = [...workbook.matchAll(/<sheet\b[^>]*>/gi)]
|
||||
.map((match) => ({
|
||||
name: xmlAttribute(match[0], 'name'),
|
||||
relationshipId: xmlAttribute(match[0], 'r:id'),
|
||||
hidden: ['hidden', 'veryHidden'].includes(xmlAttribute(match[0], 'state') ?? ''),
|
||||
}))
|
||||
.filter((sheet) => sheet.name && sheet.relationshipId && !sheet.hidden);
|
||||
const firstSheet = sheets[0];
|
||||
if (!firstSheet?.name || !firstSheet.relationshipId) {
|
||||
throw new Error('The workbook has no visible worksheet.');
|
||||
}
|
||||
const relationship = [...relationships.matchAll(/<Relationship\b[^>]*>/gi)]
|
||||
.map((match) => match[0])
|
||||
.find((tag) => xmlAttribute(tag, 'Id') === firstSheet.relationshipId);
|
||||
const target = relationship ? xmlAttribute(relationship, 'Target') : null;
|
||||
if (!target) throw new Error('The workbook worksheet relationship is invalid.');
|
||||
const sheetPath = normaliseZipPath(target.startsWith('/') ? target.slice(1) : `xl/${target}`);
|
||||
const sharedStrings = entries.has('xl/sharedStrings.xml')
|
||||
? parseSharedStrings(readXml(entries, 'xl/sharedStrings.xml'))
|
||||
: [];
|
||||
const worksheet = readXml(entries, sheetPath);
|
||||
const parsed = parseWorksheetXml(worksheet, sharedStrings);
|
||||
return { sheetName: decodeXml(firstSheet.name), ...parsed };
|
||||
}
|
||||
|
||||
export function parseWorksheetXml(
|
||||
worksheet: string,
|
||||
sharedStrings: readonly string[] = [],
|
||||
): Omit<ParsedTable, 'fileName' | 'sheetName'> {
|
||||
rejectActiveXml(worksheet);
|
||||
const table: string[][] = [];
|
||||
const warnings: string[] = [];
|
||||
let sawFormula = false;
|
||||
for (const rowMatch of worksheet.matchAll(/<row\b[^>]*>([\s\S]*?)<\/row>/gi)) {
|
||||
const row: string[] = [];
|
||||
let sequentialColumn = 0;
|
||||
for (const cellMatch of rowMatch[1]!.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/gi)) {
|
||||
const attributes = cellMatch[1]!;
|
||||
const body = cellMatch[2]!;
|
||||
const reference = xmlAttribute(attributes, 'r');
|
||||
const column = reference ? columnFromReference(reference) : sequentialColumn;
|
||||
if (column >= MAX_IMPORT_COLUMNS) throw new Error('Imports may not exceed 100 columns.');
|
||||
const type = xmlAttribute(attributes, 't') ?? 'n';
|
||||
const rawValue = body.match(/<v\b[^>]*>([\s\S]*?)<\/v>/i)?.[1] ?? '';
|
||||
if (/<f\b/i.test(body)) sawFormula = true;
|
||||
let value: string;
|
||||
if (type === 's') value = sharedStrings[Number(rawValue)] ?? '';
|
||||
else if (type === 'inlineStr') value = extractTextNodes(body);
|
||||
else if (type === 'b') value = rawValue === '1' ? 'true' : 'false';
|
||||
else value = decodeXml(rawValue);
|
||||
if (value.length > MAX_IMPORT_CELL_CHARS) throw new Error('A cell exceeds 10,000 characters.');
|
||||
row[column] = value;
|
||||
sequentialColumn = column + 1;
|
||||
}
|
||||
if (row.some((value) => value !== undefined && value !== '')) table.push(row);
|
||||
if (table.length > MAX_IMPORT_ROWS + 1) throw new Error('Imports may not exceed 2,000 data rows.');
|
||||
}
|
||||
if (sawFormula) {
|
||||
warnings.push('Formula cells were not executed; only cached values stored in the workbook were read.');
|
||||
}
|
||||
return normaliseTabularRows(table, warnings);
|
||||
}
|
||||
|
||||
export function normaliseTabularRows(
|
||||
table: string[][],
|
||||
warnings: string[],
|
||||
): Omit<ParsedTable, 'fileName' | 'sheetName'> {
|
||||
while (table.length > 0 && table.at(-1)!.every((cell) => !cell?.trim())) table.pop();
|
||||
const headerRow = table.shift();
|
||||
if (!headerRow) throw new Error('The file has no header row.');
|
||||
let width = headerRow.length;
|
||||
while (width > 0 && !headerRow[width - 1]?.trim()) width -= 1;
|
||||
if (width === 0) throw new Error('The file has no named columns.');
|
||||
const headers = headerRow.slice(0, width).map((header) => header.trim());
|
||||
if (headers.some((header) => !header)) throw new Error('Every imported column needs a header.');
|
||||
const normalised = headers.map((header) => header.toLocaleLowerCase());
|
||||
if (new Set(normalised).size !== normalised.length) {
|
||||
throw new Error('Column headers must be unique, ignoring letter case.');
|
||||
}
|
||||
const rows = table
|
||||
.map((sourceRow) => Array.from({ length: width }, (_, index) => sourceRow[index] ?? ''))
|
||||
.filter((sourceRow) => sourceRow.some((cell) => cell.trim() !== ''));
|
||||
if (rows.length === 0) throw new Error('The file has headers but no data rows.');
|
||||
if (rows.length > MAX_IMPORT_ROWS) throw new Error('Imports may not exceed 2,000 data rows.');
|
||||
if (rows.some((sourceRow) => sourceRow.some((cell) => /^[=+@]/.test(cell.trim())))) {
|
||||
warnings.push('Formula-like text from the source remains inert text and is never executed.');
|
||||
}
|
||||
return { headers, rows, warnings };
|
||||
}
|
||||
|
||||
function readZip(bytes: Uint8Array): Map<string, Uint8Array> {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
let end = -1;
|
||||
for (let offset = bytes.byteLength - 22; offset >= Math.max(0, bytes.byteLength - 65_557); offset -= 1) {
|
||||
if (view.getUint32(offset, true) === 0x06054b50) {
|
||||
end = offset;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (end < 0) throw new Error('The .xlsx ZIP directory is invalid.');
|
||||
const count = view.getUint16(end + 10, true);
|
||||
const centralOffset = view.getUint32(end + 16, true);
|
||||
if (count > MAX_XLSX_ENTRIES || count === 0xffff || centralOffset === 0xffffffff) {
|
||||
throw new Error('The workbook archive is too large or uses unsupported ZIP64 metadata.');
|
||||
}
|
||||
const entries = new Map<string, Uint8Array>();
|
||||
let offset = centralOffset;
|
||||
let totalUncompressed = 0;
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
if (offset + 46 > bytes.byteLength || view.getUint32(offset, true) !== 0x02014b50) {
|
||||
throw new Error('The .xlsx ZIP directory is corrupt.');
|
||||
}
|
||||
const flags = view.getUint16(offset + 8, true);
|
||||
const method = view.getUint16(offset + 10, true);
|
||||
const compressedSize = view.getUint32(offset + 20, true);
|
||||
const uncompressedSize = view.getUint32(offset + 24, true);
|
||||
const nameLength = view.getUint16(offset + 28, true);
|
||||
const extraLength = view.getUint16(offset + 30, true);
|
||||
const commentLength = view.getUint16(offset + 32, true);
|
||||
const localOffset = view.getUint32(offset + 42, true);
|
||||
const name = new TextDecoder().decode(bytes.subarray(offset + 46, offset + 46 + nameLength));
|
||||
const safeName = normaliseZipPath(name);
|
||||
if ((flags & 1) !== 0) throw new Error('Encrypted workbooks are not supported.');
|
||||
if (method !== 0 && method !== 8) throw new Error('The workbook uses unsupported ZIP compression.');
|
||||
totalUncompressed += uncompressedSize;
|
||||
if (totalUncompressed > MAX_XLSX_UNCOMPRESSED_BYTES) {
|
||||
throw new Error('The expanded workbook may not exceed 20 MB.');
|
||||
}
|
||||
if (localOffset + 30 > bytes.byteLength || view.getUint32(localOffset, true) !== 0x04034b50) {
|
||||
throw new Error('The workbook contains an invalid ZIP entry.');
|
||||
}
|
||||
const localNameLength = view.getUint16(localOffset + 26, true);
|
||||
const localExtraLength = view.getUint16(localOffset + 28, true);
|
||||
const dataOffset = localOffset + 30 + localNameLength + localExtraLength;
|
||||
const compressed = bytes.subarray(dataOffset, dataOffset + compressedSize);
|
||||
if (compressed.byteLength !== compressedSize) throw new Error('The workbook ZIP entry is truncated.');
|
||||
const output = method === 0
|
||||
? Uint8Array.from(compressed)
|
||||
: inflateRawSync(compressed, { maxOutputLength: MAX_XLSX_UNCOMPRESSED_BYTES });
|
||||
if (output.byteLength !== uncompressedSize) throw new Error('The workbook ZIP entry size is inconsistent.');
|
||||
entries.set(safeName, output);
|
||||
offset += 46 + nameLength + extraLength + commentLength;
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function readXml(entries: Map<string, Uint8Array>, name: string): string {
|
||||
const bytes = entries.get(name);
|
||||
if (!bytes) throw new Error(`The workbook is missing ${name}.`);
|
||||
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
}
|
||||
|
||||
function rejectActiveXml(xml: string): void {
|
||||
if (/<!DOCTYPE|<!ENTITY/i.test(xml)) throw new Error('Workbook XML declarations are not allowed.');
|
||||
}
|
||||
|
||||
function normaliseZipPath(path: string): string {
|
||||
const segments: string[] = [];
|
||||
for (const segment of path.replace(/\\/g, '/').split('/')) {
|
||||
if (!segment || segment === '.') continue;
|
||||
if (segment === '..') {
|
||||
if (segments.length === 0) throw new Error('The workbook contains an unsafe ZIP path.');
|
||||
segments.pop();
|
||||
} else {
|
||||
segments.push(segment);
|
||||
}
|
||||
}
|
||||
return segments.join('/');
|
||||
}
|
||||
|
||||
function xmlAttribute(tag: string, name: string): string | null {
|
||||
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const match = tag.match(new RegExp(`${escaped}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, 'i'));
|
||||
return match ? decodeXml(match[1] ?? match[2] ?? '') : null;
|
||||
}
|
||||
|
||||
function parseSharedStrings(xml: string): string[] {
|
||||
rejectActiveXml(xml);
|
||||
return [...xml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/gi)].map((match) =>
|
||||
extractTextNodes(match[1]!),
|
||||
);
|
||||
}
|
||||
|
||||
function extractTextNodes(xml: string): string {
|
||||
return [...xml.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/gi)]
|
||||
.map((match) => decodeXml(match[1]!))
|
||||
.join('');
|
||||
}
|
||||
|
||||
function decodeXml(value: string): string {
|
||||
return value.replace(/&(?:#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (entity) => {
|
||||
if (entity === '&') return '&';
|
||||
if (entity === '<') return '<';
|
||||
if (entity === '>') return '>';
|
||||
if (entity === '"') return '"';
|
||||
if (entity === ''') return "'";
|
||||
const numeric = entity.startsWith('&#x')
|
||||
? Number.parseInt(entity.slice(3, -1), 16)
|
||||
: Number.parseInt(entity.slice(2, -1), 10);
|
||||
return Number.isFinite(numeric) ? String.fromCodePoint(numeric) : entity;
|
||||
});
|
||||
}
|
||||
|
||||
function columnFromReference(reference: string): number {
|
||||
const letters = reference.match(/^[A-Za-z]+/)?.[0];
|
||||
if (!letters) throw new Error('The worksheet contains an invalid cell reference.');
|
||||
let column = 0;
|
||||
for (const letter of letters.toUpperCase()) column = column * 26 + letter.charCodeAt(0) - 64;
|
||||
return column - 1;
|
||||
}
|
||||
Reference in New Issue
Block a user