Build the agent-native compute CRM platform

2026-08-13 01:39:01 -07:00
parent bfd2f8d95a
commit 853bde2265
160 changed files with 61812 additions and 483 deletions
@@ -0,0 +1,551 @@
+import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
+import type { Database, GoogleConnection, GoogleOauthFlow } from '@pig/db';
+import { activities, googleConnections, googleOauthFlows } from '@pig/db';
+import { and, eq, gt, isNull } from 'drizzle-orm';
+import type { Principal } from '../lib/auth';
+import { decryptSecret, encryptionReady, encryptSecret } from '../lib/secrets';
+import { MutationError } from '../lib/mutation';
+import {
+  MAX_IMPORT_CELL_CHARS,
+  MAX_IMPORT_COLUMNS,
+  MAX_IMPORT_ROWS,
+  normaliseTabularRows,
+  type ParsedTable,
+} from './tabular-import';
+
+export const GOOGLE_OAUTH_SCOPES = [
+  'https://www.googleapis.com/auth/drive.metadata.readonly',
+  'https://www.googleapis.com/auth/spreadsheets.readonly',
+] as const;
+const GOOGLE_AUTHORIZATION_ENDPOINT = 'https://accounts.google.com/o/oauth2/v2/auth';
+const GOOGLE_TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token';
+const GOOGLE_REVOCATION_ENDPOINT = 'https://oauth2.googleapis.com/revoke';
+const GOOGLE_DRIVE_FILES_ENDPOINT = 'https://www.googleapis.com/drive/v3/files';
+const GOOGLE_SHEETS_ENDPOINT = 'https://sheets.googleapis.com/v4/spreadsheets';
+const OAUTH_FLOW_TTL_MS = 10 * 60 * 1_000;
+const ACCESS_TOKEN_SKEW_MS = 60 * 1_000;
+const DRIVE_PAGE_SIZE = 50;
+const PKCE_PURPOSE = 'google-oauth:pkce-verifier';
+const REFRESH_TOKEN_PURPOSE = 'google-oauth:refresh-token';
+const ACCESS_TOKEN_PURPOSE = 'google-oauth:access-token';
+
+export interface GoogleSheetsConfig {
+  clientId?: string;
+  clientSecret?: string;
+  redirectUri?: string;
+  encryptionKey?: string;
+  publicUrl: string;
+}
+
+export interface GoogleConnectionMetadata {
+  configured: boolean;
+  connected: boolean;
+  connectedAt: string | null;
+  scopes: string[];
+}
+
+export interface GoogleDriveFile {
+  id: string;
+  name: string;
+  modifiedTime: string | null;
+}
+
+export interface GoogleSheetMetadata {
+  sheetId: number;
+  title: string;
+  rowCount: number;
+  columnCount: number;
+}
+
+interface GoogleTokenResponse {
+  access_token?: string;
+  expires_in?: number;
+  refresh_token?: string;
+  scope?: string;
+  token_type?: string;
+  error?: string;
+}
+
+interface DriveListResponse {
+  files?: { id?: string; name?: string; modifiedTime?: string }[];
+  nextPageToken?: string;
+  incompleteSearch?: boolean;
+}
+
+interface SpreadsheetMetadataResponse {
+  properties?: { title?: string };
+  sheets?: {
+    properties?: {
+      sheetId?: number;
+      title?: string;
+      sheetType?: string;
+      hidden?: boolean;
+      gridProperties?: { rowCount?: number; columnCount?: number };
+    };
+  }[];
+}
+
+interface ValuesResponse {
+  values?: unknown[][];
+}
+
+export class GoogleApiError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+    readonly reconnectRequired = false,
+  ) {
+    super(message);
+    this.name = 'GoogleApiError';
+  }
+}
+
+export class GoogleSheetsService {
+  constructor(
+    private readonly db: Database,
+    private readonly config: GoogleSheetsConfig,
+    private readonly fetchImpl: typeof fetch = fetch,
+  ) {}
+
+  configured(): boolean {
+    return Boolean(
+      this.config.clientId &&
+      this.config.clientSecret &&
+      this.config.redirectUri &&
+      encryptionReady(this.config.encryptionKey),
+    );
+  }
+
+  async connectionMetadata(userId: string): Promise<GoogleConnectionMetadata> {
+    const [connection] = await this.db
+      .select()
+      .from(googleConnections)
+      .where(eq(googleConnections.userId, userId))
+      .limit(1);
+    return googleConnectionMetadata(this.configured(), connection);
+  }
+
+  async beginOAuth(principal: Principal, now = new Date()): Promise<{ authorizationUrl: string; browserBinding: string }> {
+    this.requireConfigured();
+    const state = randomBytes(32).toString('base64url');
+    const browserBinding = randomBytes(32).toString('base64url');
+    const verifier = randomBytes(32).toString('base64url');
+    const challenge = createHash('sha256').update(verifier).digest('base64url');
+    await this.db.insert(googleOauthFlows).values({
+      userId: principal.userId,
+      stateHash: oauthStateHash(state),
+      browserBindingHash: oauthStateHash(browserBinding),
+      pkceVerifierEncrypted: encryptSecret(verifier, this.config.encryptionKey, PKCE_PURPOSE),
+      expiresAt: new Date(now.getTime() + OAUTH_FLOW_TTL_MS),
+      createdAt: now,
+    });
+    return {
+      authorizationUrl: buildGoogleAuthorizationUrl({
+        clientId: this.config.clientId!,
+        redirectUri: this.config.redirectUri!,
+        state,
+        challenge,
+      }),
+      browserBinding,
+    };
+  }
+
+  async completeOAuth(input: { state: string; code: string; browserBinding: string }, now = new Date()): Promise<void> {
+    this.requireConfigured();
+    const [flow] = await this.db
+      .update(googleOauthFlows)
+      .set({ consumedAt: now })
+      .where(and(
+        eq(googleOauthFlows.stateHash, oauthStateHash(input.state)),
+        isNull(googleOauthFlows.consumedAt),
+        gt(googleOauthFlows.expiresAt, now),
+      ))
+      .returning();
+    if (!flow || !oauthFlowMatches(flow, input.state, input.browserBinding, now)) {
+      throw new MutationError('invalid_oauth_state', 'The Google authorization request is invalid or expired.', 400);
+    }
+    const verifier = decryptSecret(
+      flow.pkceVerifierEncrypted,
+      this.config.encryptionKey,
+      PKCE_PURPOSE,
+    );
+    const token = await this.exchangeToken(new URLSearchParams({
+      client_id: this.config.clientId!,
+      client_secret: this.config.clientSecret!,
+      code: input.code,
+      code_verifier: verifier,
+      grant_type: 'authorization_code',
+      redirect_uri: this.config.redirectUri!,
+    }));
+    if (!token.access_token) {
+      throw new GoogleApiError('Google did not return an access token. Connect again.', 502, true);
+    }
+    const [existing] = await this.db
+      .select()
+      .from(googleConnections)
+      .where(eq(googleConnections.userId, flow.userId))
+      .limit(1);
+    const refreshTokenEncrypted = token.refresh_token
+      ? encryptSecret(token.refresh_token, this.config.encryptionKey, REFRESH_TOKEN_PURPOSE)
+      : existing?.refreshTokenEncrypted;
+    if (!refreshTokenEncrypted) {
+      throw new GoogleApiError('Google did not grant offline access. Connect again and approve access.', 502, true);
+    }
+    const expiresAt = token.expires_in
+      ? new Date(now.getTime() + token.expires_in * 1_000)
+      : null;
+    const scopes = token.scope?.split(/\s+/).filter(Boolean) ?? [...GOOGLE_OAUTH_SCOPES];
+    await this.db.transaction(async (tx) => {
+      await tx.insert(googleConnections).values({
+        userId: flow.userId,
+        refreshTokenEncrypted,
+        accessTokenEncrypted: encryptSecret(token.access_token!, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE),
+        accessTokenExpiresAt: expiresAt,
+        scopes,
+        connectedAt: now,
+        updatedAt: now,
+      }).onConflictDoUpdate({
+        target: googleConnections.userId,
+        set: {
+          refreshTokenEncrypted,
+          accessTokenEncrypted: encryptSecret(token.access_token!, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE),
+          accessTokenExpiresAt: expiresAt,
+          scopes,
+          connectedAt: now,
+          updatedAt: now,
+        },
+      });
+      await tx.insert(activities).values({
+        type: 'note',
+        subject: 'Connected Google Sheets import',
+        actorUserId: flow.userId,
+        source: 'manual',
+        occurredAt: now,
+        meta: { action: 'integration.connected', integration: 'google_sheets' },
+      });
+    });
+  }
+
+  async disconnect(principal: Principal, now = new Date()): Promise<void> {
+    const [connection] = await this.db
+      .select()
+      .from(googleConnections)
+      .where(eq(googleConnections.userId, principal.userId))
+      .limit(1);
+    if (!connection) return;
+    const token = decryptSecret(
+      connection.refreshTokenEncrypted,
+      this.config.encryptionKey,
+      REFRESH_TOKEN_PURPOSE,
+    );
+    const response = await this.fetchImpl(GOOGLE_REVOCATION_ENDPOINT, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: new URLSearchParams({ token }),
+    });
+    if (!response.ok && response.status !== 400) {
+      throw new GoogleApiError('Google access could not be revoked. Try again.', response.status);
+    }
+    await this.db.transaction(async (tx) => {
+      await tx.delete(googleConnections).where(eq(googleConnections.userId, principal.userId));
+      await tx.insert(activities).values({
+        type: 'note',
+        subject: 'Disconnected Google Sheets import',
+        actorUserId: principal.userId,
+        source: 'manual',
+        occurredAt: now,
+        meta: { action: 'integration.disconnected', integration: 'google_sheets' },
+      });
+    });
+  }
+
+  async listSpreadsheets(
+    userId: string,
+    options: { pageToken?: string; search?: string },
+  ): Promise<{ files: GoogleDriveFile[]; nextPageToken: string | null; incomplete: boolean }> {
+    const token = await this.accessToken(userId);
+    const query = [
+      "mimeType='application/vnd.google-apps.spreadsheet'",
+      'trashed=false',
+      ...(options.search?.trim()
+        ? [`name contains '${escapeDriveQuery(options.search.trim().slice(0, 100))}'`]
+        : []),
+    ].join(' and ');
+    const url = new URL(GOOGLE_DRIVE_FILES_ENDPOINT);
+    url.searchParams.set('q', query);
+    url.searchParams.set('spaces', 'drive');
+    url.searchParams.set('orderBy', 'modifiedTime desc,name_natural');
+    url.searchParams.set('pageSize', String(DRIVE_PAGE_SIZE));
+    url.searchParams.set('fields', 'files(id,name,modifiedTime),nextPageToken,incompleteSearch');
+    url.searchParams.set('supportsAllDrives', 'true');
+    url.searchParams.set('includeItemsFromAllDrives', 'true');
+    if (options.pageToken) url.searchParams.set('pageToken', options.pageToken);
+    const body = await this.googleJson<DriveListResponse>(url, token, 'Google Drive could not list spreadsheets.');
+    return {
+      files: (body.files ?? []).flatMap((file) => file.id && file.name
+        ? [{ id: file.id, name: file.name, modifiedTime: file.modifiedTime ?? null }]
+        : []),
+      nextPageToken: body.nextPageToken ?? null,
+      incomplete: body.incompleteSearch === true,
+    };
+  }
+
+  async spreadsheetMetadata(
+    userId: string,
+    spreadsheetId: string,
+  ): Promise<{ title: string; sheets: GoogleSheetMetadata[] }> {
+    const token = await this.accessToken(userId);
+    const url = new URL(`${GOOGLE_SHEETS_ENDPOINT}/${encodeURIComponent(spreadsheetId)}`);
+    url.searchParams.set(
+      'fields',
+      'properties(title),sheets(properties(sheetId,title,index,sheetType,hidden,gridProperties(rowCount,columnCount)))',
+    );
+    const body = await this.googleJson<SpreadsheetMetadataResponse>(url, token, 'Google Sheets could not read spreadsheet metadata.');
+    const sheets = (body.sheets ?? []).flatMap((sheet) => {
+      const properties = sheet.properties;
+      if (
+        !properties ||
+        properties.sheetType !== 'GRID' ||
+        properties.hidden ||
+        properties.sheetId == null ||
+        !properties.title
+      ) return [];
+      return [{
+        sheetId: properties.sheetId,
+        title: properties.title,
+        rowCount: properties.gridProperties?.rowCount ?? 0,
+        columnCount: properties.gridProperties?.columnCount ?? 0,
+      }];
+    });
+    return { title: body.properties?.title ?? 'Google spreadsheet', sheets };
+  }
+
+  async readTable(
+    userId: string,
+    input: { spreadsheetId: string; sheetId: number; range: string },
+  ): Promise<ParsedTable> {
+    const metadata = await this.spreadsheetMetadata(userId, input.spreadsheetId);
+    const sheet = metadata.sheets.find((candidate) => candidate.sheetId === input.sheetId);
+    if (!sheet) throw new MutationError('google_sheet_not_found', 'The selected visible grid sheet no longer exists.', 404);
+    const bounded = parseBoundedGoogleRange(input.range, {
+      rowCount: sheet.rowCount,
+      columnCount: sheet.columnCount,
+    });
+    const a1 = `'${sheet.title.replace(/'/g, "''")}'!${bounded.a1}`;
+    const token = await this.accessToken(userId);
+    const url = new URL(
+      `${GOOGLE_SHEETS_ENDPOINT}/${encodeURIComponent(input.spreadsheetId)}/values/${encodeURIComponent(a1)}`,
+    );
+    url.searchParams.set('majorDimension', 'ROWS');
+    // Formatted values expose cached/display results, never executable formula source.
+    url.searchParams.set('valueRenderOption', 'FORMATTED_VALUE');
+    const body = await this.googleJson<ValuesResponse>(url, token, 'Google Sheets could not read the selected range.');
+    const table = normaliseGoogleValues(body.values ?? [], bounded.columns);
+    return {
+      fileName: `Google Sheets · ${metadata.title} · ${sheet.title} · ${bounded.a1}`,
+      sheetName: sheet.title,
+      ...table,
+    };
+  }
+
+  private async accessToken(userId: string, now = new Date()): Promise<string> {
+    const [connection] = await this.db
+      .select()
+      .from(googleConnections)
+      .where(eq(googleConnections.userId, userId))
+      .limit(1);
+    if (!connection) throw new GoogleApiError('Connect Google Sheets before selecting a spreadsheet.', 409, true);
+    if (
+      connection.accessTokenEncrypted &&
+      connection.accessTokenExpiresAt &&
+      connection.accessTokenExpiresAt.getTime() > now.getTime() + ACCESS_TOKEN_SKEW_MS
+    ) {
+      return decryptSecret(connection.accessTokenEncrypted, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE);
+    }
+    this.requireConfigured();
+    const refreshToken = decryptSecret(
+      connection.refreshTokenEncrypted,
+      this.config.encryptionKey,
+      REFRESH_TOKEN_PURPOSE,
+    );
+    const token = await this.exchangeToken(new URLSearchParams({
+      client_id: this.config.clientId!,
+      client_secret: this.config.clientSecret!,
+      refresh_token: refreshToken,
+      grant_type: 'refresh_token',
+    }));
+    if (!token.access_token) throw new GoogleApiError('Google access expired. Connect again.', 401, true);
+    await this.db.update(googleConnections).set({
+      accessTokenEncrypted: encryptSecret(token.access_token, this.config.encryptionKey, ACCESS_TOKEN_PURPOSE),
+      accessTokenExpiresAt: token.expires_in
+        ? new Date(now.getTime() + token.expires_in * 1_000)
+        : null,
+      updatedAt: now,
+    }).where(eq(googleConnections.userId, userId));
+    return token.access_token;
+  }
+
+  private async exchangeToken(parameters: URLSearchParams): Promise<GoogleTokenResponse> {
+    const response = await this.fetchImpl(GOOGLE_TOKEN_ENDPOINT, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: parameters,
+    });
+    if (!response.ok) {
+      throw new GoogleApiError('Google authorization failed. Connect again.', response.status, true);
+    }
+    return await response.json() as GoogleTokenResponse;
+  }
+
+  private async googleJson<T>(url: URL, token: string, message: string): Promise<T> {
+    const response = await this.fetchImpl(url, {
+      headers: { authorization: `Bearer ${token}` },
+    });
+    if (!response.ok) {
+      throw new GoogleApiError(message, response.status, response.status === 401 || response.status === 403);
+    }
+    return await response.json() as T;
+  }
+
+  private requireConfigured(): void {
+    if (!this.configured()) {
+      throw new MutationError(
+        'google_not_configured',
+        'Google Sheets import requires OAuth credentials and the settings encryption key.',
+        503,
+      );
+    }
+  }
+}
+
+export function oauthStateHash(state: string): string {
+  return createHash('sha256').update(state).digest('hex');
+}
+
+export function oauthFlowMatches(
+  flow: Pick<GoogleOauthFlow, 'stateHash' | 'browserBindingHash' | 'expiresAt' | 'consumedAt'>,
+  state: string,
+  browserBinding: string,
+  now: Date,
+): boolean {
+  if (!flow.consumedAt || flow.consumedAt.getTime() !== now.getTime() || flow.expiresAt <= now) return false;
+  const expected = Buffer.from(flow.stateHash, 'hex');
+  const actual = Buffer.from(oauthStateHash(state), 'hex');
+  const expectedBinding = Buffer.from(flow.browserBindingHash, 'hex');
+  const actualBinding = Buffer.from(oauthStateHash(browserBinding), 'hex');
+  return (
+    expected.length === actual.length &&
+    timingSafeEqual(expected, actual) &&
+    expectedBinding.length === actualBinding.length &&
+    timingSafeEqual(expectedBinding, actualBinding)
+  );
+}
+
+export function buildGoogleAuthorizationUrl(input: {
+  clientId: string;
+  redirectUri: string;
+  state: string;
+  challenge: string;
+}): string {
+  const url = new URL(GOOGLE_AUTHORIZATION_ENDPOINT);
+  url.searchParams.set('client_id', input.clientId);
+  url.searchParams.set('redirect_uri', input.redirectUri);
+  url.searchParams.set('response_type', 'code');
+  url.searchParams.set('scope', GOOGLE_OAUTH_SCOPES.join(' '));
+  url.searchParams.set('state', input.state);
+  url.searchParams.set('code_challenge', input.challenge);
+  url.searchParams.set('code_challenge_method', 'S256');
+  url.searchParams.set('access_type', 'offline');
+  url.searchParams.set('include_granted_scopes', 'true');
+  url.searchParams.set('prompt', 'consent');
+  return url.toString();
+}
+
+export function googleConnectionMetadata(
+  configured: boolean,
+  connection: GoogleConnection | undefined,
+): GoogleConnectionMetadata {
+  return {
+    configured,
+    connected: Boolean(connection),
+    connectedAt: connection?.connectedAt.toISOString() ?? null,
+    scopes: connection?.scopes ?? [],
+  };
+}
+
+export interface BoundedGoogleRange {
+  a1: string;
+  rows: number;
+  columns: number;
+}
+
+export function parseBoundedGoogleRange(
+  value: string,
+  grid?: { rowCount: number; columnCount: number },
+): BoundedGoogleRange {
+  const match = value.trim().match(/^([A-Za-z]{1,3})([1-9]\d*):([A-Za-z]{1,3})([1-9]\d*)$/);
+  if (!match) {
+    throw new MutationError('invalid_google_range', 'Use a rectangular A1 range such as A1:H500.', 400);
+  }
+  const startColumn = columnNumber(match[1]!);
+  const endColumn = columnNumber(match[3]!);
+  const startRow = Number(match[2]);
+  const endRow = Number(match[4]);
+  if (endColumn < startColumn || endRow < startRow) {
+    throw new MutationError('invalid_google_range', 'The range end must follow its start.', 400);
+  }
+  const columns = endColumn - startColumn + 1;
+  const rows = endRow - startRow + 1;
+  if (columns > MAX_IMPORT_COLUMNS || rows > MAX_IMPORT_ROWS + 1) {
+    throw new MutationError(
+      'google_range_too_large',
+      `Select at most ${MAX_IMPORT_COLUMNS} columns and ${MAX_IMPORT_ROWS + 1} rows including the header.`,
+      400,
+    );
+  }
+  if (grid && (endColumn > grid.columnCount || endRow > grid.rowCount)) {
+    throw new MutationError('google_range_outside_sheet', 'The selected range extends beyond the sheet grid.', 400);
+  }
+  return { a1: `${match[1]!.toUpperCase()}${startRow}:${match[3]!.toUpperCase()}${endRow}`, rows, columns };
+}
+
+export function normaliseGoogleValues(
+  values: readonly (readonly unknown[])[],
+  requestedColumns: number,
+): Omit<ParsedTable, 'fileName' | 'sheetName'> {
+  const table = values.map((row) => {
+    if (row.length > requestedColumns) {
+      throw new MutationError('invalid_google_values', 'Google returned cells outside the requested range.', 502);
+    }
+    return row.map((value) => normaliseGoogleCell(value));
+  });
+  return normaliseTabularRows(table, [
+    'Google formula source was not imported; formula cells use only their formatted cached result.',
+  ]);
+}
+
+function normaliseGoogleCell(value: unknown): string {
+  if (value == null) return '';
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value)) throw new MutationError('invalid_google_values', 'Google returned a non-finite number.', 502);
+    return String(value);
+  }
+  if (typeof value === 'string') {
+    if (value.length > MAX_IMPORT_CELL_CHARS) {
+      throw new MutationError('google_cell_too_large', `A Google Sheets cell exceeds ${MAX_IMPORT_CELL_CHARS} characters.`, 400);
+    }
+    return value;
+  }
+  throw new MutationError('invalid_google_values', 'Google returned an unsupported cell value.', 502);
+}
+
+function columnNumber(letters: string): number {
+  let result = 0;
+  for (const letter of letters.toUpperCase()) result = result * 26 + letter.charCodeAt(0) - 64;
+  return result;
+}
+
+function escapeDriveQuery(value: string): string {
+  return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+}