From f56a0e4676c3ed549d184e6de3e5b314ca9f2680 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 2 Jun 2026 14:06:02 -0700 Subject: [PATCH 1/5] fix(schedules): count usage lim error schedule as failed run (#4853) * fix(schedules): count usage lim error schedule as failed run * remove backoff logic --- .../app/api/schedules/execute/route.test.ts | 9 ++ apps/sim/app/api/schedules/execute/route.ts | 22 +---- apps/sim/background/schedule-execution.ts | 82 +++++++++---------- 3 files changed, 51 insertions(+), 62 deletions(-) diff --git a/apps/sim/app/api/schedules/execute/route.test.ts b/apps/sim/app/api/schedules/execute/route.test.ts index 2fe434e87b2..2d0fa95ff55 100644 --- a/apps/sim/app/api/schedules/execute/route.test.ts +++ b/apps/sim/app/api/schedules/execute/route.test.ts @@ -50,6 +50,15 @@ vi.mock('@/background/schedule-execution', () => ({ executeScheduleJob: mockExecuteScheduleJob, executeJobInline: mockExecuteJobInline, releaseScheduleLock: mockReleaseScheduleLock, + buildScheduleFailureUpdate: (now: Date, nextRunAt: Date | null) => ({ + updatedAt: now, + lastQueuedAt: null, + nextRunAt, + failedCount: { type: 'sql' }, + lastFailedAt: now, + status: { type: 'sql' }, + infraRetryCount: 0, + }), })) vi.mock('@/lib/core/config/feature-flags', () => mockFeatureFlags) diff --git a/apps/sim/app/api/schedules/execute/route.ts b/apps/sim/app/api/schedules/execute/route.ts index 6dedd8d7494..414ae81047b 100644 --- a/apps/sim/app/api/schedules/execute/route.ts +++ b/apps/sim/app/api/schedules/execute/route.ts @@ -27,12 +27,12 @@ import { SCHEDULE_WORKFLOW_ENQUEUE_LIMIT, } from '@/lib/workflows/schedules/execution-limits' import { + buildScheduleFailureUpdate, executeJobInline, executeScheduleJob, releaseScheduleLock, type ScheduleExecutionPayload, } from '@/background/schedule-execution' -import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants' export const dynamic = 'force-dynamic' export const maxDuration = 3600 @@ -321,15 +321,7 @@ async function markClaimedScheduleFailed( const now = new Date() await db .update(workflowSchedule) - .set({ - updatedAt: now, - lastQueuedAt: null, - lastFailedAt: now, - nextRunAt: getScheduleNextRunAt(schedule, now), - failedCount: sql`COALESCE(${workflowSchedule.failedCount}, 0) + 1`, - status: sql`CASE WHEN COALESCE(${workflowSchedule.failedCount}, 0) + 1 >= ${MAX_CONSECUTIVE_FAILURES} THEN 'disabled' ELSE 'active' END`, - infraRetryCount: 0, - }) + .set(buildScheduleFailureUpdate(now, getScheduleNextRunAt(schedule, now))) .where( and( eq(workflowSchedule.id, schedule.id), @@ -482,15 +474,7 @@ async function recoverStaleDatabaseScheduleJobs(now: Date): Promise { await tx .update(workflowSchedule) - .set({ - updatedAt: now, - lastQueuedAt: null, - lastFailedAt: now, - nextRunAt: getScheduleNextRunAt(payload, now), - failedCount: sql`COALESCE(${workflowSchedule.failedCount}, 0) + 1`, - status: sql`CASE WHEN COALESCE(${workflowSchedule.failedCount}, 0) + 1 >= ${MAX_CONSECUTIVE_FAILURES} THEN 'disabled' ELSE 'active' END`, - infraRetryCount: 0, - }) + .set(buildScheduleFailureUpdate(now, getScheduleNextRunAt(payload, now))) .where( and( eq(workflowSchedule.id, payload.scheduleId), diff --git a/apps/sim/background/schedule-execution.ts b/apps/sim/background/schedule-execution.ts index 4c240e5f1e7..b90886d7f71 100644 --- a/apps/sim/background/schedule-execution.ts +++ b/apps/sim/background/schedule-execution.ts @@ -76,6 +76,29 @@ function resetScheduleInfraRetryCount(): Pick Date: Tue, 2 Jun 2026 16:57:30 -0700 Subject: [PATCH 2/5] feat(connectors): add 11 knowledge base connectors (#4849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(connectors): add 11 knowledge base connectors (Gong, Grain, Fathom, Granola, incident.io, Rootly, Ashby, Greenhouse, DocuSign, Monday, GitLab) * fix(granola): restore correct calendar_event field names (event_title/organiser/calendar_event_id/scheduled_start_time/scheduled_end_time/invitees) * fix(connectors): correct Grain id field/include flags and incident.io timestamp value shape (verified vs raw API specs) * feat(connectors): add scoping filters for Gong (host users), Fathom (meeting type/domain), Granola (folder/created-after) * feat(connectors): add verified scoping filters (Grain, Rootly, Ashby, Greenhouse, DocuSign, GitLab) + clarify Monday board scope * fix(connectors): docusign sandbox source URL, greenhouse scorecard cap, gong overlap; add incident.io status/mode filters * chore(connectors): remove non-TSDoc inline comments * fix(fathom): apply 14-day overlap to incremental created_after so late transcripts are recaptured * fix(fathom): cache sourceUrl in header so getDocument preserves it; return concrete contentHash (drop omit/cast) * fix(connectors): grain check transcript before formatting; gong cache date window across cursor pages * fix(connectors): fathom skip meetings with no transcript/summary; docusign cache from_date across cursor pages * fix(fathom): return metadata from getDocument (carried via header cache) for tag consistency * fix(connectors): grain cache filter window across pages, greenhouse guard NaN page, rootly skip empty incidents * feat(connectors): final audit — add verified scoping filters (grain team/type, granola+greenhouse createdBefore, gitlab milestone, rootly service/team/env), fix incident.io paused category + rootly severity-slug copy --- apps/docs/content/docs/en/tools/granola.mdx | 2 + apps/sim/blocks/blocks/granola.ts | 10 + apps/sim/connectors/ashby/ashby.ts | 619 ++++++++++++++++ apps/sim/connectors/ashby/index.ts | 1 + apps/sim/connectors/docusign/docusign.ts | 682 +++++++++++++++++ apps/sim/connectors/docusign/index.ts | 1 + apps/sim/connectors/fathom/fathom.ts | 605 +++++++++++++++ apps/sim/connectors/fathom/index.ts | 1 + apps/sim/connectors/gitlab/gitlab.ts | 704 ++++++++++++++++++ apps/sim/connectors/gitlab/index.ts | 1 + apps/sim/connectors/gong/gong.ts | 594 +++++++++++++++ apps/sim/connectors/gong/index.ts | 1 + apps/sim/connectors/grain/grain.ts | 587 +++++++++++++++ apps/sim/connectors/grain/index.ts | 1 + apps/sim/connectors/granola/granola.ts | 528 +++++++++++++ apps/sim/connectors/granola/index.ts | 1 + apps/sim/connectors/greenhouse/greenhouse.ts | 737 +++++++++++++++++++ apps/sim/connectors/greenhouse/index.ts | 1 + apps/sim/connectors/incidentio/incidentio.ts | 623 ++++++++++++++++ apps/sim/connectors/incidentio/index.ts | 1 + apps/sim/connectors/monday/index.ts | 1 + apps/sim/connectors/monday/monday.ts | 553 ++++++++++++++ apps/sim/connectors/registry.ts | 22 + apps/sim/connectors/rootly/index.ts | 1 + apps/sim/connectors/rootly/rootly.ts | 660 +++++++++++++++++ apps/sim/lib/oauth/oauth.ts | 2 +- apps/sim/tools/granola/get_note.ts | 2 + apps/sim/tools/granola/list_notes.ts | 7 + apps/sim/tools/granola/types.ts | 2 + apps/sim/tools/monday/utils.ts | 2 +- 30 files changed, 6950 insertions(+), 2 deletions(-) create mode 100644 apps/sim/connectors/ashby/ashby.ts create mode 100644 apps/sim/connectors/ashby/index.ts create mode 100644 apps/sim/connectors/docusign/docusign.ts create mode 100644 apps/sim/connectors/docusign/index.ts create mode 100644 apps/sim/connectors/fathom/fathom.ts create mode 100644 apps/sim/connectors/fathom/index.ts create mode 100644 apps/sim/connectors/gitlab/gitlab.ts create mode 100644 apps/sim/connectors/gitlab/index.ts create mode 100644 apps/sim/connectors/gong/gong.ts create mode 100644 apps/sim/connectors/gong/index.ts create mode 100644 apps/sim/connectors/grain/grain.ts create mode 100644 apps/sim/connectors/grain/index.ts create mode 100644 apps/sim/connectors/granola/granola.ts create mode 100644 apps/sim/connectors/granola/index.ts create mode 100644 apps/sim/connectors/greenhouse/greenhouse.ts create mode 100644 apps/sim/connectors/greenhouse/index.ts create mode 100644 apps/sim/connectors/incidentio/incidentio.ts create mode 100644 apps/sim/connectors/incidentio/index.ts create mode 100644 apps/sim/connectors/monday/index.ts create mode 100644 apps/sim/connectors/monday/monday.ts create mode 100644 apps/sim/connectors/rootly/index.ts create mode 100644 apps/sim/connectors/rootly/rootly.ts diff --git a/apps/docs/content/docs/en/tools/granola.mdx b/apps/docs/content/docs/en/tools/granola.mdx index 88eb50622a3..1d5bf775c5d 100644 --- a/apps/docs/content/docs/en/tools/granola.mdx +++ b/apps/docs/content/docs/en/tools/granola.mdx @@ -30,6 +30,7 @@ Lists meeting notes from Granola with optional date filters and pagination. | `createdBefore` | string | No | Return notes created before this date \(ISO 8601\) | | `createdAfter` | string | No | Return notes created after this date \(ISO 8601\) | | `updatedAfter` | string | No | Return notes updated after this date \(ISO 8601\) | +| `folderId` | string | No | Return notes in this folder and its child folders \(e.g., fol_4y6LduVdwSKC27\) | | `cursor` | string | No | Pagination cursor from a previous response | | `pageSize` | number | No | Number of notes per page \(1-30, default 10\) | @@ -69,6 +70,7 @@ Retrieves a specific meeting note from Granola by ID, including summary, attende | `ownerEmail` | string | Note owner email | | `createdAt` | string | Creation timestamp | | `updatedAt` | string | Last update timestamp | +| `webUrl` | string | URL to view the note in Granola | | `summaryText` | string | Plain text summary of the meeting | | `summaryMarkdown` | string | Markdown-formatted summary of the meeting | | `attendees` | json | Meeting attendees | diff --git a/apps/sim/blocks/blocks/granola.ts b/apps/sim/blocks/blocks/granola.ts index 0062fbee815..3a2c46efb00 100644 --- a/apps/sim/blocks/blocks/granola.ts +++ b/apps/sim/blocks/blocks/granola.ts @@ -96,6 +96,14 @@ export const GranolaBlock: BlockConfig = { generationType: 'timestamp', }, }, + { + id: 'folderId', + title: 'Folder ID', + type: 'short-input', + placeholder: 'e.g., fol_4y6LduVdwSKC27', + condition: { field: 'operation', value: 'list_notes' }, + mode: 'advanced', + }, { id: 'pageSize', title: 'Page Size', @@ -134,6 +142,7 @@ export const GranolaBlock: BlockConfig = { createdAfter: { type: 'string', description: 'Filter notes created after this date' }, createdBefore: { type: 'string', description: 'Filter notes created before this date' }, updatedAfter: { type: 'string', description: 'Filter notes updated after this date' }, + folderId: { type: 'string', description: 'Filter notes by folder ID' }, pageSize: { type: 'number', description: 'Results per page (1-30)' }, cursor: { type: 'string', description: 'Pagination cursor' }, }, @@ -151,6 +160,7 @@ export const GranolaBlock: BlockConfig = { ownerEmail: { type: 'string', description: 'Note owner email' }, createdAt: { type: 'string', description: 'Creation timestamp' }, updatedAt: { type: 'string', description: 'Last update timestamp' }, + webUrl: { type: 'string', description: 'URL to view the note in Granola' }, summaryText: { type: 'string', description: 'Plain text meeting summary' }, summaryMarkdown: { type: 'string', description: 'Markdown meeting summary' }, attendees: { type: 'json', description: 'Meeting attendees (name, email)' }, diff --git a/apps/sim/connectors/ashby/ashby.ts b/apps/sim/connectors/ashby/ashby.ts new file mode 100644 index 00000000000..fdf1c21e8b2 --- /dev/null +++ b/apps/sim/connectors/ashby/ashby.ts @@ -0,0 +1,619 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { AshbyIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { parseTagDate } from '@/connectors/utils' + +const logger = createLogger('AshbyConnector') + +const ASHBY_API_BASE = 'https://api.ashbyhq.com' +const CANDIDATES_PER_PAGE = 100 +const NOTES_PER_PAGE = 100 +const FEEDBACK_PER_PAGE = 100 + +/** + * Hard cap on the number of applications whose interview feedback is fetched for a + * single candidate document. Candidates with many applications are rare, but this + * bounds the number of feedback API calls per `getDocument` invocation. + */ +const MAX_APPLICATIONS_FOR_FEEDBACK = 10 + +type UnknownRecord = Record + +/** + * Builds the standard Ashby Authorization header. Ashby uses HTTP Basic auth with + * the API key as the username and an empty password, i.e. `Basic base64(apiKey + ':')`. + */ +function ashbyHeaders(accessToken: string): Record { + return { + 'Content-Type': 'application/json', + Accept: 'application/json; version=1', + Authorization: `Basic ${Buffer.from(`${accessToken}:`).toString('base64')}`, + } +} + +interface AshbyEnvelope { + success: boolean + results?: unknown + moreDataAvailable?: boolean + nextCursor?: string | null + errors?: unknown + errorInfo?: { message?: string } +} + +/** + * Extracts a human-readable error message from an Ashby error envelope. Ashby returns + * errors as either `errorInfo.message` or an `errors` string array. + */ +function ashbyErrorMessage(data: AshbyEnvelope, fallback: string): string { + if (data.errorInfo?.message) return data.errorInfo.message + if (Array.isArray(data.errors) && data.errors.length > 0) { + return data.errors.map((e) => String(e)).join('; ') + } + return fallback +} + +/** + * Executes an Ashby RPC-style POST request and returns the parsed envelope. + * Ashby exposes a flat set of POST endpoints under `https://api.ashbyhq.com`. + */ +async function ashbyPost( + accessToken: string, + endpoint: string, + body: UnknownRecord, + retryOptions?: Parameters[2] +): Promise { + const response = await fetchWithRetry( + `${ASHBY_API_BASE}/${endpoint}`, + { + method: 'POST', + headers: ashbyHeaders(accessToken), + body: JSON.stringify(body), + }, + retryOptions + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error( + `Ashby ${endpoint} HTTP error: ${response.status}${errorText ? ` — ${errorText.slice(0, 300)}` : ''}` + ) + } + + const data = (await response.json()) as AshbyEnvelope + if (!data.success) { + throw new Error(ashbyErrorMessage(data, `Ashby ${endpoint} request failed`)) + } + return data +} + +interface AshbyCandidateSummary { + id: string + name: string + position: string | null + company: string | null + school: string | null + location: string | null + source: string | null + emailDomain: string | null + profileUrl: string | null + applicationIds: string[] + createdAt: string | null + updatedAt: string | null +} + +/** + * Extracts a human-readable location string from an Ashby candidate's `location` + * object. Prefers the API-provided `locationSummary`. Falls back to joining the + * `name` values of the `locationComponents` array (each entry is `{ type, name }` + * ordered city → region → country, per the candidate entity returned by + * `candidate.list`/`candidate.info`). As a final fallback, supports the flat + * `{ city, region, country }` shape used by candidate write inputs. + */ +function extractLocation(raw: UnknownRecord): string | null { + const location = raw.location as UnknownRecord | undefined + if (!location) return null + + const summary = location.locationSummary as string | undefined + if (summary?.trim()) return summary.trim() + + if (Array.isArray(location.locationComponents)) { + const parts = (location.locationComponents as UnknownRecord[]) + .map((c) => c?.name) + .filter((n): n is string => typeof n === 'string' && n.trim().length > 0) + .map((n) => n.trim()) + if (parts.length > 0) return parts.join(', ') + } + + const parts = [location.city, location.region, location.country] + .filter((p): p is string => typeof p === 'string' && p.trim().length > 0) + .map((p) => p.trim()) + return parts.length > 0 ? parts.join(', ') : null +} + +/** + * Extracts the source title from an Ashby candidate's `source` object, which + * references the organization's sources list (e.g. "LinkedIn", "Referral"). + */ +function extractSource(raw: UnknownRecord): string | null { + const source = raw.source as UnknownRecord | undefined + const title = source?.title as string | undefined + return title?.trim() || null +} + +/** + * Extracts the lowercased domain from an Ashby candidate's primary email address + * (`primaryEmailAddress.value`), enabling filtering candidates by email domain. + */ +function extractEmailDomain(raw: UnknownRecord): string | null { + const email = raw.primaryEmailAddress as UnknownRecord | undefined + const value = email?.value as string | undefined + const at = value?.lastIndexOf('@') ?? -1 + if (!value || at < 0 || at === value.length - 1) return null + return ( + value + .slice(at + 1) + .trim() + .toLowerCase() || null + ) +} + +/** + * Normalizes a raw Ashby candidate record into the fields this connector cares about. + * Field names mirror the Ashby candidate object returned by `candidate.list` and + * `candidate.info` (`position`, `company`, `school`, `location`, `source`, + * `primaryEmailAddress`, `profileUrl`, `applicationIds`, `createdAt`, `updatedAt`). + * Stage and status live on applications rather than candidates, so they are + * intentionally not surfaced here. + */ +function mapCandidate(raw: unknown): AshbyCandidateSummary { + const c = (raw ?? {}) as UnknownRecord + return { + id: (c.id as string) ?? '', + name: (c.name as string) ?? '', + position: (c.position as string) ?? null, + company: (c.company as string) ?? null, + school: (c.school as string) ?? null, + location: extractLocation(c), + source: extractSource(c), + emailDomain: extractEmailDomain(c), + profileUrl: (c.profileUrl as string) ?? null, + applicationIds: Array.isArray(c.applicationIds) ? (c.applicationIds as string[]) : [], + createdAt: (c.createdAt as string) ?? null, + updatedAt: (c.updatedAt as string) ?? null, + } +} + +interface AshbyNote { + content: string | null + authorName: string | null + createdAt: string | null +} + +/** + * Maps a raw Ashby candidate note into a plain-text-friendly shape, combining the + * author's first and last name into a single display name. + */ +function mapNote(raw: unknown): AshbyNote { + const n = (raw ?? {}) as UnknownRecord + const author = n.author as UnknownRecord | undefined + const first = (author?.firstName as string) ?? '' + const last = (author?.lastName as string) ?? '' + const authorName = `${first} ${last}`.trim() || (author?.email as string) || null + return { + content: (n.content as string) ?? null, + authorName, + createdAt: (n.createdAt as string) ?? null, + } +} + +interface AshbyFeedbackSummary { + submittedByName: string | null + submittedAt: string | null + lines: string[] +} + +/** + * Collects `{ field.path -> field.title }` entries from a feedback form definition. + * Ashby's `formDefinition` exposes fields either flat under `fields[]` or grouped + * under `sections[].fields[]`, and individual entries are sometimes wrapped in a + * `{ field }` envelope — all variants are handled. + */ +function collectFieldTitles(formDefinition: UnknownRecord | undefined): Map { + const titleByPath = new Map() + if (!formDefinition) return titleByPath + + const addField = (entry: UnknownRecord): void => { + const field = (entry?.field ?? entry) as UnknownRecord + const path = field?.path as string | undefined + const title = (field?.title as string) || (field?.humanReadablePath as string) + if (path && title) titleByPath.set(path, title) + } + + if (Array.isArray(formDefinition.fields)) { + for (const entry of formDefinition.fields as UnknownRecord[]) addField(entry) + } + + if (Array.isArray(formDefinition.sections)) { + for (const section of formDefinition.sections as UnknownRecord[]) { + const fields = Array.isArray(section?.fields) ? (section.fields as UnknownRecord[]) : [] + for (const entry of fields) addField(entry) + } + } + + return titleByPath +} + +/** + * Maps a raw Ashby application feedback submission into a flat list of + * `Title: value` lines, resolving each `submittedValues` key (the field's `path`) + * to its human-readable title via the form definition. Falls back to the raw path + * when no title is found. + */ +function mapFeedback(raw: unknown): AshbyFeedbackSummary { + const f = (raw ?? {}) as UnknownRecord + const submittedBy = f.submittedByUser as UnknownRecord | undefined + const first = (submittedBy?.firstName as string) ?? '' + const last = (submittedBy?.lastName as string) ?? '' + const submittedByName = `${first} ${last}`.trim() || (submittedBy?.email as string) || null + + const titleByPath = collectFieldTitles(f.formDefinition as UnknownRecord | undefined) + + const submittedValues = (f.submittedValues as UnknownRecord | undefined) ?? {} + const lines: string[] = [] + for (const [path, value] of Object.entries(submittedValues)) { + if (value == null) continue + const label = titleByPath.get(path) ?? path + const rendered = renderFeedbackValue(value) + if (rendered) lines.push(`${label}: ${rendered}`) + } + + const submittedAt = + (f.submittedAt as string) ?? (f.completedAt as string) ?? (f.createdAt as string) ?? null + + return { submittedByName, submittedAt, lines } +} + +/** + * Renders an arbitrary submitted feedback value (string, number, boolean, or a + * rich-text / structured object) into a single-line plain-text string. + */ +function renderFeedbackValue(value: unknown): string { + if (typeof value === 'string') return value.trim() + if (typeof value === 'number' || typeof value === 'boolean') return String(value) + if (Array.isArray(value)) { + return value + .map((v) => renderFeedbackValue(v)) + .filter(Boolean) + .join(', ') + } + if (value && typeof value === 'object') { + const obj = value as UnknownRecord + const label = obj.label ?? obj.value ?? obj.text ?? obj.content + if (typeof label === 'string') return label.trim() + } + return '' +} + +/** + * Stable, metadata-based content hash for a candidate document. Identical between the + * listing stub and the fully-fetched document so unchanged candidates are skipped. + */ +function buildContentHash(id: string, updatedAt: string | null): string { + return `ashby:${id}:${updatedAt ?? ''}` +} + +/** + * Creates a lightweight document stub from a candidate listing entry. Content is + * deferred and only fetched (via `getDocument`) for new or changed candidates. + */ +function candidateToStub(candidate: AshbyCandidateSummary): ExternalDocument { + return { + externalId: candidate.id, + title: candidate.name || 'Unnamed Candidate', + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: candidate.profileUrl ?? undefined, + contentHash: buildContentHash(candidate.id, candidate.updatedAt), + metadata: candidateMetadata(candidate), + } +} + +/** + * Builds the tag-carrying metadata block shared by the listing stub and the + * fully-fetched document, keeping the keys aligned with `mapTags`/`tagDefinitions`. + */ +function candidateMetadata(candidate: AshbyCandidateSummary): Record { + return { + candidateName: candidate.name, + company: candidate.company, + school: candidate.school, + location: candidate.location, + source: candidate.source, + emailDomain: candidate.emailDomain, + createdAt: candidate.createdAt, + updatedAt: candidate.updatedAt, + } +} + +/** + * Fetches all notes for a candidate, following cursor pagination. + */ +async function fetchAllNotes(accessToken: string, candidateId: string): Promise { + const notes: AshbyNote[] = [] + let cursor: string | undefined + let hasMore = true + + while (hasMore) { + const body: UnknownRecord = { candidateId, limit: NOTES_PER_PAGE } + if (cursor) body.cursor = cursor + const data = await ashbyPost(accessToken, 'candidate.listNotes', body) + const results = Array.isArray(data.results) ? data.results : [] + for (const raw of results) notes.push(mapNote(raw)) + cursor = data.nextCursor ?? undefined + hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor) + } + + return notes +} + +/** + * Fetches all interview feedback submissions for a single application, following + * cursor pagination. + */ +async function fetchFeedbackForApplication( + accessToken: string, + applicationId: string +): Promise { + const feedback: AshbyFeedbackSummary[] = [] + let cursor: string | undefined + let hasMore = true + + while (hasMore) { + const body: UnknownRecord = { applicationId, limit: FEEDBACK_PER_PAGE } + if (cursor) body.cursor = cursor + const data = await ashbyPost(accessToken, 'applicationFeedback.list', body) + const results = Array.isArray(data.results) ? data.results : [] + for (const raw of results) feedback.push(mapFeedback(raw)) + cursor = data.nextCursor ?? undefined + hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor) + } + + return feedback +} + +/** + * Assembles a candidate's profile, notes, and interview feedback into a single + * plain-text document body for indexing. + */ +function formatCandidateContent( + candidate: AshbyCandidateSummary, + notes: AshbyNote[], + feedback: AshbyFeedbackSummary[] +): string { + const parts: string[] = [] + + parts.push(`Candidate: ${candidate.name || 'Unnamed Candidate'}`) + if (candidate.position) parts.push(`Current Role: ${candidate.position}`) + if (candidate.company) parts.push(`Current Company: ${candidate.company}`) + if (candidate.school) parts.push(`School: ${candidate.school}`) + if (candidate.location) parts.push(`Location: ${candidate.location}`) + if (candidate.source) parts.push(`Source: ${candidate.source}`) + if (candidate.createdAt) parts.push(`Created: ${candidate.createdAt}`) + if (candidate.updatedAt) parts.push(`Last Updated: ${candidate.updatedAt}`) + + const nonEmptyNotes = notes.filter((n) => n.content?.trim()) + if (nonEmptyNotes.length > 0) { + parts.push('') + parts.push('--- Notes ---') + for (const note of nonEmptyNotes) { + const header = [note.authorName, note.createdAt].filter(Boolean).join(' — ') + if (header) parts.push(`[${header}]`) + parts.push((note.content ?? '').trim()) + parts.push('') + } + } + + const nonEmptyFeedback = feedback.filter((f) => f.lines.length > 0) + if (nonEmptyFeedback.length > 0) { + parts.push('--- Interview Feedback ---') + for (const f of nonEmptyFeedback) { + const header = [f.submittedByName, f.submittedAt].filter(Boolean).join(' — ') + if (header) parts.push(`[${header}]`) + for (const line of f.lines) parts.push(line) + parts.push('') + } + } + + return parts.join('\n').trim() +} + +export const ashbyConnector: ConnectorConfig = { + id: 'ashby', + name: 'Ashby', + description: 'Sync candidate notes and interview feedback from Ashby', + version: '1.0.0', + icon: AshbyIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your Ashby API key', + }, + + configFields: [ + { + id: 'maxCandidates', + title: 'Max Candidates', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + description: + 'Cap the number of candidates synced. Leave empty to sync ALL candidates in the organization.', + }, + { + id: 'createdAfter', + title: 'Created After', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 2025-01-01 or 2025-01-01T00:00:00Z', + description: + 'Only sync candidates created on or after this date (ISO 8601). Leave blank to sync candidates regardless of creation date.', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record + ): Promise => { + const maxCandidates = sourceConfig.maxCandidates ? Number(sourceConfig.maxCandidates) : 0 + const createdAfterMs = (() => { + const raw = sourceConfig.createdAfter + if (typeof raw !== 'string' || !raw.trim()) return undefined + const ms = new Date(raw.trim()).getTime() + return Number.isNaN(ms) ? undefined : ms + })() + + const prevFetched = (syncContext?.totalCandidatesFetched as number) ?? 0 + if (maxCandidates > 0 && prevFetched >= maxCandidates) { + if (syncContext) syncContext.listingCapped = true + return { documents: [], hasMore: false } + } + + const body: UnknownRecord = { limit: CANDIDATES_PER_PAGE } + if (cursor) body.cursor = cursor + if (createdAfterMs !== undefined) body.createdAfter = createdAfterMs + + logger.info('Listing Ashby candidates', { + cursor: cursor ?? 'initial', + maxCandidates: maxCandidates || 'unlimited', + }) + + const data = await ashbyPost(accessToken, 'candidate.list', body) + const results = Array.isArray(data.results) ? data.results : [] + const candidates = results.map(mapCandidate).filter((c) => c.id) + + let documents = candidates.map(candidateToStub) + if (maxCandidates > 0) { + const remaining = Math.max(0, maxCandidates - prevFetched) + if (documents.length > remaining) documents = documents.slice(0, remaining) + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalCandidatesFetched = totalFetched + const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates + if (hitLimit && syncContext) syncContext.listingCapped = true + + const nextCursor = data.nextCursor ?? undefined + const hasMore = !hitLimit && Boolean(data.moreDataAvailable) && Boolean(nextCursor) + + return { + documents, + nextCursor: hasMore ? nextCursor : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const infoData = await ashbyPost(accessToken, 'candidate.info', { id: externalId }) + if (!infoData.results) return null + const candidate = mapCandidate(infoData.results) + if (!candidate.id) return null + + const notes = await fetchAllNotes(accessToken, candidate.id) + + const feedback: AshbyFeedbackSummary[] = [] + const applicationIds = candidate.applicationIds.slice(0, MAX_APPLICATIONS_FOR_FEEDBACK) + for (const applicationId of applicationIds) { + try { + const applicationFeedback = await fetchFeedbackForApplication(accessToken, applicationId) + feedback.push(...applicationFeedback) + } catch (error) { + logger.warn('Failed to fetch Ashby feedback for application', { + applicationId, + error: toError(error).message, + }) + } + } + + const content = formatCandidateContent(candidate, notes, feedback) + if (!content.trim()) return null + + return { + externalId: candidate.id, + title: candidate.name || 'Unnamed Candidate', + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: candidate.profileUrl ?? undefined, + contentHash: buildContentHash(candidate.id, candidate.updatedAt), + metadata: candidateMetadata(candidate), + } + } catch (error) { + logger.warn('Failed to get Ashby candidate', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxCandidates = sourceConfig.maxCandidates as string | undefined + if (maxCandidates && (Number.isNaN(Number(maxCandidates)) || Number(maxCandidates) < 0)) { + return { valid: false, error: 'Max candidates must be a non-negative number' } + } + + try { + await ashbyPost(accessToken, 'candidate.list', { limit: 1 }, VALIDATE_RETRY_OPTIONS) + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'candidateName', displayName: 'Candidate Name', fieldType: 'text' }, + { id: 'company', displayName: 'Current Company', fieldType: 'text' }, + { id: 'school', displayName: 'School', fieldType: 'text' }, + { id: 'location', displayName: 'Location', fieldType: 'text' }, + { id: 'source', displayName: 'Source', fieldType: 'text' }, + { id: 'emailDomain', displayName: 'Email Domain', fieldType: 'text' }, + { id: 'createdAt', displayName: 'Created', fieldType: 'date' }, + { id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + const textTags = ['candidateName', 'company', 'school', 'location', 'source', 'emailDomain'] + for (const key of textTags) { + const value = metadata[key] + if (typeof value === 'string' && value.trim()) result[key] = value.trim() + } + + const createdAt = parseTagDate(metadata.createdAt) + if (createdAt) result.createdAt = createdAt + + const updatedAt = parseTagDate(metadata.updatedAt) + if (updatedAt) result.updatedAt = updatedAt + + return result + }, +} diff --git a/apps/sim/connectors/ashby/index.ts b/apps/sim/connectors/ashby/index.ts new file mode 100644 index 00000000000..707a7e54c71 --- /dev/null +++ b/apps/sim/connectors/ashby/index.ts @@ -0,0 +1 @@ +export { ashbyConnector } from '@/connectors/ashby/ashby' diff --git a/apps/sim/connectors/docusign/docusign.ts b/apps/sim/connectors/docusign/docusign.ts new file mode 100644 index 00000000000..dd6ee757049 --- /dev/null +++ b/apps/sim/connectors/docusign/docusign.ts @@ -0,0 +1,682 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { DocuSignIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { parseTagDate } from '@/connectors/utils' + +const logger = createLogger('DocuSignConnector') + +/** + * DocuSign OAuth userinfo endpoint. Sim's DocuSign OAuth integration is wired to the + * demo/sandbox authorization server (`account-d.docusign.com`, see lib/oauth/oauth.ts token + * endpoint), so the connector resolves account info from the matching demo userinfo host. + * The production host is `https://account.docusign.com/oauth/userinfo`. + */ +const DOCUSIGN_USERINFO_URL = 'https://account-d.docusign.com/oauth/userinfo' + +/** + * DocuSign web-app base for envelope deep links. MUST match the same environment as + * {@link DOCUSIGN_USERINFO_URL}: demo/sandbox envelopes only exist in the demo web app + * (`appdemo.docusign.com`), not production (`app.docusign.com`). Keep these in lockstep + * if the OAuth environment ever changes. + */ +const DOCUSIGN_WEB_BASE = 'https://appdemo.docusign.com' + +const DEFAULT_LOOKBACK_DAYS = 90 +const MAX_PAGE_SIZE = 100 +const DEFAULT_MAX_ENVELOPES = 0 +/** + * Days of overlap added to the incremental sync window. DocuSign status changes are + * indexed by `statusChangedDateTime`, but webhook/processing lag and clock skew can let + * a change land slightly before the recorded sync time. A small overlap re-scans recent + * envelopes so late-recorded changes are not missed. + */ +const INCREMENTAL_OVERLAP_DAYS = 2 +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +interface DocuSignAccount { + account_id?: string + base_uri?: string + is_default?: boolean +} + +interface DocuSignUserInfo { + accounts?: DocuSignAccount[] +} + +interface ResolvedAccount { + accountId: string + baseUri: string +} + +interface DocuSignSigner { + recipientId?: string + name?: string + email?: string + status?: string +} + +interface DocuSignRecipients { + signers?: DocuSignSigner[] + carbonCopies?: DocuSignSigner[] + agents?: DocuSignSigner[] + editors?: DocuSignSigner[] + certifiedDeliveries?: DocuSignSigner[] +} + +interface DocuSignCustomField { + name?: string + value?: string +} + +interface DocuSignCustomFields { + textCustomFields?: DocuSignCustomField[] + listCustomFields?: DocuSignCustomField[] +} + +interface DocuSignDocument { + documentId?: string + name?: string +} + +interface DocuSignEnvelope { + envelopeId?: string + status?: string + emailSubject?: string + emailBlurb?: string + sentDateTime?: string + completedDateTime?: string + createdDateTime?: string + statusChangedDateTime?: string + lastModifiedDateTime?: string + sender?: { userName?: string; email?: string } + recipients?: DocuSignRecipients + customFields?: DocuSignCustomFields + envelopeDocuments?: DocuSignDocument[] +} + +interface DocuSignEnvelopesListResponse { + envelopes?: DocuSignEnvelope[] + resultSetSize?: string + totalSetSize?: string + endPosition?: string + nextUri?: string +} + +interface DocuSignFormValue { + name?: string + value?: string +} + +interface DocuSignRecipientFormData { + formData?: DocuSignFormValue[] +} + +/** + * Response shape of the envelope `form_data` endpoint. The envelope-level entered tab + * values are returned as a top-level `formData` array of `{ name, value }` pairs (there + * is no nested `formValues` property). Per-recipient values live under `recipientFormData`. + */ +interface DocuSignFormData { + formData?: DocuSignFormValue[] + recipientFormData?: DocuSignRecipientFormData[] +} + +/** + * Formats a Date as a UTC ISO 8601 string with explicit time zone offset, the format + * DocuSign recommends for the `from_date` filter on listStatusChanges. + */ +function formatFromDate(date: Date): string { + return date.toISOString() +} + +/** + * Computes the effective lookback window in days, narrowing to the time since the last + * successful sync (plus an overlap to catch late-recorded status changes) when incremental + * sync is active. + */ +function computeLookbackDays( + sourceConfig: Record, + lastSyncAt: Date | undefined +): number { + const raw = sourceConfig.lookback as string | undefined + const configured = Number(raw) + const baseline = + Number.isFinite(configured) && configured > 0 ? Math.floor(configured) : DEFAULT_LOOKBACK_DAYS + + if (!lastSyncAt) return baseline + + const sinceLastSync = Math.ceil((Date.now() - lastSyncAt.getTime()) / MS_PER_DAY) + const incremental = Math.max(sinceLastSync + INCREMENTAL_OVERLAP_DAYS, INCREMENTAL_OVERLAP_DAYS) + return Math.min(incremental, baseline) +} + +/** + * Resolves and caches the user's DocuSign account ID and base URI in the sync context. + * The userinfo lookup is expensive and identical across every page of a sync run, so it is + * cached on the shared `syncContext` (mirrors the Gmail connector's label cache pattern). + */ +async function resolveAccount( + accessToken: string, + syncContext: Record | undefined, + retryOptions?: Parameters[2] +): Promise { + const cacheKey = '_docusignAccount' + const cached = syncContext?.[cacheKey] as ResolvedAccount | undefined + if (cached) return cached + + const response = await fetchWithRetry( + DOCUSIGN_USERINFO_URL, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }, + retryOptions + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error( + `Failed to resolve DocuSign account: ${response.status}${ + errorText ? ` — ${errorText.slice(0, 200)}` : '' + }` + ) + } + + const data = (await response.json()) as DocuSignUserInfo + const accounts = Array.isArray(data.accounts) ? data.accounts : [] + const account = accounts.find((a) => a.is_default) ?? accounts[0] + + if (!account?.account_id || !account.base_uri) { + throw new Error('No accessible DocuSign account found for this user') + } + + const resolved: ResolvedAccount = { + accountId: account.account_id, + baseUri: account.base_uri, + } + if (syncContext) syncContext[cacheKey] = resolved + return resolved +} + +/** + * Builds the REST API base for an account, e.g. + * `https://demo.docusign.net/restapi/v2.1/accounts/{accountId}`. + */ +function apiBaseFor(account: ResolvedAccount): string { + return `${account.baseUri}/restapi/v2.1/accounts/${account.accountId}` +} + +/** + * Builds a metadata-based content hash. Identical between the listDocuments stub and the + * getDocument result so the sync engine can detect changes without downloading content. + */ +function buildContentHash(envelope: DocuSignEnvelope): string { + const envelopeId = envelope.envelopeId ?? '' + const changeIndicator = + envelope.statusChangedDateTime ?? envelope.lastModifiedDateTime ?? envelope.status ?? '' + return `docusign:${envelopeId}:${changeIndicator}` +} + +/** + * Builds a DocuSign web console link to the envelope, when an envelope ID is present. + */ +function buildSourceUrl(envelopeId: string | undefined): string | undefined { + if (!envelopeId) return undefined + return `${DOCUSIGN_WEB_BASE}/documents/details/${envelopeId}` +} + +/** + * Resolves the sender's display name. The envelope summary exposes sender details only via + * the nested `sender` (userInfo) object — there is no top-level `senderName` field. + */ +function resolveSenderName(envelope: DocuSignEnvelope): string | undefined { + return envelope.sender?.userName ?? undefined +} + +/** + * Resolves the sender's email. As with the name, this is only available on the nested + * `sender` (userInfo) object. + */ +function resolveSenderEmail(envelope: DocuSignEnvelope): string | undefined { + return envelope.sender?.email ?? undefined +} + +/** + * Collects all named recipients across roles into a flat list of name/email/status entries. + */ +function collectRecipients( + recipients: DocuSignRecipients | undefined +): { name: string; email: string; status: string }[] { + if (!recipients) return [] + const groups = [ + recipients.signers, + recipients.agents, + recipients.editors, + recipients.carbonCopies, + recipients.certifiedDeliveries, + ] + const out: { name: string; email: string; status: string }[] = [] + for (const group of groups) { + if (!Array.isArray(group)) continue + for (const r of group) { + out.push({ + name: r.name?.trim() ?? '', + email: r.email?.trim() ?? '', + status: r.status?.trim() ?? '', + }) + } + } + return out +} + +/** + * Builds shared metadata used by both the stub and the full document (and by mapTags). + * + * Every field consumed by `mapTags` is read from the envelope object that the list + * endpoint (`GET /envelopes?from_date=...`) returns directly on each list entry — the + * list response carries the full envelope summary (status, subject, sender, and all + * lifecycle timestamps), not a thin stub. The sync engine calls `mapTags` on this stub + * metadata, so tags are populated without any per-envelope detail fetch. + */ +function buildMetadata(envelope: DocuSignEnvelope): Record { + const recipients = collectRecipients(envelope.recipients) + return { + status: envelope.status, + subject: envelope.emailSubject, + senderName: resolveSenderName(envelope), + senderEmail: resolveSenderEmail(envelope), + sentDate: envelope.sentDateTime, + completedDate: envelope.completedDateTime, + recipientNames: recipients.map((r) => r.name).filter(Boolean), + } +} + +/** + * Creates a lightweight document stub from an envelope list entry. No per-envelope API + * calls are made — content is fetched lazily by getDocument for new/changed envelopes only. + */ +function envelopeToStub(envelope: DocuSignEnvelope): ExternalDocument { + return { + externalId: envelope.envelopeId ?? '', + title: envelope.emailSubject?.trim() || 'Untitled DocuSign Envelope', + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(envelope.envelopeId), + contentHash: buildContentHash(envelope), + metadata: buildMetadata(envelope), + } +} + +/** + * Renders the envelope's text metadata into a plain-text document. Envelope documents are + * PDFs/binary and are intentionally NOT downloaded or text-extracted — only the document + * names are indexed alongside subject, status, sender, recipients, custom fields, and form + * data field name/value pairs. + */ +function buildContent( + envelope: DocuSignEnvelope, + customFields: DocuSignCustomField[], + documents: DocuSignDocument[], + formValues: DocuSignFormValue[] +): string { + const parts: string[] = [] + + if (envelope.emailSubject) parts.push(`Subject: ${envelope.emailSubject}`) + if (envelope.emailBlurb) parts.push(`Message: ${envelope.emailBlurb}`) + if (envelope.status) parts.push(`Status: ${envelope.status}`) + + const senderName = resolveSenderName(envelope) + const senderEmail = resolveSenderEmail(envelope) + if (senderName || senderEmail) { + parts.push(`Sender: ${[senderName, senderEmail].filter(Boolean).join(' ')}`.trim()) + } + if (envelope.sentDateTime) parts.push(`Sent: ${envelope.sentDateTime}`) + if (envelope.completedDateTime) parts.push(`Completed: ${envelope.completedDateTime}`) + + const recipients = collectRecipients(envelope.recipients) + if (recipients.length > 0) { + parts.push('') + parts.push('--- Recipients ---') + for (const r of recipients) { + const label = [r.name, r.email ? `<${r.email}>` : '', r.status ? `(${r.status})` : ''] + .filter(Boolean) + .join(' ') + if (label) parts.push(label) + } + } + + const fields = customFields.filter((f) => f.name?.trim()) + if (fields.length > 0) { + parts.push('') + parts.push('--- Custom Fields ---') + for (const f of fields) { + parts.push(`${f.name}: ${f.value ?? ''}`) + } + } + + const docNames = documents.map((d) => d.name?.trim()).filter((n): n is string => Boolean(n)) + if (docNames.length > 0) { + parts.push('') + parts.push('--- Documents ---') + for (const name of docNames) parts.push(name) + } + + const formPairs = formValues.filter((v) => v.name?.trim()) + if (formPairs.length > 0) { + parts.push('') + parts.push('--- Form Data ---') + for (const v of formPairs) { + parts.push(`${v.name}: ${v.value ?? ''}`) + } + } + + return parts.join('\n').trim() +} + +/** + * Fetches the envelope's form data (signer-entered tab values). Returns an empty list on + * 404 or any error — form data is supplementary and a missing endpoint must not fail the doc. + */ +async function fetchFormValues( + apiBase: string, + accessToken: string, + envelopeId: string +): Promise { + try { + const response = await fetchWithRetry(`${apiBase}/envelopes/${envelopeId}/form_data`, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + if (!response.ok) return [] + const data = (await response.json()) as DocuSignFormData + const values: DocuSignFormValue[] = [] + if (Array.isArray(data.formData)) values.push(...data.formData) + if (Array.isArray(data.recipientFormData)) { + for (const recipient of data.recipientFormData) { + if (Array.isArray(recipient.formData)) values.push(...recipient.formData) + } + } + return values + } catch (error) { + logger.warn('Failed to fetch DocuSign form data', { + envelopeId, + error: toError(error).message, + }) + return [] + } +} + +export const docusignConnector: ConnectorConfig = { + id: 'docusign', + name: 'DocuSign', + description: 'Sync envelope and agreement metadata from DocuSign into your knowledge base', + version: '1.0.0', + icon: DocuSignIcon, + + auth: { + mode: 'oauth', + provider: 'docusign', + requiredScopes: ['signature'], + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'lookback', + title: 'Date Range', + type: 'dropdown', + required: false, + options: [ + { label: 'Last 30 days', id: '30' }, + { label: 'Last 90 days (recommended)', id: '90' }, + { label: 'Last 6 months', id: '180' }, + ], + description: + 'On initial sync only. Filters envelopes by when their status last changed (from_date).', + }, + { + id: 'status', + title: 'Filter by Status', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. completed (or completed,sent)', + description: + 'Only sync envelopes with these statuses (comma-separated: created, sent, delivered, completed, declined, voided). Leave blank to sync all.', + }, + { + id: 'maxEnvelopes', + title: 'Max Envelopes', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const account = await resolveAccount(accessToken, syncContext) + const apiBase = apiBaseFor(account) + + const lookbackDays = computeLookbackDays(sourceConfig, lastSyncAt) + const maxEnvelopes = sourceConfig.maxEnvelopes + ? Number(sourceConfig.maxEnvelopes) + : DEFAULT_MAX_ENVELOPES + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + if (maxEnvelopes > 0 && prevFetched >= maxEnvelopes) { + return { documents: [], hasMore: false } + } + + const startPosition = cursor ? Number(cursor) : 0 + const cachedFromDate = syncContext?.docusignFromDate as string | undefined + const fromDate = cachedFromDate + ? new Date(cachedFromDate) + : new Date(Date.now() - lookbackDays * MS_PER_DAY) + if (syncContext && !cachedFromDate) syncContext.docusignFromDate = fromDate.toISOString() + + const queryParams = new URLSearchParams({ + from_date: formatFromDate(fromDate), + include: 'recipients,custom_fields', + count: String(MAX_PAGE_SIZE), + start_position: String(startPosition), + }) + const statusFilter = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : '' + if (statusFilter) queryParams.set('status', statusFilter) + + const url = `${apiBase}/envelopes?${queryParams.toString()}` + + logger.info('Listing DocuSign envelopes', { + from: formatFromDate(fromDate), + startPosition, + incremental: Boolean(lastSyncAt), + }) + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list DocuSign envelopes', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list DocuSign envelopes: ${response.status}`) + } + + const data = (await response.json()) as DocuSignEnvelopesListResponse + const envelopes = (data.envelopes ?? []).filter((e) => e.envelopeId) + const pageDocuments = envelopes.map(envelopeToStub) + + let documents = pageDocuments + if (maxEnvelopes > 0) { + const remaining = Math.max(0, maxEnvelopes - prevFetched) + if (pageDocuments.length > remaining) { + documents = pageDocuments.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + + const hitLimit = maxEnvelopes > 0 && totalFetched >= maxEnvelopes + if (hitLimit && syncContext) syncContext.listingCapped = true + + const endPosition = Number(data.endPosition) + const totalSetSize = Number(data.totalSetSize) + const hasNextPage = + pageDocuments.length === MAX_PAGE_SIZE && + Number.isFinite(endPosition) && + Number.isFinite(totalSetSize) && + endPosition + 1 < totalSetSize + + return { + documents, + nextCursor: !hitLimit && hasNextPage ? String(endPosition + 1) : undefined, + hasMore: !hitLimit && hasNextPage, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string, + syncContext?: Record + ): Promise => { + try { + if (!externalId) return null + + const account = await resolveAccount(accessToken, syncContext) + const apiBase = apiBaseFor(account) + + const response = await fetchWithRetry( + `${apiBase}/envelopes/${externalId}?include=recipients,custom_fields,documents`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + } + ) + + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch DocuSign envelope: ${response.status}`) + } + + const envelope = (await response.json()) as DocuSignEnvelope + if (!envelope.envelopeId) return null + + const customFields: DocuSignCustomField[] = [ + ...(envelope.customFields?.textCustomFields ?? []), + ...(envelope.customFields?.listCustomFields ?? []), + ] + + const documents = Array.isArray(envelope.envelopeDocuments) ? envelope.envelopeDocuments : [] + + const formValues = await fetchFormValues(apiBase, accessToken, externalId) + + const content = buildContent(envelope, customFields, documents, formValues) + if (!content.trim()) return null + + return { + externalId: envelope.envelopeId, + title: envelope.emailSubject?.trim() || 'Untitled DocuSign Envelope', + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(envelope.envelopeId), + contentHash: buildContentHash(envelope), + metadata: buildMetadata(envelope), + } + } catch (error) { + logger.warn('Failed to get DocuSign envelope', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxEnvelopes = sourceConfig.maxEnvelopes as string | undefined + if (maxEnvelopes && (Number.isNaN(Number(maxEnvelopes)) || Number(maxEnvelopes) < 0)) { + return { valid: false, error: 'Max envelopes must be a non-negative number' } + } + + try { + await resolveAccount(accessToken, undefined, VALIDATE_RETRY_OPTIONS) + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + /** + * Tag definitions are constrained by the document table's slot pools: 7 text slots but + * only 2 date slots (`date1`, `date2`). The two highest-value envelope dates — when it was + * sent and when it completed — claim both date slots. `createdDateTime` is intentionally + * NOT exposed as a date tag: it nearly always equals `sentDateTime` for sent envelopes, so + * adding it would consume a (non-existent) third date slot and be silently dropped by the + * slot allocator. `emailSubject` is exposed as a filterable text tag (distinct from the + * document title) since text slots are plentiful. + */ + tagDefinitions: [ + { id: 'status', displayName: 'Status', fieldType: 'text' }, + { id: 'sender', displayName: 'Sender', fieldType: 'text' }, + { id: 'subject', displayName: 'Subject', fieldType: 'text' }, + { id: 'sentDate', displayName: 'Sent Date', fieldType: 'date' }, + { id: 'completedDate', displayName: 'Completed Date', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.status === 'string' && metadata.status.trim()) { + result.status = metadata.status + } + + const sender = [metadata.senderName, metadata.senderEmail] + .filter((v): v is string => typeof v === 'string' && v.trim().length > 0) + .join(' ') + .trim() + if (sender) result.sender = sender + + if (typeof metadata.subject === 'string' && metadata.subject.trim()) { + result.subject = metadata.subject.trim() + } + + const sentDate = parseTagDate(metadata.sentDate) + if (sentDate) result.sentDate = sentDate + + const completedDate = parseTagDate(metadata.completedDate) + if (completedDate) result.completedDate = completedDate + + return result + }, +} diff --git a/apps/sim/connectors/docusign/index.ts b/apps/sim/connectors/docusign/index.ts new file mode 100644 index 00000000000..a59b7ba3653 --- /dev/null +++ b/apps/sim/connectors/docusign/index.ts @@ -0,0 +1 @@ +export { docusignConnector } from '@/connectors/docusign/docusign' diff --git a/apps/sim/connectors/fathom/fathom.ts b/apps/sim/connectors/fathom/fathom.ts new file mode 100644 index 00000000000..8c65ef43cd7 --- /dev/null +++ b/apps/sim/connectors/fathom/fathom.ts @@ -0,0 +1,605 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { FathomIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { parseTagDate } from '@/connectors/utils' + +const logger = createLogger('FathomConnector') + +const FATHOM_API_BASE = 'https://api.fathom.ai/external/v1' + +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +/** + * Days subtracted from `lastSyncAt` when computing the incremental `created_after` + * window. Fathom's list endpoint only filters by creation time (no update-based + * filter), so a meeting whose transcript was not yet ready on the sync that first + * saw it would otherwise never be re-listed. The overlap keeps recently-created + * meetings in the window long enough for late transcripts to be retried — the sync + * engine re-attempts meetings whose `getDocument` previously returned null, since + * those are never persisted. Matches the Gong connector's overlap approach. + */ +const INCREMENTAL_OVERLAP_DAYS = 14 + +/** + * Fathom authenticates external API requests with the `X-Api-Key` header. + * (The API also accepts `Authorization: Bearer` for OAuth-connected apps, but + * the api-key flow this connector uses requires `X-Api-Key`.) + */ +function buildHeaders(apiKey: string): Record { + return { + 'X-Api-Key': apiKey, + 'Content-Type': 'application/json', + } +} + +/** + * A meeting object as returned by `GET /meetings`. Only the fields this + * connector reads are typed; the API returns additional fields. + */ +interface FathomMeeting { + recording_id?: number + title?: string + meeting_title?: string | null + url?: string + share_url?: string + created_at?: string + scheduled_start_time?: string | null + scheduled_end_time?: string | null + recording_start_time?: string | null + recording_end_time?: string | null + transcript_language?: string + calendar_invitees_domains_type?: 'only_internal' | 'one_or_more_external' | null + recorded_by?: { + name?: string + email?: string + email_domain?: string + team?: string | null + } | null +} + +interface FathomMeetingsListResponse { + items?: FathomMeeting[] + next_cursor?: string | null +} + +/** + * A single transcript entry as returned by `GET /recordings/{id}/transcript`. + */ +interface FathomTranscriptEntry { + speaker?: { + display_name?: string + matched_calendar_invitee_email?: string | null + } + text?: string + timestamp?: string +} + +interface FathomTranscriptResponse { + transcript?: FathomTranscriptEntry[] +} + +interface FathomSummary { + template_name?: string | null + markdown_formatted?: string | null +} + +interface FathomSummaryResponse { + summary?: FathomSummary | null +} + +/** + * Header fields cached per recording during `listDocuments` so `getDocument` + * can render an identical document header. Fathom exposes no single-meeting + * GET and no `recording_ids` filter, so this metadata cannot be refetched once + * listing has moved past the page that contained it — it is carried forward in + * the shared `syncContext` instead. + */ +interface FathomMeetingHeader { + title: string + meetingDate?: string + durationSeconds?: number + recordedByEmail?: string + recordedByName?: string + team?: string + sourceUrl?: string + contentHash: string + metadata: FathomMeetingMetadata +} + +/** + * Metadata describing a Fathom meeting, attached to the listing stub. The sync + * engine merges this onto the hydrated document, so `getDocument` never needs + * to reproduce it. + */ +interface FathomMeetingMetadata { + recordingId?: string + recordedByEmail?: string + recordedByName?: string + team?: string + meetingType?: string + meetingDate?: string + durationSeconds?: number + transcriptLanguage?: string + title?: string +} + +/** + * Maps Fathom's `calendar_invitees_domains_type` to a human-readable meeting + * type. This is the only meeting-type signal the API exposes: `only_internal` + * means every invitee shares the recorder's domain; `one_or_more_external` + * means at least one external attendee (customer-facing). + */ +function resolveMeetingType(meeting: FathomMeeting): string | undefined { + switch (meeting.calendar_invitees_domains_type) { + case 'only_internal': + return 'internal' + case 'one_or_more_external': + return 'external' + default: + return undefined + } +} + +/** + * Computes the meeting duration in whole seconds from the recording window, + * or undefined when either bound is missing or unparseable. + */ +function computeDurationSeconds(meeting: FathomMeeting): number | undefined { + const start = meeting.recording_start_time ?? meeting.scheduled_start_time ?? undefined + const end = meeting.recording_end_time ?? meeting.scheduled_end_time ?? undefined + if (!start || !end) return undefined + const startMs = new Date(start).getTime() + const endMs = new Date(end).getTime() + if (Number.isNaN(startMs) || Number.isNaN(endMs) || endMs < startMs) return undefined + return Math.round((endMs - startMs) / 1000) +} + +/** + * Returns the best title for a meeting, falling back through the title fields. + */ +function resolveTitle(meeting: FathomMeeting): string { + const title = meeting.title?.trim() || meeting.meeting_title?.trim() + return title || 'Untitled Fathom Meeting' +} + +/** + * Extracts the connector metadata bag attached to a listing stub. + */ +function buildMetadata(meeting: FathomMeeting): FathomMeetingMetadata { + return { + recordingId: meeting.recording_id != null ? String(meeting.recording_id) : undefined, + recordedByEmail: meeting.recorded_by?.email, + recordedByName: meeting.recorded_by?.name, + team: meeting.recorded_by?.team ?? undefined, + meetingType: resolveMeetingType(meeting), + meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined, + durationSeconds: computeDurationSeconds(meeting), + transcriptLanguage: meeting.transcript_language, + title: resolveTitle(meeting), + } +} + +/** + * Extracts the lightweight header fields cached for `getDocument`. + */ +function buildHeader(meeting: FathomMeeting): FathomMeetingHeader { + return { + title: resolveTitle(meeting), + meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined, + durationSeconds: computeDurationSeconds(meeting), + recordedByEmail: meeting.recorded_by?.email, + recordedByName: meeting.recorded_by?.name, + team: meeting.recorded_by?.team ?? undefined, + sourceUrl: buildSourceUrl(meeting), + contentHash: buildContentHash(meeting), + metadata: buildMetadata(meeting), + } +} + +/** + * Builds a metadata-based content hash. Fathom recordings are immutable once + * processed, so the recording id plus its end/creation timestamps fully identify + * a version. The same value is cached in the header and returned by `getDocument`, + * so the stub and hydrated document hash identically. + */ +function buildContentHash(meeting: FathomMeeting): string { + return `fathom:${meeting.recording_id ?? ''}:${meeting.recording_end_time ?? ''}:${meeting.created_at ?? ''}` +} + +function buildSourceUrl(meeting: FathomMeeting): string | undefined { + return meeting.share_url || meeting.url || undefined +} + +/** + * Reads the cached header for a recording out of the shared sync context. + */ +function readCachedHeader( + syncContext: Record | undefined, + recordingId: string +): FathomMeetingHeader | undefined { + const cache = syncContext?.meetingHeaders as Record | undefined + return cache?.[recordingId] +} + +/** + * Stores the header for a recording in the shared sync context. + */ +function cacheHeader( + syncContext: Record | undefined, + recordingId: string, + header: FathomMeetingHeader +): void { + if (!syncContext) return + const cache = + (syncContext.meetingHeaders as Record | undefined) ?? {} + cache[recordingId] = header + syncContext.meetingHeaders = cache +} + +/** + * Formats the meeting header, optional summary, and transcript into a single + * plain-text document with one `Speaker: text` line per transcript entry. + */ +function formatMeetingContent( + header: FathomMeetingHeader | undefined, + transcript: FathomTranscriptEntry[], + summary: FathomSummary | null +): string { + const parts: string[] = [] + + parts.push(`Meeting: ${header?.title ?? 'Untitled Fathom Meeting'}`) + + if (header?.meetingDate) parts.push(`Date: ${header.meetingDate}`) + + if (header?.durationSeconds != null) { + parts.push(`Duration: ${Math.round(header.durationSeconds / 60)} minutes`) + } + + if (header?.recordedByEmail) { + parts.push(`Recorded by: ${header.recordedByName ?? header.recordedByEmail}`) + } + + if (header?.team) parts.push(`Team: ${header.team}`) + + if (summary?.markdown_formatted?.trim()) { + parts.push('') + parts.push('--- Summary ---') + parts.push(summary.markdown_formatted.trim()) + } + + if (transcript.length > 0) { + parts.push('') + parts.push('--- Transcript ---') + for (const entry of transcript) { + const speaker = entry.speaker?.display_name?.trim() || 'Unknown' + const text = entry.text?.trim() + if (text) parts.push(`${speaker}: ${text}`) + } + } + + return parts.join('\n') +} + +/** + * Converts a listing meeting into a deferred stub. Content is fetched lazily + * via `getDocument` only for new or changed meetings. + */ +function meetingToStub(meeting: FathomMeeting): ExternalDocument { + const metadata = buildMetadata(meeting) + return { + externalId: String(meeting.recording_id), + title: resolveTitle(meeting), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(meeting), + contentHash: buildContentHash(meeting), + metadata: { ...metadata }, + } +} + +export const fathomConnector: ConnectorConfig = { + id: 'fathom', + name: 'Fathom', + description: 'Sync meeting transcripts and summaries from Fathom', + version: '1.0.0', + icon: FathomIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your Fathom API key', + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'recordedBy', + title: 'Filter by Recorder Email', + type: 'short-input', + placeholder: 'e.g. john@example.com', + required: false, + description: 'Only sync meetings recorded by this email', + }, + { + id: 'teams', + title: 'Filter by Team', + type: 'short-input', + placeholder: 'e.g. Sales', + required: false, + description: 'Only sync meetings belonging to this team', + }, + { + id: 'meetingType', + title: 'Filter by Meeting Type', + type: 'dropdown', + mode: 'advanced', + required: false, + description: + 'Only sync internal meetings (everyone shares the recorder’s domain) or external meetings (at least one outside attendee). Leave as All to sync both.', + options: [ + { id: 'all', label: 'All meetings' }, + { id: 'one_or_more_external', label: 'External (customer-facing) only' }, + { id: 'only_internal', label: 'Internal only' }, + ], + }, + { + id: 'inviteeDomains', + title: 'Filter by Attendee Domain', + type: 'short-input', + mode: 'advanced', + placeholder: 'e.g. acme.com', + required: false, + description: + 'Only sync meetings that include a calendar invitee from this company email domain (exact match).', + }, + { + id: 'maxMeetings', + title: 'Max Meetings', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const recordedBy = (sourceConfig.recordedBy as string | undefined)?.trim() + const teams = (sourceConfig.teams as string | undefined)?.trim() + const meetingType = (sourceConfig.meetingType as string | undefined)?.trim() + const inviteeDomain = (sourceConfig.inviteeDomains as string | undefined)?.trim() + const maxMeetings = sourceConfig.maxMeetings ? Number(sourceConfig.maxMeetings) : 0 + + const url = new URL(`${FATHOM_API_BASE}/meetings`) + if (recordedBy) url.searchParams.append('recorded_by[]', recordedBy) + if (teams) url.searchParams.append('teams[]', teams) + if (meetingType && meetingType !== 'all') { + url.searchParams.append('calendar_invitees_domains_type', meetingType) + } + if (inviteeDomain) url.searchParams.append('calendar_invitees_domains[]', inviteeDomain) + if (cursor) url.searchParams.append('cursor', cursor) + if (lastSyncAt) { + const createdAfter = new Date(lastSyncAt.getTime() - INCREMENTAL_OVERLAP_DAYS * MS_PER_DAY) + url.searchParams.append('created_after', createdAfter.toISOString()) + } + + logger.info('Listing Fathom meetings', { + hasCursor: Boolean(cursor), + recordedBy, + teams, + meetingType, + inviteeDomain, + incremental: Boolean(lastSyncAt), + }) + + const response = await fetchWithRetry(url.toString(), { + method: 'GET', + headers: buildHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Fathom meetings', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Fathom meetings: ${response.status}`) + } + + const data = (await response.json()) as FathomMeetingsListResponse + const meetings = data.items ?? [] + const nextCursor = data.next_cursor?.trim() || undefined + + const allDocuments: ExternalDocument[] = [] + for (const meeting of meetings) { + if (meeting.recording_id == null) continue + const externalId = String(meeting.recording_id) + cacheHeader(syncContext, externalId, buildHeader(meeting)) + allDocuments.push(meetingToStub(meeting)) + } + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let documents = allDocuments + if (maxMeetings > 0) { + const remaining = Math.max(0, maxMeetings - prevFetched) + if (allDocuments.length > remaining) { + documents = allDocuments.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxMeetings > 0 && totalFetched >= maxMeetings + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && Boolean(nextCursor) + + return { + documents, + nextCursor: hasMore ? nextCursor : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string, + syncContext?: Record + ): Promise => { + try { + if (!externalId) return null + + const transcriptUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/transcript` + const transcriptResponse = await fetchWithRetry(transcriptUrl, { + method: 'GET', + headers: buildHeaders(accessToken), + }) + + if (!transcriptResponse.ok) { + if (transcriptResponse.status === 404) return null + throw new Error(`Failed to fetch Fathom transcript: ${transcriptResponse.status}`) + } + + const transcriptData = (await transcriptResponse.json()) as FathomTranscriptResponse + const transcript = transcriptData.transcript ?? [] + + let summary: FathomSummary | null = null + try { + const summaryUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/summary` + const summaryResponse = await fetchWithRetry(summaryUrl, { + method: 'GET', + headers: buildHeaders(accessToken), + }) + if (summaryResponse.ok) { + const summaryData = (await summaryResponse.json()) as FathomSummaryResponse + summary = summaryData.summary ?? null + } + } catch (summaryError) { + logger.warn('Failed to fetch Fathom summary', { + externalId, + error: toError(summaryError).message, + }) + } + + const hasTranscript = transcript.some((entry) => entry.text?.trim()) + const hasSummary = Boolean(summary?.markdown_formatted?.trim()) + if (!hasTranscript && !hasSummary) { + logger.info('No transcript or summary yet for Fathom meeting', { externalId }) + return null + } + + const header = readCachedHeader(syncContext, externalId) + const content = formatMeetingContent(header, transcript, summary).trim() + if (!content) return null + + return { + externalId, + title: header?.title ?? 'Untitled Fathom Meeting', + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: header?.sourceUrl, + contentHash: header?.contentHash ?? `fathom:${externalId}`, + metadata: { ...(header?.metadata ?? { recordingId: externalId }) }, + } + } catch (error) { + logger.warn('Failed to get Fathom meeting', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxMeetings = sourceConfig.maxMeetings as string | undefined + if (maxMeetings && (Number.isNaN(Number(maxMeetings)) || Number(maxMeetings) < 0)) { + return { valid: false, error: 'Max meetings must be a non-negative number' } + } + + try { + const response = await fetchWithRetry( + `${FATHOM_API_BASE}/meetings`, + { + method: 'GET', + headers: buildHeaders(accessToken), + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Fathom access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'title', displayName: 'Title', fieldType: 'text' }, + { id: 'recordedByEmail', displayName: 'Recorded By (Email)', fieldType: 'text' }, + { id: 'recordedByName', displayName: 'Recorded By (Name)', fieldType: 'text' }, + { id: 'team', displayName: 'Team', fieldType: 'text' }, + { id: 'meetingType', displayName: 'Meeting Type', fieldType: 'text' }, + { id: 'transcriptLanguage', displayName: 'Language', fieldType: 'text' }, + { id: 'durationSeconds', displayName: 'Duration (seconds)', fieldType: 'number' }, + { id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.title === 'string' && metadata.title.trim()) { + result.title = metadata.title + } + + if (typeof metadata.recordedByEmail === 'string' && metadata.recordedByEmail.trim()) { + result.recordedByEmail = metadata.recordedByEmail + } + + if (typeof metadata.recordedByName === 'string' && metadata.recordedByName.trim()) { + result.recordedByName = metadata.recordedByName + } + + if (typeof metadata.team === 'string' && metadata.team.trim()) { + result.team = metadata.team + } + + if (typeof metadata.meetingType === 'string' && metadata.meetingType.trim()) { + result.meetingType = metadata.meetingType + } + + if (typeof metadata.transcriptLanguage === 'string' && metadata.transcriptLanguage.trim()) { + result.transcriptLanguage = metadata.transcriptLanguage + } + + if (metadata.durationSeconds != null) { + const num = Number(metadata.durationSeconds) + if (!Number.isNaN(num)) result.durationSeconds = num + } + + const meetingDate = parseTagDate(metadata.meetingDate) + if (meetingDate) result.meetingDate = meetingDate + + return result + }, +} diff --git a/apps/sim/connectors/fathom/index.ts b/apps/sim/connectors/fathom/index.ts new file mode 100644 index 00000000000..c42b75eb028 --- /dev/null +++ b/apps/sim/connectors/fathom/index.ts @@ -0,0 +1 @@ +export { fathomConnector } from '@/connectors/fathom/fathom' diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts new file mode 100644 index 00000000000..f3fe3829585 --- /dev/null +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -0,0 +1,704 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { GitLabIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { computeContentHash, joinTagArray, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('GitLabConnector') + +const DEFAULT_HOST = 'gitlab.com' +const PAGE_SIZE = 100 + +/** + * Prefix encoded into each document's externalId so getDocument can route to the + * correct GitLab resource. Wiki pages are addressed by slug, issues by iid. + */ +const WIKI_PREFIX = 'wiki:' +const ISSUE_PREFIX = 'issue:' + +/** + * Selects which GitLab resources to sync. + */ +type ContentTypeChoice = 'wiki' | 'issues' | 'both' + +interface GitLabWikiPage { + slug: string + title?: string + format?: string + content?: string + encoding?: string +} + +interface GitLabUser { + username?: string + name?: string +} + +interface GitLabMilestone { + title?: string +} + +interface GitLabIssue { + iid: number + title?: string + description?: string | null + state?: string + labels?: string[] + author?: GitLabUser | null + assignees?: GitLabUser[] | null + milestone?: GitLabMilestone | null + updated_at?: string + created_at?: string + web_url?: string +} + +interface GitLabProject { + id: number + path_with_namespace?: string + web_url?: string + wiki_access_level?: string + wiki_enabled?: boolean +} + +/** + * Normalizes the host config value: trims whitespace, strips any protocol + * prefix and trailing slashes, and falls back to gitlab.com when empty. + */ +function normalizeHost(rawHost: unknown): string { + const host = typeof rawHost === 'string' ? rawHost.trim() : '' + if (!host) return DEFAULT_HOST + return host + .replace(/^https?:\/\//i, '') + .replace(/\/+$/, '') + .trim() +} + +/** + * Builds the REST API v4 base URL for the configured host. + */ +function buildApiBase(host: string): string { + return `https://${host}/api/v4` +} + +/** + * Returns the encoded project identifier (numeric ID or URL-encoded path). + * GitLab accepts a numeric ID or the URL-encoded `group/project` path. + */ +function encodeProjectId(project: unknown): string { + return encodeURIComponent(String(project ?? '').trim()) +} + +/** + * Reads the parsed content-type choice from sourceConfig (defaults to 'both'). + */ +function getContentTypeChoice(sourceConfig: Record): ContentTypeChoice { + const value = typeof sourceConfig.contentTypes === 'string' ? sourceConfig.contentTypes : 'both' + if (value === 'wiki' || value === 'issues') return value + return 'both' +} + +/** + * Standard request headers carrying the Personal Access Token. + */ +function authHeaders(accessToken: string): Record { + return { + 'PRIVATE-TOKEN': accessToken, + Accept: 'application/json', + } +} + +/** + * Builds the change-detection hash for a wiki page. + * + * GitLab wiki pages expose no version number or `updated_at` timestamp in the + * REST API, so there is no metadata field that reliably changes when a page is + * edited. As a last resort we hash the page content itself. To keep the hash + * identical between the listing stub and getDocument, both paths request the + * page content (the list endpoint supports `with_content=1`) and feed it through + * this same function. + */ +async function buildWikiContentHash( + projectId: string, + slug: string, + content: string +): Promise { + const contentDigest = await computeContentHash(content) + return `gitlab:wiki:${projectId}:${slug}:${contentDigest}` +} + +/** + * Builds the change-detection hash for an issue. Issues expose `updated_at`, + * which increments on every edit, comment, or state change — an ideal metadata + * indicator that requires no content fetch. + */ +function buildIssueContentHash(projectId: string, iid: number, updatedAt: string): string { + return `gitlab:issue:${projectId}:${iid}:${updatedAt}` +} + +/** + * Composes the document body as "Title\n\n". + */ +function composeBody(title: string, content: string): string { + const trimmedTitle = title.trim() + const trimmedContent = content.trim() + if (!trimmedTitle) return trimmedContent + if (!trimmedContent) return trimmedTitle + return `${trimmedTitle}\n\n${trimmedContent}` +} + +/** + * Builds a wiki page document (full content) from a fetched page. + */ +async function wikiPageToDocument( + apiBase: string, + encodedProject: string, + host: string, + projectPath: string, + page: GitLabWikiPage +): Promise { + const content = typeof page.content === 'string' ? page.content : '' + const title = page.title?.trim() || page.slug + const body = composeBody(title, content) + if (!body.trim()) return null + + const contentHash = await buildWikiContentHash(encodedProject, page.slug, content) + + return { + externalId: `${WIKI_PREFIX}${page.slug}`, + title, + content: body, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: projectPath + ? `https://${host}/${projectPath}/-/wikis/${page.slug}` + : `${apiBase}/projects/${encodedProject}/wikis/${page.slug}`, + contentHash, + metadata: { + contentType: 'wiki', + title, + slug: page.slug, + }, + } +} + +/** + * Builds an issue document from a fetched issue. + */ +function issueToDocument( + encodedProject: string, + host: string, + projectPath: string, + issue: GitLabIssue +): ExternalDocument | null { + const title = issue.title?.trim() || `Issue #${issue.iid}` + const description = typeof issue.description === 'string' ? issue.description : '' + const body = composeBody(title, description) + if (!body.trim()) return null + + const updatedAt = issue.updated_at ?? issue.created_at ?? '' + const createdAt = issue.created_at ?? '' + const author = issue.author?.username?.trim() || issue.author?.name?.trim() || '' + const labels = Array.isArray(issue.labels) ? issue.labels : [] + const milestone = issue.milestone?.title?.trim() || '' + + const fallbackUrl = projectPath + ? `https://${host}/${projectPath}/-/issues/${issue.iid}` + : undefined + + return { + externalId: `${ISSUE_PREFIX}${issue.iid}`, + title, + content: body, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: issue.web_url || fallbackUrl, + contentHash: buildIssueContentHash(encodedProject, issue.iid, updatedAt), + metadata: { + contentType: 'issue', + title, + iid: issue.iid, + state: issue.state, + author, + labels, + milestone, + createdAt, + updatedAt, + }, + } +} + +/** + * Fetches the project record, used to resolve the human-readable path for + * source URLs and to confirm access during validation. + */ +async function fetchProject( + apiBase: string, + encodedProject: string, + accessToken: string, + retryOptions?: typeof VALIDATE_RETRY_OPTIONS +): Promise { + return fetchWithRetry( + `${apiBase}/projects/${encodedProject}`, + { method: 'GET', headers: authHeaders(accessToken) }, + retryOptions + ) +} + +/** + * Encodes the listing cursor. The cursor packs the resource phase (wiki ➜ issues) + * and the issues page number so a single sync walks wikis first, then paginates + * issues via the X-Next-Page header. + */ +interface CursorState { + phase: 'wiki' | 'issues' + issuePage: number +} + +function encodeCursor(state: CursorState): string { + return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url') +} + +function decodeCursor(cursor: string | undefined, initialPhase: 'wiki' | 'issues'): CursorState { + if (!cursor) return { phase: initialPhase, issuePage: 1 } + try { + const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial<{ + phase: 'wiki' | 'issues' + issuePage: number + }> + return { + phase: parsed.phase === 'issues' ? 'issues' : 'wiki', + issuePage: Number(parsed.issuePage) > 0 ? Number(parsed.issuePage) : 1, + } + } catch { + return { phase: initialPhase, issuePage: 1 } + } +} + +/** + * Applies the optional maxItems cap to a batch, tracking the running total in + * syncContext and flagging `listingCapped` when the cap is hit. + */ +function applyMaxItemsCap( + documents: ExternalDocument[], + maxItems: number, + syncContext: Record | undefined +): { documents: ExternalDocument[]; capped: boolean } { + if (maxItems <= 0) return { documents, capped: false } + const prevTotal = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = Math.max(0, maxItems - prevTotal) + const sliced = documents.length > remaining ? documents.slice(0, remaining) : documents + const newTotal = prevTotal + sliced.length + if (syncContext) syncContext.totalDocsFetched = newTotal + const capped = newTotal >= maxItems + if (capped && syncContext) syncContext.listingCapped = true + return { documents: sliced, capped } +} + +export const gitlabConnector: ConnectorConfig = { + id: 'gitlab', + name: 'GitLab', + description: 'Sync wiki pages and issues from a GitLab project into your knowledge base', + version: '1.0.0', + icon: GitLabIcon, + + auth: { + mode: 'apiKey', + label: 'Personal Access Token', + placeholder: 'Enter your GitLab PAT', + }, + + /** + * Incremental sync applies to issues only (via the `updated_after` filter + * derived from lastSyncAt). Wikis lack a change timestamp, so they are always + * re-listed in full and reconciled by content hash. + */ + supportsIncrementalSync: true, + + configFields: [ + { + id: 'host', + title: 'Host', + type: 'short-input', + placeholder: 'gitlab.com', + required: false, + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + }, + { + id: 'project', + title: 'Project', + type: 'short-input', + placeholder: 'group/project or numeric ID', + required: true, + description: 'Project path (e.g. my-group/my-repo) or numeric project ID.', + }, + { + id: 'contentTypes', + title: 'Content', + type: 'dropdown', + required: false, + options: [ + { label: 'Wiki only', id: 'wiki' }, + { label: 'Issues only', id: 'issues' }, + { label: 'Both', id: 'both' }, + ], + }, + { + id: 'issueState', + title: 'Issue State', + type: 'dropdown', + required: false, + mode: 'advanced', + options: [ + { label: 'All', id: 'all' }, + { label: 'Open only', id: 'opened' }, + { label: 'Closed only', id: 'closed' }, + ], + description: 'Which issues to sync by state. Applies only when syncing issues.', + }, + { + id: 'issueLabels', + title: 'Issue Labels', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. bug,docs (comma-separated)', + description: + 'Only sync issues with all of these labels (comma-separated). Applies only when syncing issues.', + }, + { + id: 'issueMilestone', + title: 'Issue Milestone', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. v1.0 (milestone title)', + description: + 'Only sync issues assigned to this milestone (exact title). Applies only when syncing issues.', + }, + { + id: 'maxItems', + title: 'Max Items', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const host = normalizeHost(sourceConfig.host) + const apiBase = buildApiBase(host) + const encodedProject = encodeProjectId(sourceConfig.project) + const choice = getContentTypeChoice(sourceConfig) + const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0 + + const wantsWiki = choice === 'wiki' || choice === 'both' + const wantsIssues = choice === 'issues' || choice === 'both' + + if (!encodedProject) { + throw new Error('Project is required') + } + + let projectPath = (syncContext?.projectPath as string) ?? '' + if (!projectPath && syncContext) { + const projectResponse = await fetchProject(apiBase, encodedProject, accessToken) + if (projectResponse.ok) { + const project = (await projectResponse.json()) as GitLabProject + projectPath = project.path_with_namespace ?? '' + syncContext.projectPath = projectPath + } + } + + const initialPhase: 'wiki' | 'issues' = wantsWiki ? 'wiki' : 'issues' + const state = decodeCursor(cursor, initialPhase) + + if (state.phase === 'wiki' && wantsWiki) { + const url = `${apiBase}/projects/${encodedProject}/wikis?with_content=1` + logger.info('Listing GitLab wiki pages', { host, project: encodedProject }) + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list GitLab wiki pages', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list GitLab wiki pages: ${response.status}`) + } + + const pages = (await response.json()) as GitLabWikiPage[] + const documents: ExternalDocument[] = [] + for (const page of pages) { + if (!page.slug) continue + const doc = await wikiPageToDocument(apiBase, encodedProject, host, projectPath, page) + if (doc) documents.push(doc) + } + + const { documents: capped, capped: hitLimit } = applyMaxItemsCap( + documents, + maxItems, + syncContext + ) + + if (hitLimit || !wantsIssues) { + return { documents: capped, hasMore: false } + } + + return { + documents: capped, + nextCursor: encodeCursor({ phase: 'issues', issuePage: 1 }), + hasMore: true, + } + } + + if (wantsIssues) { + const params = new URLSearchParams({ + per_page: String(PAGE_SIZE), + page: String(state.issuePage), + order_by: 'updated_at', + sort: 'desc', + }) + if (lastSyncAt) params.set('updated_after', lastSyncAt.toISOString()) + const issueState = + typeof sourceConfig.issueState === 'string' ? sourceConfig.issueState.trim() : '' + if (issueState && issueState !== 'all') params.set('state', issueState) + const issueLabels = + typeof sourceConfig.issueLabels === 'string' ? sourceConfig.issueLabels.trim() : '' + if (issueLabels) params.set('labels', issueLabels) + const issueMilestone = + typeof sourceConfig.issueMilestone === 'string' ? sourceConfig.issueMilestone.trim() : '' + if (issueMilestone) params.set('milestone', issueMilestone) + + const url = `${apiBase}/projects/${encodedProject}/issues?${params.toString()}` + logger.info('Listing GitLab issues', { + host, + project: encodedProject, + page: state.issuePage, + incremental: Boolean(lastSyncAt), + }) + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list GitLab issues', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list GitLab issues: ${response.status}`) + } + + const issues = (await response.json()) as GitLabIssue[] + const documents: ExternalDocument[] = [] + for (const issue of issues) { + if (issue.iid == null) continue + const doc = issueToDocument(encodedProject, host, projectPath, issue) + if (doc) documents.push(doc) + } + + const { documents: capped, capped: hitLimit } = applyMaxItemsCap( + documents, + maxItems, + syncContext + ) + + const nextPageHeader = response.headers.get('x-next-page')?.trim() + const nextPage = nextPageHeader ? Number(nextPageHeader) : 0 + const hasMorePages = !hitLimit && Number.isFinite(nextPage) && nextPage > 0 + + return { + documents: capped, + nextCursor: hasMorePages + ? encodeCursor({ phase: 'issues', issuePage: nextPage }) + : undefined, + hasMore: hasMorePages, + } + } + + return { documents: [], hasMore: false } + }, + + getDocument: async ( + accessToken: string, + sourceConfig: Record, + externalId: string, + syncContext?: Record + ): Promise => { + const host = normalizeHost(sourceConfig.host) + const apiBase = buildApiBase(host) + const encodedProject = encodeProjectId(sourceConfig.project) + if (!encodedProject || !externalId) return null + + const projectPath = (syncContext?.projectPath as string) ?? '' + + try { + if (externalId.startsWith(WIKI_PREFIX)) { + const slug = externalId.slice(WIKI_PREFIX.length) + if (!slug) return null + + const url = `${apiBase}/projects/${encodedProject}/wikis/${encodeURIComponent(slug)}?render_html=false` + const response = await fetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) + + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch GitLab wiki page: ${response.status}`) + } + + const page = (await response.json()) as GitLabWikiPage + if (!page.slug) return null + return wikiPageToDocument(apiBase, encodedProject, host, projectPath, page) + } + + if (externalId.startsWith(ISSUE_PREFIX)) { + const iidStr = externalId.slice(ISSUE_PREFIX.length) + const iid = Number(iidStr) + if (!iidStr || Number.isNaN(iid)) return null + + const url = `${apiBase}/projects/${encodedProject}/issues/${iid}` + const response = await fetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) + + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch GitLab issue: ${response.status}`) + } + + const issue = (await response.json()) as GitLabIssue + if (issue.iid == null) return null + return issueToDocument(encodedProject, host, projectPath, issue) + } + + return null + } catch (error) { + logger.warn(`Failed to fetch GitLab document ${externalId}`, { + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const project = (sourceConfig.project as string)?.trim() + if (!project) { + return { valid: false, error: 'Project is required' } + } + + const maxItems = sourceConfig.maxItems as string | undefined + if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) { + return { valid: false, error: 'Max items must be a positive number' } + } + + const host = normalizeHost(sourceConfig.host) + const apiBase = buildApiBase(host) + const encodedProject = encodeProjectId(project) + const choice = getContentTypeChoice(sourceConfig) + + try { + const response = await fetchProject( + apiBase, + encodedProject, + accessToken, + VALIDATE_RETRY_OPTIONS + ) + + if (response.status === 404) { + return { valid: false, error: `Project "${project}" not found on ${host}` } + } + if (response.status === 401 || response.status === 403) { + return { valid: false, error: 'Invalid token or insufficient permissions' } + } + if (!response.ok) { + return { valid: false, error: `Cannot access project: ${response.status}` } + } + + const projectRecord = (await response.json()) as GitLabProject + + if (choice === 'wiki' || choice === 'both') { + const accessLevel = projectRecord.wiki_access_level + const enabled = + accessLevel != null ? accessLevel !== 'disabled' : projectRecord.wiki_enabled !== false + if (!enabled) { + if (choice === 'wiki') { + return { valid: false, error: 'The wiki feature is disabled for this project' } + } + logger.warn('Wiki feature disabled; only issues will sync', { project }) + } + } + + return { valid: true } + } catch (error) { + return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') } + } + }, + + tagDefinitions: [ + { id: 'contentType', displayName: 'Content Type', fieldType: 'text' }, + { id: 'title', displayName: 'Title', fieldType: 'text' }, + { id: 'state', displayName: 'State', fieldType: 'text' }, + { id: 'author', displayName: 'Author', fieldType: 'text' }, + { id: 'labels', displayName: 'Labels', fieldType: 'text' }, + { id: 'milestone', displayName: 'Milestone', fieldType: 'text' }, + { id: 'createdAt', displayName: 'Created At', fieldType: 'date' }, + { id: 'updatedAt', displayName: 'Updated At', fieldType: 'date' }, + ], + + /** + * Maps document metadata to tag slots. The `contentType` and `title` tags + * apply to both wikis and issues. The remaining tags (state, author, labels, + * milestone, createdAt, updatedAt) are issue-only — wiki pages expose none of + * them in the REST API, so wiki documents leave those metadata fields empty + * and the type/empty guards below skip them. + */ + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.contentType === 'string' && metadata.contentType.trim()) { + result.contentType = metadata.contentType + } + if (typeof metadata.title === 'string' && metadata.title.trim()) { + result.title = metadata.title + } + if (typeof metadata.state === 'string' && metadata.state.trim()) { + result.state = metadata.state + } + if (typeof metadata.author === 'string' && metadata.author.trim()) { + result.author = metadata.author + } + + const labels = joinTagArray(metadata.labels) + if (labels) result.labels = labels + + if (typeof metadata.milestone === 'string' && metadata.milestone.trim()) { + result.milestone = metadata.milestone + } + + const createdAt = parseTagDate(metadata.createdAt) + if (createdAt) result.createdAt = createdAt + + const updatedAt = parseTagDate(metadata.updatedAt) + if (updatedAt) result.updatedAt = updatedAt + + return result + }, +} diff --git a/apps/sim/connectors/gitlab/index.ts b/apps/sim/connectors/gitlab/index.ts new file mode 100644 index 00000000000..7077b51f798 --- /dev/null +++ b/apps/sim/connectors/gitlab/index.ts @@ -0,0 +1 @@ +export { gitlabConnector } from '@/connectors/gitlab/gitlab' diff --git a/apps/sim/connectors/gong/gong.ts b/apps/sim/connectors/gong/gong.ts new file mode 100644 index 00000000000..beb7391573c --- /dev/null +++ b/apps/sim/connectors/gong/gong.ts @@ -0,0 +1,594 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { GongIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { parseTagDate } from '@/connectors/utils' + +const logger = createLogger('GongConnector') + +const GONG_API_BASE = 'https://api.gong.io/v2' +const DEFAULT_LOOKBACK_DAYS = 90 +const MAX_LOOKBACK_DAYS = 180 +/** + * Days of overlap added when computing the incremental sync window. Gong call data + * (parties, transcript) can finish processing minutes to hours — occasionally a + * day or two — after a call ends. The sync engine re-attempts calls whose + * transcript was not yet ready (a null getDocument result is never persisted, so + * the call is re-listed and re-fetched on the next sync), but only while the call + * stays inside the incremental window. A two-week overlap keeps recently-ended + * calls in that window long enough for late transcripts to be picked up, at the + * cost of re-listing already-synced calls (skipped downstream by content hash). + */ +const INCREMENTAL_OVERLAP_DAYS = 14 +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +/** + * Metadata for a single call participant. `speakerId` cross-references the + * `speakerId` field on transcript monologues, letting the connector attribute + * each spoken line to a named participant. + */ +interface GongParty { + id?: string + name?: string + emailAddress?: string + speakerId?: string + affiliation?: string +} + +/** + * Core call metadata returned by the extensive calls endpoint. Mirrors Gong's + * `CallBasicData` (the `metaData` object) — every field here is present on the + * `/v2/calls/extensive` stub response and never requires the transcript fetch. + */ +interface GongCallMetaData { + id?: string + title?: string + scheduled?: string + started?: string + duration?: number + url?: string + workspaceId?: string + primaryUserId?: string + direction?: string + scope?: string + system?: string + language?: string + purpose?: string + isPrivate?: boolean +} + +/** + * A single call object from POST /v2/calls/extensive. + */ +interface GongExtensiveCall { + metaData?: GongCallMetaData + parties?: GongParty[] +} + +interface GongRecords { + cursor?: string + totalRecords?: number + currentPageSize?: number +} + +interface GongExtensiveCallsResponse { + calls?: GongExtensiveCall[] + records?: GongRecords +} + +/** + * A single sentence within a transcript monologue. Gong returns timing in + * `startMs`/`endMs`; only `text` is used for the formatted transcript. + */ +interface GongTranscriptSentence { + text?: string +} + +/** + * A monologue (one speaker turn) within a call transcript. + */ +interface GongMonologue { + speakerId?: string + topic?: string + sentences?: GongTranscriptSentence[] +} + +interface GongCallTranscript { + callId?: string + transcript?: GongMonologue[] +} + +interface GongTranscriptResponse { + callTranscripts?: GongCallTranscript[] + records?: GongRecords +} + +/** + * Builds the Authorization header value for Gong's Basic auth scheme. + * + * Gong authenticates with `Basic base64(accessKey:accessKeySecret)`. The sync + * engine passes the user's stored key as `accessToken`. To support both raw + * `accessKey:accessKeySecret` pairs and pre-encoded credentials, the raw form + * (containing a colon) is base64-encoded here; an already-encoded value is sent + * as-is. + */ +function buildAuthHeader(accessToken: string): string { + const token = accessToken.includes(':') + ? Buffer.from(accessToken, 'utf8').toString('base64') + : accessToken + return `Basic ${token}` +} + +function buildHeaders(accessToken: string): Record { + return { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: buildAuthHeader(accessToken), + } +} + +/** + * Parses a comma- or newline-separated list of Gong IDs into a trimmed, + * de-duplicated, non-empty array. Returns `undefined` when nothing usable + * remains so the caller can omit the filter key entirely. + */ +function parseIdList(raw: unknown): string[] | undefined { + if (typeof raw !== 'string') return undefined + const ids = Array.from( + new Set( + raw + .split(/[\n,]/) + .map((value) => value.trim()) + .filter((value) => value.length > 0) + ) + ) + return ids.length > 0 ? ids : undefined +} + +/** + * Metadata-based content hash shared by `listDocuments` stubs and `getDocument` + * results. Derived purely from call identity and its start time so the value is + * identical across both paths — guaranteeing the sync engine only re-fetches a + * transcript when the call's metadata actually changes. + */ +function buildContentHash(callId: string, started: string | undefined): string { + return `gong:${callId}:${started ?? ''}` +} + +function buildCallTitle(metaData: GongCallMetaData | undefined): string { + return metaData?.title?.trim() || 'Untitled Gong Call' +} + +/** + * Extracts named participant labels from a call's parties for tag mapping and + * the transcript header. + */ +function buildParticipantNames(parties: GongParty[] | undefined): string[] { + if (!parties) return [] + const names: string[] = [] + for (const party of parties) { + const label = party.name?.trim() || party.emailAddress?.trim() + if (label) names.push(label) + } + return names +} + +/** + * Builds a `speakerId` → display-name map from a call's parties so transcript + * monologues (keyed by `speakerId`) can be attributed to a named speaker. + */ +function buildSpeakerMap(parties: GongParty[] | undefined): Record { + const map: Record = {} + if (!parties) return map + for (const party of parties) { + if (!party.speakerId) continue + const label = party.name?.trim() || party.emailAddress?.trim() + if (label) map[party.speakerId] = label + } + return map +} + +function buildMetadata( + metaData: GongCallMetaData | undefined, + participants: string[] +): Record { + return { + callId: metaData?.id, + callTitle: metaData?.title, + callDate: metaData?.started, + scheduledDate: metaData?.scheduled, + duration: metaData?.duration, + workspaceId: metaData?.workspaceId, + primaryUserId: metaData?.primaryUserId, + direction: metaData?.direction, + scope: metaData?.scope, + system: metaData?.system, + language: metaData?.language, + purpose: metaData?.purpose, + isPrivate: metaData?.isPrivate, + participants, + } +} + +/** + * Formats a call's transcript into speaker-attributed plain text with a header + * describing the call (title, date, duration, participants). + */ +function formatTranscriptContent( + metaData: GongCallMetaData | undefined, + participants: string[], + speakerMap: Record, + monologues: GongMonologue[] +): string { + const parts: string[] = [] + + parts.push(`Call: ${buildCallTitle(metaData)}`) + if (metaData?.started) parts.push(`Date: ${metaData.started}`) + if (metaData?.duration != null) { + const minutes = Math.round(metaData.duration / 60) + parts.push(`Duration: ${minutes} minutes`) + } + if (participants.length > 0) parts.push(`Participants: ${participants.join(', ')}`) + + parts.push('') + parts.push('--- Transcript ---') + + for (const monologue of monologues) { + const speaker = (monologue.speakerId && speakerMap[monologue.speakerId]) || 'Unknown Speaker' + const text = (monologue.sentences ?? []) + .map((sentence) => sentence.text?.trim()) + .filter((value): value is string => Boolean(value)) + .join(' ') + if (text) parts.push(`${speaker}: ${text}`) + } + + return parts.join('\n') +} + +/** + * Computes the effective lookback window in days, narrowing to the time since + * the last successful sync (plus an overlap to catch transcripts that finished + * processing late) when incremental sync is active. + */ +function computeLookbackDays( + sourceConfig: Record, + lastSyncAt: Date | undefined +): number { + const raw = sourceConfig.lookback as string | undefined + const configured = Number(raw) + const baseline = + Number.isFinite(configured) && configured > 0 + ? Math.min(Math.floor(configured), MAX_LOOKBACK_DAYS) + : DEFAULT_LOOKBACK_DAYS + + if (!lastSyncAt) return baseline + + const sinceLastSync = Math.ceil((Date.now() - lastSyncAt.getTime()) / MS_PER_DAY) + const incremental = Math.max(sinceLastSync + INCREMENTAL_OVERLAP_DAYS, INCREMENTAL_OVERLAP_DAYS) + return Math.min(incremental, baseline) +} + +/** + * Fetches a single page of calls from POST /v2/calls/extensive with parties + * exposed (needed to resolve transcript speaker IDs to names). + */ +async function fetchExtensiveCalls( + accessToken: string, + filter: Record, + cursor: string | undefined, + retryOptions?: Parameters[2] +): Promise { + const body: Record = { + filter, + contentSelector: { exposedFields: { parties: true } }, + } + if (cursor) body.cursor = cursor + + const response = await fetchWithRetry( + `${GONG_API_BASE}/calls/extensive`, + { + method: 'POST', + headers: buildHeaders(accessToken), + body: JSON.stringify(body), + }, + retryOptions + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error( + `Failed to list Gong calls: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}` + ) + } + + return (await response.json()) as GongExtensiveCallsResponse +} + +export const gongConnector: ConnectorConfig = { + id: 'gong', + name: 'Gong', + description: 'Sync call transcripts from Gong revenue intelligence', + version: '1.0.0', + icon: GongIcon, + + auth: { + mode: 'apiKey', + label: 'Access Key & Secret', + placeholder: 'accessKey:accessKeySecret', + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'lookback', + title: 'Date Range', + type: 'dropdown', + required: false, + options: [ + { label: 'Last 30 days', id: '30' }, + { label: 'Last 90 days (recommended)', id: '90' }, + { label: 'Last 6 months', id: '180' }, + ], + description: + 'On initial sync only. Controls how far back to look for calls with transcripts.', + }, + { + id: 'maxCalls', + title: 'Max Calls', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + }, + { + id: 'workspaceId', + title: 'Workspace ID', + type: 'short-input', + required: false, + placeholder: 'Optional — limit to a single Gong workspace', + }, + { + id: 'primaryUserIds', + title: 'Host User IDs', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'Optional — comma-separated Gong user IDs (call hosts)', + description: + 'Only sync calls hosted by these users. Find IDs in Gong under Company Settings → Users, or via the API.', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const lookbackDays = computeLookbackDays(sourceConfig, lastSyncAt) + const maxCalls = sourceConfig.maxCalls ? Number(sourceConfig.maxCalls) : 0 + const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim() + const primaryUserIds = parseIdList(sourceConfig.primaryUserIds) + + const cachedWindow = syncContext?.gongDateWindow as + | { fromDateTime: string; toDateTime: string } + | undefined + const now = new Date() + const window = cachedWindow ?? { + fromDateTime: new Date(now.getTime() - lookbackDays * MS_PER_DAY).toISOString(), + toDateTime: now.toISOString(), + } + if (syncContext && !cachedWindow) syncContext.gongDateWindow = window + const { fromDateTime, toDateTime } = window + + const filter: Record = { fromDateTime, toDateTime } + if (workspaceId) filter.workspaceId = workspaceId + if (primaryUserIds) filter.primaryUserIds = primaryUserIds + + logger.info('Listing Gong calls', { + fromDateTime, + toDateTime, + hasCursor: Boolean(cursor), + incremental: Boolean(lastSyncAt), + }) + + const data = await fetchExtensiveCalls(accessToken, filter, cursor) + const calls = data.calls ?? [] + const nextPageCursor = data.records?.cursor?.trim() || undefined + + const allDocuments: ExternalDocument[] = [] + for (const call of calls) { + const callId = call.metaData?.id + if (!callId) continue + const participants = buildParticipantNames(call.parties) + allDocuments.push({ + externalId: callId, + title: buildCallTitle(call.metaData), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: call.metaData?.url || undefined, + contentHash: buildContentHash(callId, call.metaData?.started), + metadata: buildMetadata(call.metaData, participants), + }) + } + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let documents = allDocuments + if (maxCalls > 0) { + const remaining = Math.max(0, maxCalls - prevFetched) + if (allDocuments.length > remaining) { + documents = allDocuments.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxCalls > 0 && totalFetched >= maxCalls + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && Boolean(nextPageCursor) + + return { + documents, + nextCursor: hasMore ? nextPageCursor : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim() + const filter: Record = { callIds: [externalId] } + if (workspaceId) filter.workspaceId = workspaceId + + const callData = await fetchExtensiveCalls(accessToken, filter, undefined) + const call = callData.calls?.[0] + if (!call?.metaData?.id) { + logger.warn('Gong call not found', { externalId }) + return null + } + + const metaData = call.metaData + const participants = buildParticipantNames(call.parties) + const speakerMap = buildSpeakerMap(call.parties) + + const transcriptResponse = await fetchWithRetry(`${GONG_API_BASE}/calls/transcript`, { + method: 'POST', + headers: buildHeaders(accessToken), + body: JSON.stringify({ filter: { callIds: [externalId] } }), + }) + + if (!transcriptResponse.ok) { + if (transcriptResponse.status === 404) return null + throw new Error(`Failed to fetch Gong transcript: ${transcriptResponse.status}`) + } + + const transcriptData = (await transcriptResponse.json()) as GongTranscriptResponse + const callTranscript = transcriptData.callTranscripts?.find( + (entry) => entry.callId === externalId + ) + const monologues = callTranscript?.transcript ?? [] + if (monologues.length === 0) { + logger.info('Transcript not available for Gong call', { externalId }) + return null + } + + const hasSpokenText = monologues.some((monologue) => + (monologue.sentences ?? []).some((sentence) => Boolean(sentence.text?.trim())) + ) + if (!hasSpokenText) return null + + const content = formatTranscriptContent(metaData, participants, speakerMap, monologues) + + return { + externalId: metaData.id ?? externalId, + title: buildCallTitle(metaData), + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: metaData.url || undefined, + contentHash: buildContentHash(metaData.id ?? externalId, metaData.started), + metadata: buildMetadata(metaData, participants), + } + } catch (error) { + logger.warn('Failed to get Gong call transcript', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxCalls = sourceConfig.maxCalls as string | undefined + if (maxCalls && (Number.isNaN(Number(maxCalls)) || Number(maxCalls) < 0)) { + return { valid: false, error: 'Max calls must be a non-negative number' } + } + + try { + const response = await fetchWithRetry( + `${GONG_API_BASE}/users`, + { + method: 'GET', + headers: buildHeaders(accessToken), + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Gong access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'callTitle', displayName: 'Call Title', fieldType: 'text' }, + { id: 'participants', displayName: 'Participants', fieldType: 'text' }, + { id: 'duration', displayName: 'Duration (seconds)', fieldType: 'number' }, + { id: 'callDate', displayName: 'Call Date', fieldType: 'date' }, + { id: 'scheduledDate', displayName: 'Scheduled Date', fieldType: 'date' }, + { id: 'direction', displayName: 'Direction', fieldType: 'text' }, + { id: 'scope', displayName: 'Scope', fieldType: 'text' }, + { id: 'system', displayName: 'System', fieldType: 'text' }, + { id: 'language', displayName: 'Language', fieldType: 'text' }, + { id: 'purpose', displayName: 'Purpose', fieldType: 'text' }, + { id: 'isPrivate', displayName: 'Private', fieldType: 'boolean' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.callTitle === 'string' && metadata.callTitle.trim()) { + result.callTitle = metadata.callTitle + } + + const participants = Array.isArray(metadata.participants) + ? (metadata.participants as string[]) + : [] + if (participants.length > 0) { + result.participants = participants.join(', ') + } + + if (metadata.duration != null) { + const num = Number(metadata.duration) + if (!Number.isNaN(num)) result.duration = num + } + + const callDate = parseTagDate(metadata.callDate) + if (callDate) result.callDate = callDate + + const scheduledDate = parseTagDate(metadata.scheduledDate) + if (scheduledDate) result.scheduledDate = scheduledDate + + const textTags = ['direction', 'scope', 'system', 'language', 'purpose'] as const + for (const key of textTags) { + const value = metadata[key] + if (typeof value === 'string' && value.trim()) result[key] = value.trim() + } + + if (typeof metadata.isPrivate === 'boolean') result.isPrivate = metadata.isPrivate + + return result + }, +} diff --git a/apps/sim/connectors/gong/index.ts b/apps/sim/connectors/gong/index.ts new file mode 100644 index 00000000000..a53a3d6c5b9 --- /dev/null +++ b/apps/sim/connectors/gong/index.ts @@ -0,0 +1 @@ +export { gongConnector } from '@/connectors/gong/gong' diff --git a/apps/sim/connectors/grain/grain.ts b/apps/sim/connectors/grain/grain.ts new file mode 100644 index 00000000000..be05cca037a --- /dev/null +++ b/apps/sim/connectors/grain/grain.ts @@ -0,0 +1,587 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { GrainIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { joinTagArray, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('GrainConnector') + +const GRAIN_API_BASE = 'https://api.grain.com/_/public-api/v2' +/** + * Grain's Public API requires a pinned date-based version header on every request. + * Matches the version used by the in-repo Grain tools. + */ +const GRAIN_API_VERSION = '2025-10-31' + +/** + * A participant on a Grain recording. The list endpoint only populates this when + * `include.participants` is requested in the body. + */ +interface GrainParticipant { + id: string + name: string + email: string | null +} + +/** + * A team a Grain recording belongs to. Always present on the recording object (may be + * an empty array). + */ +interface GrainTeam { + id: string + name: string +} + +/** + * The meeting type classification of a Grain recording. Always present on the recording + * object but nullable. + */ +interface GrainMeetingType { + id: string + name: string + scope?: 'internal' | 'external' +} + +/** + * A Grain recording as returned by the v2 recordings endpoints. Only the fields the + * connector reads are modeled; the API returns additional optional fields. + * + * The v2 Public API returns the recording identifier as `id` (confirmed against the + * Grain Public API reference and the in-repo Grain tools). + * + * `source`, `tags`, `teams`, and `meeting_type` are returned by default on the + * recording object. `participants` is populated only when requested via the + * `include.participants` flag — the connector requests it (see {@link RECORDING_INCLUDE}) + * so participant names are available for tag mapping. + */ +interface GrainRecording { + id?: string + title?: string + start_datetime?: string + end_datetime?: string + duration_ms?: number + url?: string + source?: string + tags?: string[] + teams?: GrainTeam[] + meeting_type?: GrainMeetingType | null + participants?: GrainParticipant[] +} + +interface GrainRecordingsListResponse { + recordings?: GrainRecording[] + cursor?: string | null +} + +/** + * A single speaker-attributed segment of a Grain transcript. The transcript endpoint + * returns a bare JSON array of these. + */ +interface GrainTranscriptSegment { + participant_id: string | null + speaker?: string + start?: number + end?: number + text?: string +} + +/** + * The `include` flags requested on every recordings call. Grain returns `teams` and + * `meeting_type` by default, but gates `participants` behind an include flag. Participant + * names feed connector tag mapping, so the flag is always requested. Only documented + * include flags are sent to avoid the API rejecting unknown keys. + */ +const RECORDING_INCLUDE = { participants: true } as const + +/** Number of milliseconds in a day, used to convert the lookback window to a timestamp. */ +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +/** + * Valid values for the recordings list `participant_scope` filter (verified against the + * Grain Public API recordings list request body, which the in-repo Grain tools also use). + */ +const PARTICIPANT_SCOPES = ['internal', 'external'] as const +type ParticipantScope = (typeof PARTICIPANT_SCOPES)[number] + +function isParticipantScope(value: unknown): value is ParticipantScope { + return typeof value === 'string' && PARTICIPANT_SCOPES.includes(value as ParticipantScope) +} + +/** + * Builds the recordings list `filter` object from the connector's scoping config. Only + * documented Grain filter keys are emitted, and only when configured, so an empty config + * produces no `filter` (full sync). Returns undefined when no scoping is configured. + * + * Supported keys (verified against the in-repo Grain list_recordings tool / Public API): + * - `after_datetime` — derived from `lookbackDays`; recordings on/after the window start + * - `participant_scope` — `internal` or `external` + * - `title_search` — substring match against recording titles + * - `team` — recordings belonging to the given team UUID + * - `meeting_type` — recordings classified as the given meeting type UUID + */ +function buildRecordingFilter( + sourceConfig: Record +): Record | undefined { + const filter: Record = {} + + const lookbackDays = sourceConfig.lookbackDays ? Number(sourceConfig.lookbackDays) : 0 + if (Number.isFinite(lookbackDays) && lookbackDays > 0) { + filter.after_datetime = new Date(Date.now() - lookbackDays * MS_PER_DAY).toISOString() + } + + if (isParticipantScope(sourceConfig.participantScope)) { + filter.participant_scope = sourceConfig.participantScope + } + + const titleSearch = + typeof sourceConfig.titleSearch === 'string' ? sourceConfig.titleSearch.trim() : '' + if (titleSearch) filter.title_search = titleSearch + + const team = typeof sourceConfig.teamId === 'string' ? sourceConfig.teamId.trim() : '' + if (team) filter.team = team + + const meetingType = + typeof sourceConfig.meetingTypeId === 'string' ? sourceConfig.meetingTypeId.trim() : '' + if (meetingType) filter.meeting_type = meetingType + + return Object.keys(filter).length > 0 ? filter : undefined +} + +/** + * Builds the auth + version headers shared by every Grain API request. + */ +function grainHeaders(accessToken: string): Record { + return { + 'Content-Type': 'application/json', + Authorization: `Bearer ${accessToken}`, + 'Public-Api-Version': GRAIN_API_VERSION, + } +} + +/** + * Resolves the recording's unique identifier. The v2 Public API returns the recording + * id as the `id` field. Returns an empty string when it is absent. + */ +function recordingId(recording: GrainRecording): string { + return (recording.id ?? '').trim() +} + +/** + * Derives the document title for a recording, falling back to a stable placeholder. + */ +function recordingTitle(recording: GrainRecording): string { + return recording.title?.trim() || 'Untitled Grain Recording' +} + +/** + * Extracts participant display names from a recording, dropping blanks. + */ +function participantNames(recording: GrainRecording): string[] { + return (recording.participants ?? []) + .map((p) => p.name?.trim()) + .filter((name): name is string => Boolean(name)) +} + +/** + * Extracts team names from a recording, dropping blanks. + */ +function teamNames(recording: GrainRecording): string[] { + return (recording.teams ?? []) + .map((t) => t.name?.trim()) + .filter((name): name is string => Boolean(name)) +} + +/** + * Extracts user-applied tag labels from a recording, dropping blanks. + */ +function recordingLabels(recording: GrainRecording): string[] { + return (recording.tags ?? []) + .map((tag) => tag?.trim()) + .filter((tag): tag is string => Boolean(tag)) +} + +/** + * Computes the metadata-based change-detection hash for a recording. + * + * Grain exposes no `updated_at`/`modified` field, so the hash combines the stable + * recording id with `end_datetime` and `duration_ms` — the values that change when a + * recording is re-processed or re-cut. The identical formula is used for both the + * listing stub and the fully-fetched document so unchanged recordings are skipped. + */ +function buildContentHash(recording: GrainRecording): string { + return `grain:${recordingId(recording)}:${recording.end_datetime ?? ''}:${recording.duration_ms ?? ''}` +} + +/** + * Builds the metadata bag attached to both stubs and fetched documents. Keeping a + * single source ensures the stub and getDocument agree on tag inputs. + */ +function buildMetadata(recording: GrainRecording): Record { + return { + title: recordingTitle(recording), + duration: recording.duration_ms, + meetingDate: recording.start_datetime, + participants: participantNames(recording), + source: recording.source, + labels: recordingLabels(recording), + teams: teamNames(recording), + meetingType: recording.meeting_type?.name, + } +} + +/** + * Builds the deferred listing stub for a recording. Content is fetched lazily in + * getDocument; only metadata and the change hash are computed here. + */ +function recordingToStub(recording: GrainRecording): ExternalDocument { + return { + externalId: recordingId(recording), + title: recordingTitle(recording), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: recording.url || undefined, + contentHash: buildContentHash(recording), + metadata: buildMetadata(recording), + } +} + +/** + * Formats a recording header plus speaker-attributed transcript lines into plain text. + */ +function formatTranscriptContent( + recording: GrainRecording, + segments: GrainTranscriptSegment[] +): string { + const parts: string[] = [] + + parts.push(`Meeting: ${recordingTitle(recording)}`) + if (recording.start_datetime) parts.push(`Date: ${recording.start_datetime}`) + if (recording.duration_ms != null) { + const minutes = Math.round(recording.duration_ms / 60000) + parts.push(`Duration: ${minutes} minutes`) + } + const names = participantNames(recording) + if (names.length > 0) parts.push(`Participants: ${names.join(', ')}`) + + parts.push('') + parts.push('--- Transcript ---') + for (const segment of segments) { + const text = segment.text?.trim() + if (!text) continue + const speaker = segment.speaker?.trim() || 'Unknown' + parts.push(`${speaker}: ${text}`) + } + + return parts.join('\n') +} + +/** + * Fetches a single recording's metadata from the v2 recordings endpoint. + * Returns null on 404 (recording deleted/inaccessible). + */ +async function fetchRecording(accessToken: string, id: string): Promise { + const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}`, { + method: 'POST', + headers: grainHeaders(accessToken), + body: JSON.stringify({ include: RECORDING_INCLUDE }), + }) + + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch Grain recording: ${response.status}`) + } + + return (await response.json()) as GrainRecording +} + +/** + * Fetches the speaker-attributed transcript segments for a recording. + * Returns null on 404, or an empty array when the recording has no transcript yet. + */ +async function fetchTranscript( + accessToken: string, + id: string +): Promise { + const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}/transcript`, { + method: 'GET', + headers: grainHeaders(accessToken), + }) + + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch Grain transcript: ${response.status}`) + } + + const data = await response.json() + return Array.isArray(data) ? (data as GrainTranscriptSegment[]) : [] +} + +export const grainConnector: ConnectorConfig = { + id: 'grain', + name: 'Grain', + description: 'Sync meeting recording transcripts from Grain', + version: '1.0.0', + icon: GrainIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your Grain API key', + }, + + configFields: [ + { + id: 'maxRecordings', + title: 'Max Recordings', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + description: 'Cap the total number of recordings synced. Leave blank to sync all.', + }, + { + id: 'lookbackDays', + title: 'Lookback Window (days)', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 90 (default: all time)', + description: 'Only sync recordings from the last N days. Leave blank to sync any age.', + }, + { + id: 'participantScope', + title: 'Participant Scope', + type: 'dropdown', + required: false, + mode: 'advanced', + description: + 'Limit to internal-only meetings or meetings that include an external participant. Leave as Any to sync both.', + options: [ + { label: 'Any', id: '' }, + { label: 'Internal only', id: 'internal' }, + { label: 'External (has external participant)', id: 'external' }, + ], + }, + { + id: 'titleSearch', + title: 'Title Search', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. weekly standup', + description: 'Only sync recordings whose title matches this text. Leave blank to sync all.', + }, + { + id: 'teamId', + title: 'Team ID', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. a1b2c3d4-e5f6-7890-abcd-ef1234567890', + description: + 'Only sync recordings belonging to this team (Grain team UUID). Leave blank to sync all teams.', + }, + { + id: 'meetingTypeId', + title: 'Meeting Type ID', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. a1b2c3d4-e5f6-7890-abcd-ef1234567890', + description: + 'Only sync recordings of this meeting type (Grain meeting type UUID). Leave blank to sync all types.', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record + ): Promise => { + const maxRecordings = sourceConfig.maxRecordings ? Number(sourceConfig.maxRecordings) : 0 + + const body: Record = { include: RECORDING_INCLUDE } + if (cursor) body.cursor = cursor + + const cachedFilter = syncContext?.grainFilter as Record | undefined | null + const filter = cachedFilter !== undefined ? cachedFilter : buildRecordingFilter(sourceConfig) + if (syncContext && cachedFilter === undefined) syncContext.grainFilter = filter ?? null + if (filter) body.filter = filter + + logger.info('Listing Grain recordings', { + hasCursor: Boolean(cursor), + hasFilter: Boolean(filter), + }) + + const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings`, { + method: 'POST', + headers: grainHeaders(accessToken), + body: JSON.stringify(body), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Grain recordings', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Grain recordings: ${response.status}`) + } + + const data = (await response.json()) as GrainRecordingsListResponse + const recordings = data.recordings ?? [] + const nextCursor = data.cursor?.trim() || undefined + + const allDocuments: ExternalDocument[] = [] + for (const recording of recordings) { + if (!recordingId(recording)) continue + allDocuments.push(recordingToStub(recording)) + } + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let documents = allDocuments + if (maxRecordings > 0) { + const remaining = Math.max(0, maxRecordings - prevFetched) + if (allDocuments.length > remaining) { + documents = allDocuments.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxRecordings > 0 && totalFetched >= maxRecordings + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && Boolean(nextCursor) + + return { + documents, + nextCursor: hasMore ? nextCursor : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const recording = await fetchRecording(accessToken, externalId) + if (!recording) return null + + const segments = await fetchTranscript(accessToken, externalId) + if (!segments) return null + + const hasTranscript = segments.some((segment) => segment.text?.trim()) + if (!hasTranscript) { + logger.info('Transcript not yet available for Grain recording', { externalId }) + return null + } + + const content = formatTranscriptContent(recording, segments) + + return { + externalId, + title: recordingTitle(recording), + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: recording.url || undefined, + contentHash: buildContentHash(recording), + metadata: buildMetadata(recording), + } + } catch (error) { + logger.warn('Failed to get Grain recording', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxRecordings = sourceConfig.maxRecordings as string | undefined + if (maxRecordings && (Number.isNaN(Number(maxRecordings)) || Number(maxRecordings) < 0)) { + return { valid: false, error: 'Max recordings must be a non-negative number' } + } + + try { + const response = await fetchWithRetry( + `${GRAIN_API_BASE}/recordings`, + { + method: 'POST', + headers: grainHeaders(accessToken), + body: JSON.stringify({}), + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Grain access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'title', displayName: 'Title', fieldType: 'text' }, + { id: 'participants', displayName: 'Participants', fieldType: 'text' }, + { id: 'source', displayName: 'Source', fieldType: 'text' }, + { id: 'labels', displayName: 'Labels', fieldType: 'text' }, + { id: 'teams', displayName: 'Teams', fieldType: 'text' }, + { id: 'meetingType', displayName: 'Meeting Type', fieldType: 'text' }, + { id: 'duration', displayName: 'Duration (ms)', fieldType: 'number' }, + { id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.title === 'string' && metadata.title.trim()) { + result.title = metadata.title + } + + const participants = joinTagArray(metadata.participants) + if (participants) result.participants = participants + + if (typeof metadata.source === 'string' && metadata.source.trim()) { + result.source = metadata.source.trim() + } + + const labels = joinTagArray(metadata.labels) + if (labels) result.labels = labels + + const teams = joinTagArray(metadata.teams) + if (teams) result.teams = teams + + if (typeof metadata.meetingType === 'string' && metadata.meetingType.trim()) { + result.meetingType = metadata.meetingType.trim() + } + + if (metadata.duration != null) { + const num = Number(metadata.duration) + if (!Number.isNaN(num)) result.duration = num + } + + const meetingDate = parseTagDate(metadata.meetingDate) + if (meetingDate) result.meetingDate = meetingDate + + return result + }, +} diff --git a/apps/sim/connectors/grain/index.ts b/apps/sim/connectors/grain/index.ts new file mode 100644 index 00000000000..6d9ffac56cd --- /dev/null +++ b/apps/sim/connectors/grain/index.ts @@ -0,0 +1 @@ +export { grainConnector } from '@/connectors/grain/grain' diff --git a/apps/sim/connectors/granola/granola.ts b/apps/sim/connectors/granola/granola.ts new file mode 100644 index 00000000000..288ebd915bd --- /dev/null +++ b/apps/sim/connectors/granola/granola.ts @@ -0,0 +1,528 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { GranolaIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('GranolaConnector') + +const GRANOLA_API_BASE = 'https://public-api.granola.ai/v1' +/** Granola caps page_size at 30; request the maximum to minimize round trips. */ +const PAGE_SIZE = 30 + +/** Granola folder identifiers match `fol_` followed by 14 alphanumeric chars. */ +const FOLDER_ID_PATTERN = /^fol_[a-zA-Z0-9]{14}$/ + +/** + * A note owner or attendee as returned by the Granola API. + */ +interface GranolaUser { + name: string | null + email: string +} + +/** + * The lightweight note shape returned by the list endpoint. It contains only + * metadata — no summary or transcript content — so content must be fetched per + * note via the get endpoint (the deferred-content pattern). + */ +interface GranolaNoteSummary { + id: string + object?: string + title: string | null + owner?: GranolaUser + created_at: string + updated_at: string +} + +/** + * A folder the note belongs to, as returned by the get endpoint. + */ +interface GranolaFolderMembership { + id: string + name: string + parent_folder_id?: string | null +} + +/** + * Calendar event details attached to a note, when available. + * Field names match the Granola API's CalendarEvent schema. + */ +interface GranolaCalendarEvent { + event_title?: string | null + organiser?: string | null + calendar_event_id?: string | null + scheduled_start_time?: string | null + scheduled_end_time?: string | null + invitees?: { email: string }[] +} + +/** + * The full note shape returned by the get endpoint, including summary content. + */ +interface GranolaNoteDetail extends GranolaNoteSummary { + web_url?: string | null + calendar_event?: GranolaCalendarEvent | null + attendees?: GranolaUser[] + folder_membership?: GranolaFolderMembership[] + summary_text?: string | null + summary_markdown?: string | null +} + +/** + * The list endpoint response envelope. + */ +interface GranolaListNotesResponse { + notes?: GranolaNoteSummary[] + hasMore?: boolean + cursor?: string | null +} + +/** + * Builds the authorization headers for a Granola API request. + */ +function granolaHeaders(accessToken: string): Record { + return { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + } +} + +/** + * Produces the change-detection hash for a note from its stable identifiers. + * Granola exposes `updated_at`, which advances whenever the note (or its summary) + * changes, so a metadata-only hash is sufficient and stays identical between the + * list stub and the fetched document — letting the sync engine skip re-fetching + * unchanged notes. + */ +function buildContentHash(id: string, updatedAt: string): string { + return `granola:${id}:${updatedAt}` +} + +/** + * Parses the optional `maxNotes` cap from source config. + * Returns 0 (unlimited) when unset or invalid. + */ +function parseMaxNotes(sourceConfig: Record): number { + const raw = sourceConfig.maxNotes + if (raw == null || raw === '') return 0 + const num = Number(raw) + return Number.isFinite(num) && num > 0 ? Math.floor(num) : 0 +} + +/** + * Parses the optional `folderId` scope from source config. Returns a trimmed + * folder id only when it matches Granola's `fol_…` identifier shape; otherwise + * returns undefined so the request is not scoped to an invalid folder. + */ +function parseFolderId(sourceConfig: Record): string | undefined { + const raw = sourceConfig.folderId + if (typeof raw !== 'string') return undefined + const trimmed = raw.trim() + if (!trimmed) return undefined + return FOLDER_ID_PATTERN.test(trimmed) ? trimmed : undefined +} + +/** + * Parses an optional ISO 8601 date filter from a named source-config field. + * Returns a normalized ISO 8601 string when the value is a valid date; otherwise + * returns undefined so the request is not scoped to an invalid date. + */ +function parseDateFilter(sourceConfig: Record, key: string): string | undefined { + const raw = sourceConfig[key] + if (typeof raw !== 'string') return undefined + const trimmed = raw.trim() + if (!trimmed) return undefined + const parsed = new Date(trimmed) + return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString() +} + +/** + * Detects whether a string contains HTML markup. Granola returns markdown for + * `summary_markdown`, but this guard lets us defensively strip tags if the API + * ever emits HTML, without mangling legitimate markdown. + */ +function looksLikeHtml(value: string): boolean { + return /<\/?[a-z][\s\S]*?>/i.test(value) +} + +/** + * Assembles the document content from a note's title and summary. Prefers the + * markdown summary, falling back to plain-text summary. HTML is stripped only + * when detected so markdown formatting is preserved. + */ +function buildContent(note: GranolaNoteDetail): string { + const parts: string[] = [] + + const title = note.title?.trim() + if (title) parts.push(`# ${title}`) + + const rawSummary = note.summary_markdown?.trim() || note.summary_text?.trim() || '' + if (rawSummary) { + parts.push('') + parts.push(looksLikeHtml(rawSummary) ? htmlToPlainText(rawSummary) : rawSummary) + } + + return parts.join('\n').trim() +} + +/** + * Resolves an owner's display name, falling back to email, for tag mapping. + */ +function ownerDisplay(owner?: GranolaUser): string | undefined { + if (!owner) return undefined + return owner.name?.trim() || owner.email?.trim() || undefined +} + +/** + * Collects attendee display names (falling back to email) for tag mapping. + */ +function collectAttendees(note: GranolaNoteDetail): string[] { + if (!Array.isArray(note.attendees)) return [] + return note.attendees + .map((a) => a.name?.trim() || a.email?.trim() || '') + .filter((name): name is string => name.length > 0) +} + +/** + * Collects folder names for tag mapping. + */ +function collectFolders(note: GranolaNoteDetail): string[] { + if (!Array.isArray(note.folder_membership)) return [] + return note.folder_membership + .map((f) => f.name?.trim() || '') + .filter((name): name is string => name.length > 0) +} + +/** + * Builds the deferred stub for a note from list metadata. Content is empty and + * fetched later via `getDocument` only for new/changed notes. + */ +function noteSummaryToStub(note: GranolaNoteSummary): ExternalDocument { + return { + externalId: note.id, + title: note.title?.trim() || 'Untitled Note', + content: '', + contentDeferred: true, + mimeType: 'text/markdown', + contentHash: buildContentHash(note.id, note.updated_at), + metadata: { + title: note.title?.trim() || undefined, + owner: ownerDisplay(note.owner), + ownerName: note.owner?.name ?? undefined, + ownerEmail: note.owner?.email ?? undefined, + noteDate: note.created_at, + updatedAt: note.updated_at, + }, + } +} + +export const granolaConnector: ConnectorConfig = { + id: 'granola', + name: 'Granola', + description: 'Sync AI meeting notes and summaries from Granola', + version: '1.0.0', + icon: GranolaIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your Granola API key', + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'maxNotes', + title: 'Max Notes', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + description: 'Cap the number of notes synced. Leave blank to sync all notes.', + }, + { + id: 'folderId', + title: 'Folder ID', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. fol_4y6LduVdwSKC27', + description: + 'Scope the sync to a single folder and its child folders. Leave blank to sync notes from all folders.', + }, + { + id: 'createdAfter', + title: 'Created After', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 2025-01-01 or 2025-01-01T00:00:00Z', + description: + 'Only sync notes created on or after this date (ISO 8601). Leave blank to sync notes regardless of creation date.', + }, + { + id: 'createdBefore', + title: 'Created Before', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 2025-12-31 or 2025-12-31T23:59:59Z', + description: + 'Only sync notes created on or before this date (ISO 8601). Leave blank to sync notes regardless of creation date.', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const maxNotes = parseMaxNotes(sourceConfig) + const folderId = parseFolderId(sourceConfig) + const createdAfter = parseDateFilter(sourceConfig, 'createdAfter') + const createdBefore = parseDateFilter(sourceConfig, 'createdBefore') + + const url = new URL(`${GRANOLA_API_BASE}/notes`) + url.searchParams.set('page_size', String(PAGE_SIZE)) + if (cursor) url.searchParams.set('cursor', cursor) + if (lastSyncAt) url.searchParams.set('updated_after', lastSyncAt.toISOString()) + if (folderId) url.searchParams.set('folder_id', folderId) + if (createdAfter) url.searchParams.set('created_after', createdAfter) + if (createdBefore) url.searchParams.set('created_before', createdBefore) + + logger.info('Listing Granola notes', { + hasCursor: Boolean(cursor), + incremental: Boolean(lastSyncAt), + scopedToFolder: Boolean(folderId), + scopedByCreatedAfter: Boolean(createdAfter), + scopedByCreatedBefore: Boolean(createdBefore), + }) + + const response = await fetchWithRetry(url.toString(), { + method: 'GET', + headers: granolaHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Granola notes', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Granola notes: ${response.status}`) + } + + const data = (await response.json()) as GranolaListNotesResponse + const notes = Array.isArray(data.notes) ? data.notes : [] + const nextCursor = data.cursor?.trim() || undefined + + const allStubs = notes.filter((note) => Boolean(note.id)).map((note) => noteSummaryToStub(note)) + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let documents = allStubs + if (maxNotes > 0) { + const remaining = Math.max(0, maxNotes - prevFetched) + if (allStubs.length > remaining) { + documents = allStubs.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + + const hitLimit = maxNotes > 0 && totalFetched >= maxNotes + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && Boolean(data.hasMore) && Boolean(nextCursor) + + return { + documents, + nextCursor: hasMore ? nextCursor : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const url = `${GRANOLA_API_BASE}/notes/${encodeURIComponent(externalId)}` + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: granolaHeaders(accessToken), + }) + + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch Granola note: ${response.status}`) + } + + const note = (await response.json()) as GranolaNoteDetail + if (!note?.id) return null + + const content = buildContent(note) + if (!content) { + logger.info('Granola note has no content', { externalId }) + return null + } + + const attendees = collectAttendees(note) + const folders = collectFolders(note) + const meeting = note.calendar_event?.event_title?.trim() || undefined + const meetingDate = note.calendar_event?.scheduled_start_time?.trim() || undefined + + return { + externalId: note.id, + title: note.title?.trim() || 'Untitled Note', + content, + contentDeferred: false, + mimeType: 'text/markdown', + sourceUrl: note.web_url?.trim() || undefined, + contentHash: buildContentHash(note.id, note.updated_at), + metadata: { + title: note.title?.trim() || undefined, + owner: ownerDisplay(note.owner), + ownerName: note.owner?.name ?? undefined, + ownerEmail: note.owner?.email ?? undefined, + noteDate: note.created_at, + updatedAt: note.updated_at, + attendees, + folders, + meeting, + meetingDate, + }, + } + } catch (error) { + logger.warn('Failed to get Granola note', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxNotes = sourceConfig.maxNotes as string | undefined + if (maxNotes && (Number.isNaN(Number(maxNotes)) || Number(maxNotes) < 0)) { + return { valid: false, error: 'Max notes must be a non-negative number' } + } + + const folderId = sourceConfig.folderId + if ( + typeof folderId === 'string' && + folderId.trim() && + !FOLDER_ID_PATTERN.test(folderId.trim()) + ) { + return { + valid: false, + error: + 'Folder ID must look like fol_ followed by 14 alphanumeric characters (e.g. fol_4y6LduVdwSKC27)', + } + } + + const createdAfter = sourceConfig.createdAfter + if ( + typeof createdAfter === 'string' && + createdAfter.trim() && + Number.isNaN(new Date(createdAfter.trim()).getTime()) + ) { + return { + valid: false, + error: + 'Created After must be a valid date (ISO 8601, e.g. 2025-01-01 or 2025-01-01T00:00:00Z)', + } + } + + const createdBefore = sourceConfig.createdBefore + if ( + typeof createdBefore === 'string' && + createdBefore.trim() && + Number.isNaN(new Date(createdBefore.trim()).getTime()) + ) { + return { + valid: false, + error: + 'Created Before must be a valid date (ISO 8601, e.g. 2025-12-31 or 2025-12-31T23:59:59Z)', + } + } + + try { + const url = new URL(`${GRANOLA_API_BASE}/notes`) + url.searchParams.set('page_size', '1') + + const response = await fetchWithRetry( + url.toString(), + { + method: 'GET', + headers: granolaHeaders(accessToken), + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Granola access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'title', displayName: 'Title', fieldType: 'text' }, + { id: 'owner', displayName: 'Owner', fieldType: 'text' }, + { id: 'attendees', displayName: 'Attendees', fieldType: 'text' }, + { id: 'folders', displayName: 'Folders', fieldType: 'text' }, + { id: 'meeting', displayName: 'Meeting', fieldType: 'text' }, + { id: 'noteDate', displayName: 'Note Date', fieldType: 'date' }, + { id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.title === 'string' && metadata.title.trim()) { + result.title = metadata.title.trim() + } + + if (typeof metadata.owner === 'string' && metadata.owner.trim()) { + result.owner = metadata.owner.trim() + } + + const attendees = joinTagArray(metadata.attendees) + if (attendees) result.attendees = attendees + + const folders = joinTagArray(metadata.folders) + if (folders) result.folders = folders + + if (typeof metadata.meeting === 'string' && metadata.meeting.trim()) { + result.meeting = metadata.meeting.trim() + } + + const noteDate = parseTagDate(metadata.noteDate) + if (noteDate) result.noteDate = noteDate + + const meetingDate = parseTagDate(metadata.meetingDate) + if (meetingDate) result.meetingDate = meetingDate + + return result + }, +} diff --git a/apps/sim/connectors/granola/index.ts b/apps/sim/connectors/granola/index.ts new file mode 100644 index 00000000000..c55215fba2a --- /dev/null +++ b/apps/sim/connectors/granola/index.ts @@ -0,0 +1 @@ +export { granolaConnector } from '@/connectors/granola/granola' diff --git a/apps/sim/connectors/greenhouse/greenhouse.ts b/apps/sim/connectors/greenhouse/greenhouse.ts new file mode 100644 index 00000000000..227c9912655 --- /dev/null +++ b/apps/sim/connectors/greenhouse/greenhouse.ts @@ -0,0 +1,737 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { GreenhouseIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { htmlToPlainText, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('GreenhouseConnector') + +const GREENHOUSE_API_BASE = 'https://harvest.greenhouse.io/v1' + +/** + * Upper bound on per-application scorecard requests during a single getDocument call. + * A candidate can have many applications (e.g. internal-transfer tracking); without a + * cap, one document fetch could fan out into dozens of sequential requests. Mirrors the + * Ashby connector's MAX_APPLICATIONS_FOR_FEEDBACK bound. + */ +const MAX_APPLICATIONS_FOR_SCORECARDS = 10 + +/** + * Greenhouse Harvest allows up to 500 candidates per page. We page through the + * full list using the `page` query parameter and stop when the `Link` response + * header no longer advertises a `rel="next"` relationship. + */ +const CANDIDATES_PER_PAGE = 500 + +/** + * Minutes of overlap subtracted from `lastSyncAt` when computing the incremental + * `updated_after` window. Catches candidates whose `updated_at` lands fractionally + * before the recorded sync boundary (clock skew, late-committed writes) at the + * cost of re-listing a small number of recently-touched candidates. + */ +const INCREMENTAL_OVERLAP_MINUTES = 5 +const MS_PER_MINUTE = 60 * 1000 + +interface GreenhouseUser { + id?: number + first_name?: string | null + last_name?: string | null + name?: string | null + employee_id?: string | null +} + +interface GreenhouseEmailAddress { + value?: string + type?: string +} + +interface GreenhouseSource { + id?: number + public_name?: string | null +} + +interface GreenhouseApplication { + id?: number + status?: string | null + applied_at?: string | null + last_activity_at?: string | null + source?: GreenhouseSource | null + recruiter?: GreenhouseUser | null + coordinator?: GreenhouseUser | null +} + +interface GreenhouseCandidate { + id: number + first_name?: string | null + last_name?: string | null + company?: string | null + title?: string | null + created_at?: string | null + updated_at?: string | null + last_activity?: string | null + email_addresses?: GreenhouseEmailAddress[] + tags?: string[] + application_ids?: number[] + applications?: GreenhouseApplication[] + recruiter?: GreenhouseUser | null + coordinator?: GreenhouseUser | null +} + +interface GreenhouseActivityNote { + id?: number + created_at?: string | null + body?: string | null + user?: GreenhouseUser | null + visibility?: string | null +} + +interface GreenhouseActivityEmail { + id?: number + created_at?: string | null + subject?: string | null + body?: string | null + to?: string | null + from?: string | null + user?: GreenhouseUser | null +} + +interface GreenhouseActivityEvent { + id?: number + created_at?: string | null + subject?: string | null + body?: string | null + user?: GreenhouseUser | null +} + +interface GreenhouseActivityFeed { + notes?: GreenhouseActivityNote[] + emails?: GreenhouseActivityEmail[] + activities?: GreenhouseActivityEvent[] +} + +interface GreenhouseScorecardAttribute { + name?: string | null + type?: string | null + note?: string | null + rating?: string | null +} + +interface GreenhouseScorecardQuestion { + question?: string | null + answer?: string | null +} + +interface GreenhouseScorecard { + id?: number + interview?: string | null + interviewer?: GreenhouseUser | null + submitted_by?: GreenhouseUser | null + submitted_at?: string | null + interviewed_at?: string | null + overall_recommendation?: string | null + attributes?: GreenhouseScorecardAttribute[] + questions?: GreenhouseScorecardQuestion[] +} + +/** + * Builds the HTTP Basic authorization header for Greenhouse Harvest. The API key + * is used as the username with an empty password, base64-encoded as `apiKey:`. + */ +function buildAuthHeader(accessToken: string): string { + return `Basic ${Buffer.from(`${accessToken}:`).toString('base64')}` +} + +/** + * Parses the RFC 5988 `Link` response header and returns true when a + * `rel="next"` relationship is present, indicating another page exists. + */ +function hasNextPage(linkHeader: string | null): boolean { + if (!linkHeader) return false + return /;\s*rel\s*=\s*"?next"?/i.test(linkHeader) +} + +/** + * Builds a display name from a candidate's first and last name, falling back to + * the candidate ID when both are missing. + */ +function candidateDisplayName(candidate: GreenhouseCandidate): string { + const parts = [candidate.first_name, candidate.last_name] + .map((part) => part?.trim()) + .filter((part): part is string => Boolean(part)) + return parts.length > 0 ? parts.join(' ') : `Candidate ${candidate.id}` +} + +/** + * Computes the metadata-based content hash for a candidate. Both the listing stub + * and `getDocument` use the same formula so the sync engine can detect changes + * without downloading the deferred content. Greenhouse advances a candidate's + * `updated_at` when the candidate record changes; profile-affecting activity + * (notes, emails, stage changes, scorecard submissions) typically also touches it, + * which is why `updated_after` listing and this hash track the same field. + */ +function buildContentHash(id: number, updatedAt?: string | null): string { + return `greenhouse:${id}:${updatedAt ?? ''}` +} + +/** + * Resolves the source URL for a candidate in the Greenhouse recruiting UI. + */ +function buildSourceUrl(id: number): string { + return `https://app.greenhouse.io/people/${id}` +} + +/** + * Resolves the recruiter, coordinator, and source for a candidate. Greenhouse + * exposes these both at the candidate level and per-application; the candidate + * level is preferred and the most-recent application is used as a fallback so + * the tags are populated even when the candidate-level fields are empty. + */ +function resolveOwnership(candidate: GreenhouseCandidate): { + recruiter?: string + coordinator?: string + source?: string +} { + const applications = Array.isArray(candidate.applications) ? candidate.applications : [] + const latest = applications.reduce((acc, app) => { + if (!acc) return app + const accTime = acc.applied_at ? Date.parse(acc.applied_at) : 0 + const appTime = app.applied_at ? Date.parse(app.applied_at) : 0 + return appTime >= accTime ? app : acc + }, undefined) + + const recruiterName = userName(candidate.recruiter ?? latest?.recruiter) + const coordinatorName = userName(candidate.coordinator ?? latest?.coordinator) + const sourceName = latest?.source?.public_name?.trim() + + return { + recruiter: recruiterName !== 'Unknown' ? recruiterName : undefined, + coordinator: coordinatorName !== 'Unknown' ? coordinatorName : undefined, + source: sourceName || undefined, + } +} + +/** + * Builds the shared metadata block for a candidate, used by both the listing + * stub and the fully-hydrated document. + */ +function buildMetadata(candidate: GreenhouseCandidate): Record { + const emails = (candidate.email_addresses ?? []) + .map((e) => e.value?.trim()) + .filter((value): value is string => Boolean(value)) + + const { recruiter, coordinator, source } = resolveOwnership(candidate) + const applicationCount = Array.isArray(candidate.application_ids) + ? candidate.application_ids.length + : Array.isArray(candidate.applications) + ? candidate.applications.length + : 0 + + return { + candidateName: candidateDisplayName(candidate), + company: candidate.company ?? undefined, + title: candidate.title ?? undefined, + emails, + tags: Array.isArray(candidate.tags) ? candidate.tags : [], + createdAt: candidate.created_at ?? undefined, + updatedAt: candidate.updated_at ?? undefined, + lastActivity: candidate.last_activity ?? undefined, + recruiter, + coordinator, + source, + applicationCount, + } +} + +/** + * Creates a lightweight, content-deferred document stub from a candidate listing + * entry. The real content is fetched lazily via `getDocument` for new or changed + * candidates only. + */ +function candidateToStub(candidate: GreenhouseCandidate): ExternalDocument { + return { + externalId: String(candidate.id), + title: candidateDisplayName(candidate), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(candidate.id), + contentHash: buildContentHash(candidate.id, candidate.updated_at), + metadata: buildMetadata(candidate), + } +} + +/** + * Formats a user's display name for inclusion in content lines. + */ +function userName(user?: GreenhouseUser | null): string { + if (!user) return 'Unknown' + if (user.name?.trim()) return user.name.trim() + const parts = [user.first_name, user.last_name] + .map((part) => part?.trim()) + .filter((part): part is string => Boolean(part)) + return parts.length > 0 ? parts.join(' ') : 'Unknown' +} + +/** + * Renders a candidate's activity feed (notes, emails, and events) into plain text. + */ +function formatActivityFeed(feed: GreenhouseActivityFeed): string { + const lines: string[] = [] + + const notes = feed.notes ?? [] + if (notes.length > 0) { + lines.push('--- Notes ---') + for (const note of notes) { + const body = htmlToPlainText(note.body ?? '').trim() + if (!body) continue + const when = note.created_at ? ` (${note.created_at})` : '' + lines.push(`[${userName(note.user)}${when}] ${body}`) + } + lines.push('') + } + + const activities = feed.activities ?? [] + if (activities.length > 0) { + lines.push('--- Activities ---') + for (const activity of activities) { + const body = htmlToPlainText(activity.body ?? '').trim() + const subject = activity.subject?.trim() + const text = [subject, body].filter(Boolean).join(': ') + if (!text) continue + const when = activity.created_at ? ` (${activity.created_at})` : '' + lines.push(`[${userName(activity.user)}${when}] ${text}`) + } + lines.push('') + } + + const emails = feed.emails ?? [] + if (emails.length > 0) { + lines.push('--- Emails ---') + for (const email of emails) { + const body = htmlToPlainText(email.body ?? '').trim() + const subject = email.subject?.trim() + if (!subject && !body) continue + const when = email.created_at ? ` (${email.created_at})` : '' + if (subject) lines.push(`[${userName(email.user)}${when}] Subject: ${subject}`) + if (body) lines.push(body) + } + lines.push('') + } + + return lines.join('\n').trim() +} + +/** + * Renders interview scorecards (recommendations, attribute ratings, and free-text + * question feedback) into plain text. + */ +function formatScorecards(scorecards: GreenhouseScorecard[]): string { + if (scorecards.length === 0) return '' + + const lines: string[] = ['--- Scorecards ---'] + + for (const scorecard of scorecards) { + const interview = scorecard.interview?.trim() || 'Interview' + const reviewer = userName(scorecard.submitted_by ?? scorecard.interviewer) + const when = scorecard.submitted_at ? ` (${scorecard.submitted_at})` : '' + lines.push(`# ${interview} — ${reviewer}${when}`) + + if (scorecard.overall_recommendation?.trim()) { + lines.push(`Overall recommendation: ${scorecard.overall_recommendation.trim()}`) + } + + for (const attribute of scorecard.attributes ?? []) { + const name = attribute.name?.trim() + if (!name) continue + const rating = attribute.rating?.trim() + const note = htmlToPlainText(attribute.note ?? '').trim() + const detail = [rating ? `rating: ${rating}` : '', note].filter(Boolean).join(' — ') + lines.push(detail ? `- ${name}: ${detail}` : `- ${name}`) + } + + for (const question of scorecard.questions ?? []) { + const q = htmlToPlainText(question.question ?? '').trim() + const a = htmlToPlainText(question.answer ?? '').trim() + if (!q && !a) continue + lines.push(q ? `Q: ${q}` : 'Q:') + if (a) lines.push(`A: ${a}`) + } + + lines.push('') + } + + return lines.join('\n').trim() +} + +/** + * Assembles the full document content for a candidate from their profile header, + * activity feed, and interview scorecards. + */ +function formatContent( + candidate: GreenhouseCandidate, + feed: GreenhouseActivityFeed, + scorecards: GreenhouseScorecard[] +): string { + const sections: string[] = [] + + const header: string[] = [`Candidate: ${candidateDisplayName(candidate)}`] + if (candidate.title?.trim()) header.push(`Title: ${candidate.title.trim()}`) + if (candidate.company?.trim()) header.push(`Company: ${candidate.company.trim()}`) + const emails = (candidate.email_addresses ?? []) + .map((e) => e.value?.trim()) + .filter((value): value is string => Boolean(value)) + if (emails.length > 0) header.push(`Email: ${emails.join(', ')}`) + if (Array.isArray(candidate.tags) && candidate.tags.length > 0) { + header.push(`Tags: ${candidate.tags.join(', ')}`) + } + sections.push(header.join('\n')) + + const activity = formatActivityFeed(feed) + if (activity) sections.push(activity) + + const scorecardText = formatScorecards(scorecards) + if (scorecardText) sections.push(scorecardText) + + return sections.join('\n\n').trim() +} + +/** + * Fetches a single candidate by ID. Returns null on 404. + */ +async function fetchCandidate( + accessToken: string, + id: string +): Promise { + const response = await fetchWithRetry(`${GREENHOUSE_API_BASE}/candidates/${id}`, { + method: 'GET', + headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, + }) + + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch Greenhouse candidate: ${response.status}`) + } + + return (await response.json()) as GreenhouseCandidate +} + +/** + * Fetches a candidate's activity feed. Returns an empty feed when the endpoint + * 404s so the candidate's profile header still produces content. + */ +async function fetchActivityFeed(accessToken: string, id: string): Promise { + const response = await fetchWithRetry(`${GREENHOUSE_API_BASE}/candidates/${id}/activity_feed`, { + method: 'GET', + headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, + }) + + if (!response.ok) { + if (response.status === 404) return {} + throw new Error(`Failed to fetch Greenhouse activity feed: ${response.status}`) + } + + return (await response.json()) as GreenhouseActivityFeed +} + +/** + * Fetches all scorecards across a candidate's applications. Individual + * application failures are tolerated so partial feedback is still indexed. + */ +async function fetchScorecards( + accessToken: string, + applicationIds: number[] +): Promise { + const all: GreenhouseScorecard[] = [] + + for (const applicationId of applicationIds.slice(0, MAX_APPLICATIONS_FOR_SCORECARDS)) { + try { + const response = await fetchWithRetry( + `${GREENHOUSE_API_BASE}/applications/${applicationId}/scorecards`, + { + method: 'GET', + headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, + } + ) + + if (!response.ok) { + if (response.status === 404) continue + throw new Error(`Failed to fetch Greenhouse scorecards: ${response.status}`) + } + + const data = (await response.json()) as GreenhouseScorecard[] + if (Array.isArray(data)) all.push(...data) + } catch (error) { + logger.warn('Failed to fetch scorecards for application', { + applicationId, + error: toError(error).message, + }) + } + } + + return all +} + +/** + * Computes the `updated_after` value for an incremental sync, subtracting a small + * overlap from the last sync timestamp. Returns undefined for full syncs. + */ +function computeUpdatedAfter(lastSyncAt: Date | undefined): string | undefined { + if (!lastSyncAt) return undefined + const since = new Date(lastSyncAt.getTime() - INCREMENTAL_OVERLAP_MINUTES * MS_PER_MINUTE) + return since.toISOString() +} + +export const greenhouseConnector: ConnectorConfig = { + id: 'greenhouse', + name: 'Greenhouse', + description: 'Sync candidate activity and interview scorecards from Greenhouse', + version: '1.0.0', + icon: GreenhouseIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your Greenhouse Harvest API key', + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'maxCandidates', + title: 'Max Candidates', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + description: + 'Cap the number of candidates synced. Leave empty to sync ALL candidates in the organization.', + }, + { + id: 'jobId', + title: 'Job ID', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 123456', + description: + 'Sync only candidates who applied to this Greenhouse job. Leave empty to sync candidates across all jobs.', + }, + { + id: 'createdAfter', + title: 'Created After', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 2024-01-01T00:00:00Z', + description: + 'Sync only candidates created at or after this ISO 8601 timestamp. Leave empty to sync candidates regardless of creation date.', + }, + { + id: 'createdBefore', + title: 'Created Before', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 2024-12-31T23:59:59Z', + description: + 'Sync only candidates created before this ISO 8601 timestamp. Combine with Created After to backfill a bounded date range.', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const maxCandidates = sourceConfig.maxCandidates ? Number(sourceConfig.maxCandidates) : 0 + const parsedPage = cursor ? Number(cursor) : 1 + const page = Number.isFinite(parsedPage) && parsedPage > 0 ? Math.floor(parsedPage) : 1 + const updatedAfter = computeUpdatedAfter(lastSyncAt) + const jobId = typeof sourceConfig.jobId === 'string' ? sourceConfig.jobId.trim() : '' + const createdAfter = + typeof sourceConfig.createdAfter === 'string' ? sourceConfig.createdAfter.trim() : '' + const createdBefore = + typeof sourceConfig.createdBefore === 'string' ? sourceConfig.createdBefore.trim() : '' + + const queryParams = new URLSearchParams({ + per_page: String(CANDIDATES_PER_PAGE), + page: String(page), + }) + if (updatedAfter) queryParams.set('updated_after', updatedAfter) + if (jobId) queryParams.set('job_id', jobId) + if (createdAfter) queryParams.set('created_after', createdAfter) + if (createdBefore) queryParams.set('created_before', createdBefore) + + const url = `${GREENHOUSE_API_BASE}/candidates?${queryParams.toString()}` + + logger.info('Listing Greenhouse candidates', { + page, + perPage: CANDIDATES_PER_PAGE, + incremental: Boolean(updatedAfter), + }) + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Greenhouse candidates', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Greenhouse candidates: ${response.status}`) + } + + const data = (await response.json()) as GreenhouseCandidate[] + const candidates = Array.isArray(data) ? data : [] + const linkHasNext = hasNextPage(response.headers.get('link')) + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let pageCandidates = candidates + if (maxCandidates > 0) { + const remaining = Math.max(0, maxCandidates - prevFetched) + if (pageCandidates.length > remaining) { + pageCandidates = pageCandidates.slice(0, remaining) + } + } + + const documents = pageCandidates.map(candidateToStub) + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && linkHasNext + + return { + documents, + nextCursor: hasMore ? String(page + 1) : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const candidate = await fetchCandidate(accessToken, externalId) + if (!candidate) return null + + const applicationIds = Array.isArray(candidate.application_ids) + ? candidate.application_ids + : [] + + const [feed, scorecards] = await Promise.all([ + fetchActivityFeed(accessToken, externalId), + fetchScorecards(accessToken, applicationIds), + ]) + + const content = formatContent(candidate, feed, scorecards) + if (!content.trim()) return null + + return { + externalId: String(candidate.id), + title: candidateDisplayName(candidate), + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(candidate.id), + contentHash: buildContentHash(candidate.id, candidate.updated_at), + metadata: buildMetadata(candidate), + } + } catch (error) { + logger.warn('Failed to get Greenhouse candidate', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxCandidates = sourceConfig.maxCandidates as string | undefined + if (maxCandidates && (Number.isNaN(Number(maxCandidates)) || Number(maxCandidates) < 0)) { + return { valid: false, error: 'Max candidates must be a non-negative number' } + } + + try { + const response = await fetchWithRetry( + `${GREENHOUSE_API_BASE}/candidates?per_page=1`, + { + method: 'GET', + headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Greenhouse access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'candidateName', displayName: 'Candidate Name', fieldType: 'text' }, + { id: 'company', displayName: 'Company', fieldType: 'text' }, + { id: 'title', displayName: 'Title', fieldType: 'text' }, + { id: 'recruiter', displayName: 'Recruiter', fieldType: 'text' }, + { id: 'coordinator', displayName: 'Coordinator', fieldType: 'text' }, + { id: 'source', displayName: 'Source', fieldType: 'text' }, + { id: 'applicationCount', displayName: 'Application Count', fieldType: 'number' }, + { id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' }, + { id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + const textFields = [ + 'candidateName', + 'company', + 'title', + 'recruiter', + 'coordinator', + 'source', + ] as const + for (const field of textFields) { + const value = metadata[field] + if (typeof value === 'string' && value.trim()) { + result[field] = value.trim() + } + } + + if (typeof metadata.applicationCount === 'number' && metadata.applicationCount >= 0) { + result.applicationCount = metadata.applicationCount + } + + const dateFields = ['updatedAt', 'lastActivity'] as const + for (const field of dateFields) { + const parsed = parseTagDate(metadata[field]) + if (parsed) result[field] = parsed + } + + return result + }, +} diff --git a/apps/sim/connectors/greenhouse/index.ts b/apps/sim/connectors/greenhouse/index.ts new file mode 100644 index 00000000000..ba51fb91d88 --- /dev/null +++ b/apps/sim/connectors/greenhouse/index.ts @@ -0,0 +1 @@ +export { greenhouseConnector } from '@/connectors/greenhouse/greenhouse' diff --git a/apps/sim/connectors/incidentio/incidentio.ts b/apps/sim/connectors/incidentio/incidentio.ts new file mode 100644 index 00000000000..c18d234fab1 --- /dev/null +++ b/apps/sim/connectors/incidentio/incidentio.ts @@ -0,0 +1,623 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { IncidentioIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { htmlToPlainText, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('IncidentioConnector') + +const INCIDENTIO_API_BASE = 'https://api.incident.io' +const PAGE_SIZE = 100 +/** Cap incident updates fetched per document so a noisy incident can't blow the time budget. */ +const MAX_UPDATES_PER_INCIDENT = 100 + +interface IncidentioNamedRef { + id?: string + name?: string + /** Present on incident_status: the stable status category enum (triage/live/closed/...). */ + category?: string + rank?: number +} + +interface IncidentioCustomFieldValue { + value_text?: string + value_link?: string + value_numeric?: string + value_option?: { value?: string } + value_catalog_entry?: { name?: string } +} + +interface IncidentioCustomFieldEntry { + custom_field?: { id?: string; name?: string; field_type?: string } + values?: IncidentioCustomFieldValue[] +} + +interface IncidentioRoleAssignment { + role?: { id?: string; name?: string; role_type?: string } + assignee?: { id?: string; name?: string; email?: string } +} + +interface IncidentioTimestampValue { + incident_timestamp?: { id?: string; name?: string; rank?: number } + /** + * The v2 API nests the timestamp string under `value.value` (an object), not a + * flat string. A flat string is tolerated defensively for older shapes. + */ + value?: { value?: string } | string +} + +interface IncidentioIncident { + id: string + reference?: string + name?: string + summary?: string + mode?: string + visibility?: string + permalink?: string + call_url?: string + created_at?: string + updated_at?: string + slack_channel_id?: string + slack_channel_name?: string + severity?: IncidentioNamedRef + incident_status?: IncidentioNamedRef + incident_type?: IncidentioNamedRef + custom_field_entries?: IncidentioCustomFieldEntry[] + incident_role_assignments?: IncidentioRoleAssignment[] + incident_timestamp_values?: IncidentioTimestampValue[] +} + +interface IncidentioPaginationMeta { + after?: string + page_size?: number + total_record_count?: number +} + +interface IncidentioIncidentsListResponse { + incidents?: IncidentioIncident[] + pagination_meta?: IncidentioPaginationMeta +} + +interface IncidentioIncidentShowResponse { + incident?: IncidentioIncident +} + +/** + * incident.io ActorV2: the entity behind an action, used for incident update + * authors. The v2 API documents this as an ActorV2 with exactly one of the + * nested `user`/`api_key`/`workflow`/`alert` variants populated. A flat + * `{ name, email }` shape is also tolerated defensively; we read whichever is + * present so the author resolves regardless of which form the API returns. + */ +interface IncidentioActor { + user?: { id?: string; name?: string; email?: string } + api_key?: { id?: string; name?: string } + alert?: { id?: string; title?: string } + workflow?: { id?: string; name?: string } + name?: string + email?: string +} + +interface IncidentioUpdate { + id?: string + message?: string + new_severity?: IncidentioNamedRef + /** + * The status change on an update. The v2 API documents `new_incident_status`, + * but a flat `new_status` shape also appears in the wild; both are read. + */ + new_incident_status?: IncidentioNamedRef + new_status?: IncidentioNamedRef + updater?: IncidentioActor + /** Some responses use `author` instead of `updater`; both are read. */ + author?: IncidentioActor + created_at?: string +} + +/** + * Resolves a human-readable display name for an update author, covering the + * ActorV2 variants and the flat `{ name, email }` shape. + */ +function actorName(actor: IncidentioActor | undefined): string | undefined { + if (!actor) return undefined + return ( + actor.user?.name || + actor.user?.email || + actor.api_key?.name || + actor.workflow?.name || + actor.alert?.title || + actor.name || + actor.email || + undefined + ) +} + +interface IncidentioUpdatesListResponse { + incident_updates?: IncidentioUpdate[] + pagination_meta?: IncidentioPaginationMeta +} + +/** + * Builds the metadata-based content hash for an incident. + * + * Uses only the incident `updated_at` timestamp, which incident.io bumps whenever any + * field, role assignment, custom field, or status changes. This guarantees the hash is + * identical whether produced from the list stub or the fully-hydrated getDocument result, + * so the sync engine can detect changes without downloading content. + */ +function buildContentHash(incident: IncidentioIncident): string { + return `incidentio:${incident.id}:${incident.updated_at ?? ''}` +} + +/** + * Builds the public URL for an incident, preferring the API-provided permalink. + */ +function buildSourceUrl(incident: IncidentioIncident): string | undefined { + return incident.permalink || undefined +} + +/** + * Derives the document title from the incident reference and name. + */ +function buildTitle(incident: IncidentioIncident): string { + const name = incident.name?.trim() + const reference = incident.reference?.trim() + if (reference && name) return `${reference}: ${name}` + return name || reference || 'Untitled Incident' +} + +/** + * Collects the source-specific metadata fed to mapTags. Shared between the list stub + * and getDocument so tag values stay consistent regardless of which path produced the doc. + */ +function buildMetadata(incident: IncidentioIncident): Record { + const lead = incident.incident_role_assignments?.find( + (assignment) => assignment.role?.role_type === 'lead' + ) + const reporter = incident.incident_role_assignments?.find( + (assignment) => assignment.role?.role_type === 'reporter' + ) + const reportedBy = (reporter ?? lead)?.assignee?.name + + return { + reference: incident.reference, + status: incident.incident_status?.name, + statusCategory: incident.incident_status?.category, + severity: incident.severity?.name, + incidentType: incident.incident_type?.name, + mode: incident.mode, + visibility: incident.visibility, + incidentDate: incident.created_at, + reportedBy, + } +} + +/** + * Renders a single custom field value to a human-readable string, covering each of + * incident.io's value variants (text, link, numeric, select option, catalog entry). + */ +function renderCustomFieldValue(value: IncidentioCustomFieldValue): string | undefined { + if (value.value_text) return htmlToPlainText(value.value_text) + if (value.value_link) return value.value_link + if (value.value_numeric) return value.value_numeric + if (value.value_option?.value) return value.value_option.value + if (value.value_catalog_entry?.name) return value.value_catalog_entry.name + return undefined +} + +/** + * Formats the incident, its custom fields, role assignments, timeline, and status + * updates into a single plain-text document. HTML in summary / custom field text is + * stripped via htmlToPlainText. + */ +function formatIncidentContent(incident: IncidentioIncident, updates: IncidentioUpdate[]): string { + const parts: string[] = [] + + parts.push(`Incident: ${buildTitle(incident)}`) + if (incident.incident_status?.name) parts.push(`Status: ${incident.incident_status.name}`) + if (incident.severity?.name) parts.push(`Severity: ${incident.severity.name}`) + if (incident.incident_type?.name) parts.push(`Type: ${incident.incident_type.name}`) + if (incident.created_at) parts.push(`Reported: ${incident.created_at}`) + + if (incident.summary?.trim()) { + parts.push('') + parts.push('--- Summary ---') + parts.push(htmlToPlainText(incident.summary)) + } + + const roleLines = (incident.incident_role_assignments ?? []) + .map((assignment) => { + const role = assignment.role?.name + const assignee = assignment.assignee?.name || assignment.assignee?.email + if (!role || !assignee) return undefined + return `${role}: ${assignee}` + }) + .filter((line): line is string => Boolean(line)) + if (roleLines.length > 0) { + parts.push('') + parts.push('--- Roles ---') + parts.push(...roleLines) + } + + const customFieldLines = (incident.custom_field_entries ?? []) + .map((entry) => { + const name = entry.custom_field?.name + if (!name) return undefined + const rendered = (entry.values ?? []) + .map(renderCustomFieldValue) + .filter((v): v is string => Boolean(v)) + if (rendered.length === 0) return undefined + return `${name}: ${rendered.join(', ')}` + }) + .filter((line): line is string => Boolean(line)) + if (customFieldLines.length > 0) { + parts.push('') + parts.push('--- Custom Fields ---') + parts.push(...customFieldLines) + } + + const timestampLines = (incident.incident_timestamp_values ?? []) + .map((entry) => { + const name = entry.incident_timestamp?.name + const value = typeof entry.value === 'string' ? entry.value : entry.value?.value + if (!name || !value) return undefined + return `${name}: ${value}` + }) + .filter((line): line is string => Boolean(line)) + if (timestampLines.length > 0) { + parts.push('') + parts.push('--- Timeline ---') + parts.push(...timestampLines) + } + + if (updates.length > 0) { + parts.push('') + parts.push('--- Updates ---') + for (const update of updates) { + const segments: string[] = [] + if (update.created_at) segments.push(`[${update.created_at}]`) + const author = actorName(update.updater ?? update.author) + if (author) segments.push(author) + const changes: string[] = [] + const newStatusName = update.new_incident_status?.name ?? update.new_status?.name + if (newStatusName) { + changes.push(`status → ${newStatusName}`) + } + if (update.new_severity?.name) changes.push(`severity → ${update.new_severity.name}`) + const message = update.message ? htmlToPlainText(update.message) : '' + const tail = [changes.join(', '), message].filter(Boolean).join(' — ') + const line = [segments.join(' '), tail].filter(Boolean).join(': ') + if (line.trim()) parts.push(line) + } + } + + return parts.join('\n').trim() +} + +/** + * Creates a lightweight document stub from a list entry. No API calls — content is + * deferred to getDocument and only fetched for new or changed incidents. + */ +function incidentToStub(incident: IncidentioIncident): ExternalDocument { + return { + externalId: incident.id, + title: buildTitle(incident), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(incident), + contentHash: buildContentHash(incident), + metadata: buildMetadata(incident), + } +} + +/** + * Fetches all status updates for an incident, following the `after` cursor and capping + * the total to keep getDocument bounded for very long-running incidents. + */ +async function fetchIncidentUpdates( + accessToken: string, + incidentId: string +): Promise { + const updates: IncidentioUpdate[] = [] + let after: string | undefined + + while (updates.length < MAX_UPDATES_PER_INCIDENT) { + const url = new URL(`${INCIDENTIO_API_BASE}/v2/incident_updates`) + url.searchParams.set('incident_id', incidentId) + url.searchParams.set('page_size', String(PAGE_SIZE)) + if (after) url.searchParams.set('after', after) + + const response = await fetchWithRetry(url.toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + if (!response.ok) { + logger.warn('Failed to fetch incident updates', { incidentId, status: response.status }) + break + } + + const data = (await response.json()) as IncidentioUpdatesListResponse + const page = data.incident_updates ?? [] + updates.push(...page) + + after = data.pagination_meta?.after?.trim() || undefined + if (!after || page.length === 0) break + } + + return updates.slice(0, MAX_UPDATES_PER_INCIDENT) +} + +export const incidentioConnector: ConnectorConfig = { + id: 'incidentio', + name: 'incident.io', + description: 'Sync incidents and postmortems from incident.io into your knowledge base', + version: '1.0.0', + icon: IncidentioIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your incident.io API key', + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'statusCategory', + title: 'Status Category', + type: 'dropdown', + required: false, + mode: 'advanced', + options: [ + { label: 'All', id: '' }, + { label: 'Live (active)', id: 'live' }, + { label: 'Paused', id: 'paused' }, + { label: 'Closed', id: 'closed' }, + { label: 'Triage', id: 'triage' }, + { label: 'Learning (post-incident)', id: 'learning' }, + { label: 'Declined', id: 'declined' }, + { label: 'Merged', id: 'merged' }, + { label: 'Canceled', id: 'canceled' }, + ], + description: + 'Only sync incidents in this status category. Leave as All to sync every category.', + }, + { + id: 'mode', + title: 'Mode', + type: 'dropdown', + required: false, + mode: 'advanced', + options: [ + { label: 'All', id: '' }, + { label: 'Standard (real incidents)', id: 'standard' }, + { label: 'Retrospective', id: 'retrospective' }, + { label: 'Test', id: 'test' }, + { label: 'Tutorial', id: 'tutorial' }, + ], + description: + 'Only sync incidents of this mode. Use Standard to exclude test/tutorial incidents.', + }, + { + id: 'maxIncidents', + title: 'Max Incidents', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + description: 'Cap the number of incidents synced. Leave empty to sync all incidents.', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const maxIncidents = sourceConfig.maxIncidents ? Number(sourceConfig.maxIncidents) : 0 + + const statusCategory = + typeof sourceConfig.statusCategory === 'string' ? sourceConfig.statusCategory.trim() : '' + const mode = typeof sourceConfig.mode === 'string' ? sourceConfig.mode.trim() : '' + + const url = new URL(`${INCIDENTIO_API_BASE}/v2/incidents`) + url.searchParams.set('page_size', String(PAGE_SIZE)) + url.searchParams.set('sort_by', 'created_at_oldest_first') + if (cursor) url.searchParams.set('after', cursor) + if (lastSyncAt) url.searchParams.set('updated_at[gte]', lastSyncAt.toISOString()) + if (statusCategory) url.searchParams.set('status_category[one_of]', statusCategory) + if (mode) url.searchParams.set('mode[one_of]', mode) + + logger.info('Listing incident.io incidents', { + cursor: cursor ?? 'initial', + incremental: Boolean(lastSyncAt), + maxIncidents, + }) + + const response = await fetchWithRetry(url.toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list incident.io incidents', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list incident.io incidents: ${response.status}`) + } + + const data = (await response.json()) as IncidentioIncidentsListResponse + const incidents = (data.incidents ?? []).filter((incident) => Boolean(incident.id)) + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let documents = incidents.map(incidentToStub) + if (maxIncidents > 0) { + const remaining = Math.max(0, maxIncidents - prevFetched) + if (documents.length > remaining) { + documents = documents.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents + if (hitLimit && syncContext) syncContext.listingCapped = true + + const after = data.pagination_meta?.after?.trim() || undefined + const hasMore = !hitLimit && Boolean(after) + + return { + documents, + nextCursor: hasMore ? after : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const url = `${INCIDENTIO_API_BASE}/v2/incidents/${encodeURIComponent(externalId)}` + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch incident.io incident: ${response.status}`) + } + + const data = (await response.json()) as IncidentioIncidentShowResponse + const incident = data.incident + if (!incident?.id) return null + + const updates = await fetchIncidentUpdates(accessToken, incident.id) + const content = formatIncidentContent(incident, updates) + if (!content.trim()) return null + + return { + externalId: incident.id, + title: buildTitle(incident), + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(incident), + contentHash: buildContentHash(incident), + metadata: buildMetadata(incident), + } + } catch (error) { + logger.warn('Failed to get incident.io incident', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxIncidents = sourceConfig.maxIncidents as string | undefined + if (maxIncidents && (Number.isNaN(Number(maxIncidents)) || Number(maxIncidents) < 0)) { + return { valid: false, error: 'Max incidents must be a non-negative number' } + } + + try { + const response = await fetchWithRetry( + `${INCIDENTIO_API_BASE}/v2/incidents?page_size=1`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `incident.io access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'status', displayName: 'Status', fieldType: 'text' }, + { id: 'statusCategory', displayName: 'Status Category', fieldType: 'text' }, + { id: 'severity', displayName: 'Severity', fieldType: 'text' }, + { id: 'incidentType', displayName: 'Incident Type', fieldType: 'text' }, + { id: 'mode', displayName: 'Mode', fieldType: 'text' }, + { id: 'visibility', displayName: 'Visibility', fieldType: 'text' }, + { id: 'incidentDate', displayName: 'Incident Date', fieldType: 'date' }, + { id: 'reportedBy', displayName: 'Reported By', fieldType: 'text' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.status === 'string' && metadata.status.trim()) { + result.status = metadata.status + } + + if (typeof metadata.statusCategory === 'string' && metadata.statusCategory.trim()) { + result.statusCategory = metadata.statusCategory + } + + if (typeof metadata.severity === 'string' && metadata.severity.trim()) { + result.severity = metadata.severity + } + + if (typeof metadata.incidentType === 'string' && metadata.incidentType.trim()) { + result.incidentType = metadata.incidentType + } + + if (typeof metadata.mode === 'string' && metadata.mode.trim()) { + result.mode = metadata.mode + } + + if (typeof metadata.visibility === 'string' && metadata.visibility.trim()) { + result.visibility = metadata.visibility + } + + const incidentDate = parseTagDate(metadata.incidentDate) + if (incidentDate) result.incidentDate = incidentDate + + if (typeof metadata.reportedBy === 'string' && metadata.reportedBy.trim()) { + result.reportedBy = metadata.reportedBy + } + + return result + }, +} diff --git a/apps/sim/connectors/incidentio/index.ts b/apps/sim/connectors/incidentio/index.ts new file mode 100644 index 00000000000..acad6173728 --- /dev/null +++ b/apps/sim/connectors/incidentio/index.ts @@ -0,0 +1 @@ +export { incidentioConnector } from '@/connectors/incidentio/incidentio' diff --git a/apps/sim/connectors/monday/index.ts b/apps/sim/connectors/monday/index.ts new file mode 100644 index 00000000000..37d976c63e3 --- /dev/null +++ b/apps/sim/connectors/monday/index.ts @@ -0,0 +1 @@ +export { mondayConnector } from '@/connectors/monday/monday' diff --git a/apps/sim/connectors/monday/monday.ts b/apps/sim/connectors/monday/monday.ts new file mode 100644 index 00000000000..3b4529a1743 --- /dev/null +++ b/apps/sim/connectors/monday/monday.ts @@ -0,0 +1,553 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { MondayIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { parseMultiValue, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('MondayConnector') + +/** + * monday.com GraphQL endpoint. All requests are POSTed here. + * @see https://developer.monday.com/api-reference/docs/basics + */ +const MONDAY_API_URL = 'https://api.monday.com/v2' + +/** + * Stable monday.com API version pinned via the `API-Version` header. monday.com + * keeps at least three quarterly versions live; `2024-10` was deprecated on + * 2026-02-15, so this is pinned to the current stable release. + * @see https://developer.monday.com/api-reference/docs/api-versioning + */ +const MONDAY_API_VERSION = '2026-04' + +/** Max items requested per `items_page` / `next_items_page` call (monday.com max is 500). */ +const ITEMS_PAGE_SIZE = 100 + +/** Max boards requested per `boards` listing page (monday.com max is 500). */ +const BOARDS_PAGE_SIZE = 100 + +/** Max updates fetched per item for content extraction. */ +const UPDATES_LIMIT = 50 + +interface MondayColumnValue { + id: string + text: string | null + column: { id: string; title: string } | null +} + +interface MondayUpdate { + id: string + text_body: string | null + created_at: string | null + creator: { name: string | null } | null +} + +interface MondayItem { + id: string + name: string | null + state: string | null + created_at: string | null + updated_at: string | null + url: string | null + board: { id: string; name: string | null } | null + group: { id: string; title: string | null } | null + creator: { name: string | null } | null + column_values: MondayColumnValue[] + updates: MondayUpdate[] +} + +interface MondayItemsPage { + cursor: string | null + items: MondayItem[] +} + +interface MondayBoard { + id: string + name: string | null + items_page: MondayItemsPage | null +} + +/** + * Pagination state encoded into `nextCursor`. Tracks which board in the + * configured/accessible list is being read (`boardIndex`) and the opaque + * `items_page` cursor within that board (`itemsCursor`). + */ +interface CursorState { + boardIndex: number + itemsCursor?: string +} + +function encodeCursor(state: CursorState): string { + return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url') +} + +function decodeCursor(cursor?: string): CursorState { + if (!cursor) return { boardIndex: 0 } + try { + const json = Buffer.from(cursor, 'base64url').toString('utf8') + const parsed = JSON.parse(json) as Partial + return { + boardIndex: Number(parsed.boardIndex) || 0, + itemsCursor: typeof parsed.itemsCursor === 'string' ? parsed.itemsCursor : undefined, + } + } catch { + return { boardIndex: 0 } + } +} + +/** + * monday.com uses the raw access token in the `Authorization` header — it is NOT + * prefixed with "Bearer". The `API-Version` header pins the schema version. + * @see https://developer.monday.com/api-reference/docs/authentication + */ +function mondayHeaders(accessToken: string): Record { + return { + 'Content-Type': 'application/json', + Authorization: accessToken, + 'API-Version': MONDAY_API_VERSION, + } +} + +/** + * Executes a GraphQL query against the monday.com API, surfacing GraphQL-level + * errors (which return HTTP 200 with an `errors` array) as thrown errors. + */ +async function mondayGraphQL( + accessToken: string, + query: string, + variables: Record = {}, + retryOptions?: Parameters[2] +): Promise { + const response = await fetchWithRetry( + MONDAY_API_URL, + { + method: 'POST', + headers: mondayHeaders(accessToken), + body: JSON.stringify({ query, variables }), + }, + retryOptions + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error( + `monday.com API HTTP error: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}` + ) + } + + const data = (await response.json()) as { + data?: T + errors?: { message?: string }[] + error_message?: string + } + + if (data.errors && data.errors.length > 0) { + const message = data.errors + .map((e) => e.message) + .filter(Boolean) + .join('; ') + throw new Error(`monday.com API error: ${message || 'Unknown GraphQL error'}`) + } + if (data.error_message) { + throw new Error(`monday.com API error: ${data.error_message}`) + } + + return data.data as T +} + +/** + * GraphQL selection set for an item, shared between listing and single-item + * fetches so the resolved fields stay in sync. + */ +const ITEM_FIELDS = ` + id + name + state + created_at + updated_at + url + board { id name } + group { id title } + creator { name } + column_values { + id + text + column { id title } + } + updates(limit: ${UPDATES_LIMIT}) { + id + text_body + created_at + creator { name } + } +` + +/** + * Builds the change-detection hash from item identity + last-modified time. Must + * be identical whether produced inline during listing or via a `getDocument` + * fetch, since monday's `updated_at` advances on any item or column change. + */ +function buildContentHash(itemId: string, updatedAt: string | null | undefined): string { + return `monday:${itemId}:${updatedAt ?? ''}` +} + +/** + * Resolves a stable board name + id for an item, preferring the nested `board` + * field and falling back to the board the item was listed under. + */ +function resolveBoard( + item: MondayItem, + fallback?: { id: string; name: string | null } +): { id: string; name: string } { + const id = item.board?.id ?? fallback?.id ?? '' + const name = item.board?.name ?? fallback?.name ?? '' + return { id, name } +} + +function buildSourceUrl(item: MondayItem): string | undefined { + return item.url ?? undefined +} + +/** + * Builds the metadata object carried on every document, used both for tag mapping + * (`mapTags`) and for downstream display. Kept in one place so listing and + * single-item fetches emit identical metadata. + */ +function itemMetadata( + item: MondayItem, + board: { id: string; name: string } +): Record { + return { + boardId: board.id, + boardName: board.name, + itemName: item.name ?? '', + groupTitle: item.group?.title ?? '', + state: item.state ?? '', + creatorName: item.creator?.name ?? '', + createdAt: item.created_at ?? undefined, + updatedAt: item.updated_at ?? undefined, + } +} + +/** + * Produces a fully-hydrated document from a single `items_page` element. The list + * query already selects the complete item payload (`column_values` + `updates`), + * so content is built inline here — avoiding a redundant per-item `getDocument` + * round-trip. `contentHash` derives solely from id + `updated_at`, so it is + * identical whether produced here or via `getDocument`. + */ +function itemToDocument( + item: MondayItem, + fallbackBoard?: { id: string; name: string | null } +): ExternalDocument { + const board = resolveBoard(item, fallbackBoard) + return { + externalId: item.id, + title: item.name?.trim() || 'Untitled Item', + content: formatItemContent(item, board), + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(item), + contentHash: buildContentHash(item.id, item.updated_at), + metadata: itemMetadata(item, board), + } +} + +/** + * Formats an item's column values and updates into a plain-text document. The + * resolved board is passed in so listing (which has a board fallback) and + * single-item fetches produce identical content. + */ +function formatItemContent(item: MondayItem, board: { id: string; name: string }): string { + const parts: string[] = [] + + if (board.name) parts.push(`Board: ${board.name}`) + parts.push(`Item: ${item.name?.trim() || 'Untitled Item'}`) + if (item.group?.title) parts.push(`Group: ${item.group.title}`) + if (item.creator?.name) parts.push(`Created by: ${item.creator.name}`) + if (item.created_at) parts.push(`Created: ${item.created_at}`) + if (item.updated_at) parts.push(`Updated: ${item.updated_at}`) + + const columns = item.column_values.filter((cv) => cv.text?.trim()) + if (columns.length > 0) { + parts.push('') + parts.push('--- Fields ---') + for (const cv of columns) { + const title = cv.column?.title?.trim() || cv.id + parts.push(`${title}: ${cv.text}`) + } + } + + const updates = item.updates.filter((u) => u.text_body?.trim()) + if (updates.length > 0) { + parts.push('') + parts.push('--- Updates ---') + for (const update of updates) { + const author = update.creator?.name?.trim() || 'Unknown' + parts.push(`Update by ${author}: ${update.text_body}`) + } + } + + return parts.join('\n') +} + +/** + * Fetches the list of board ids the connector should sync. When `boardIds` is + * configured, those are used verbatim; otherwise all accessible active boards + * are enumerated. + */ +async function resolveBoardIds( + accessToken: string, + sourceConfig: Record +): Promise<{ id: string; name: string | null }[]> { + const configured = parseMultiValue(sourceConfig.boardIds) + if (configured.length > 0) { + return configured.map((id) => ({ id, name: null })) + } + + const boards: { id: string; name: string | null }[] = [] + let page = 1 + for (;;) { + const data = await mondayGraphQL<{ boards: { id: string; name: string | null }[] | null }>( + accessToken, + `query ($limit: Int!, $page: Int!) { + boards(limit: $limit, page: $page, state: active) { + id + name + } + }`, + { limit: BOARDS_PAGE_SIZE, page } + ) + const batch = data.boards ?? [] + boards.push(...batch) + if (batch.length < BOARDS_PAGE_SIZE) break + page += 1 + } + return boards +} + +export const mondayConnector: ConnectorConfig = { + id: 'monday', + name: 'Monday.com', + description: 'Sync board items and updates from Monday.com into your knowledge base', + version: '1.0.0', + icon: MondayIcon, + + auth: { + mode: 'oauth', + provider: 'monday', + requiredScopes: ['boards:read', 'updates:read', 'me:read'], + }, + + configFields: [ + { + id: 'boardIds', + title: 'Board IDs', + type: 'short-input', + required: false, + placeholder: 'e.g. 1234567890, 9876543210 (empty = all active boards)', + description: + 'Comma-separated board IDs to sync — find a board ID in its URL (.../boards/). Leave empty to sync items from every active board you can access.', + }, + { + id: 'maxItems', + title: 'Max Items', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record + ): Promise => { + const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0 + const state = decodeCursor(cursor) + + const boards = + (syncContext?.boards as { id: string; name: string | null }[] | undefined) ?? + (await resolveBoardIds(accessToken, sourceConfig)) + if (syncContext) syncContext.boards = boards + + if (state.boardIndex >= boards.length) { + return { documents: [], hasMore: false } + } + + const board = boards[state.boardIndex] + const fallbackBoard = { id: board.id, name: board.name } + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const pageLimit = + maxItems > 0 + ? Math.min(ITEMS_PAGE_SIZE, Math.max(1, maxItems - prevFetched)) + : ITEMS_PAGE_SIZE + + let itemsPage: MondayItemsPage | null + if (state.itemsCursor) { + const data = await mondayGraphQL<{ next_items_page: MondayItemsPage | null }>( + accessToken, + `query ($cursor: String!, $limit: Int!) { + next_items_page(cursor: $cursor, limit: $limit) { + cursor + items { ${ITEM_FIELDS} } + } + }`, + { cursor: state.itemsCursor, limit: pageLimit } + ) + itemsPage = data.next_items_page + } else { + const data = await mondayGraphQL<{ boards: MondayBoard[] | null }>( + accessToken, + `query ($ids: [ID!], $limit: Int!) { + boards(ids: $ids) { + id + name + items_page(limit: $limit) { + cursor + items { ${ITEM_FIELDS} } + } + } + }`, + { ids: [board.id], limit: pageLimit } + ) + itemsPage = data.boards?.[0]?.items_page ?? null + } + + const items = itemsPage?.items ?? [] + const nextItemsCursor = itemsPage?.cursor?.trim() || undefined + + logger.info('Listing Monday.com items', { + boardIndex: state.boardIndex, + boardTotal: boards.length, + boardId: board.id, + itemCount: items.length, + hasItemsCursor: Boolean(state.itemsCursor), + }) + + const allDocuments = items.map((item) => itemToDocument(item, fallbackBoard)) + + let documents = allDocuments + if (maxItems > 0) { + const remaining = Math.max(0, maxItems - prevFetched) + if (allDocuments.length > remaining) { + documents = allDocuments.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxItems > 0 && totalFetched >= maxItems + if (hitLimit && syncContext) syncContext.listingCapped = true + + let nextCursor: string | undefined + let hasMore = false + + if (hitLimit) { + nextCursor = undefined + } else if (nextItemsCursor) { + nextCursor = encodeCursor({ boardIndex: state.boardIndex, itemsCursor: nextItemsCursor }) + hasMore = true + } else if (state.boardIndex + 1 < boards.length) { + nextCursor = encodeCursor({ boardIndex: state.boardIndex + 1 }) + hasMore = true + } + + return { documents, nextCursor, hasMore } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const data = await mondayGraphQL<{ items: MondayItem[] | null }>( + accessToken, + `query ($ids: [ID!]) { + items(ids: $ids) { ${ITEM_FIELDS} } + }`, + { ids: [externalId] } + ) + + const item = data.items?.[0] + if (!item) return null + + const doc = itemToDocument(item) + if (!doc.content.trim()) return null + + return doc + } catch (error) { + logger.warn('Failed to get Monday.com item', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxItems = sourceConfig.maxItems as string | undefined + if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) < 0)) { + return { valid: false, error: 'Max items must be a non-negative number' } + } + + try { + await mondayGraphQL(accessToken, `query { me { id } }`, {}, VALIDATE_RETRY_OPTIONS) + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'boardName', displayName: 'Board', fieldType: 'text' }, + { id: 'groupTitle', displayName: 'Group', fieldType: 'text' }, + { id: 'itemName', displayName: 'Item', fieldType: 'text' }, + { id: 'state', displayName: 'State', fieldType: 'text' }, + { id: 'creatorName', displayName: 'Creator', fieldType: 'text' }, + { id: 'createdAt', displayName: 'Created', fieldType: 'date' }, + { id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.boardName === 'string' && metadata.boardName.trim()) { + result.boardName = metadata.boardName + } + + if (typeof metadata.groupTitle === 'string' && metadata.groupTitle.trim()) { + result.groupTitle = metadata.groupTitle + } + + if (typeof metadata.itemName === 'string' && metadata.itemName.trim()) { + result.itemName = metadata.itemName + } + + if (typeof metadata.state === 'string' && metadata.state.trim()) { + result.state = metadata.state + } + + if (typeof metadata.creatorName === 'string' && metadata.creatorName.trim()) { + result.creatorName = metadata.creatorName + } + + const createdAt = parseTagDate(metadata.createdAt) + if (createdAt) result.createdAt = createdAt + + const updatedAt = parseTagDate(metadata.updatedAt) + if (updatedAt) result.updatedAt = updatedAt + + return result + }, +} diff --git a/apps/sim/connectors/registry.ts b/apps/sim/connectors/registry.ts index 4e87861bde6..a0d468417a9 100644 --- a/apps/sim/connectors/registry.ts +++ b/apps/sim/connectors/registry.ts @@ -1,26 +1,37 @@ import { airtableConnector } from '@/connectors/airtable' import { asanaConnector } from '@/connectors/asana' +import { ashbyConnector } from '@/connectors/ashby' import { confluenceConnector } from '@/connectors/confluence' import { discordConnector } from '@/connectors/discord' +import { docusignConnector } from '@/connectors/docusign' import { dropboxConnector } from '@/connectors/dropbox' import { evernoteConnector } from '@/connectors/evernote' +import { fathomConnector } from '@/connectors/fathom' import { firefliesConnector } from '@/connectors/fireflies' import { githubConnector } from '@/connectors/github' +import { gitlabConnector } from '@/connectors/gitlab' import { gmailConnector } from '@/connectors/gmail' +import { gongConnector } from '@/connectors/gong' import { googleCalendarConnector } from '@/connectors/google-calendar' import { googleDocsConnector } from '@/connectors/google-docs' import { googleDriveConnector } from '@/connectors/google-drive' import { googleSheetsConnector } from '@/connectors/google-sheets' +import { grainConnector } from '@/connectors/grain' +import { granolaConnector } from '@/connectors/granola' +import { greenhouseConnector } from '@/connectors/greenhouse' import { hubspotConnector } from '@/connectors/hubspot' +import { incidentioConnector } from '@/connectors/incidentio' import { intercomConnector } from '@/connectors/intercom' import { jiraConnector } from '@/connectors/jira' import { linearConnector } from '@/connectors/linear' import { microsoftTeamsConnector } from '@/connectors/microsoft-teams' +import { mondayConnector } from '@/connectors/monday' import { notionConnector } from '@/connectors/notion' import { obsidianConnector } from '@/connectors/obsidian' import { onedriveConnector } from '@/connectors/onedrive' import { outlookConnector } from '@/connectors/outlook' import { redditConnector } from '@/connectors/reddit' +import { rootlyConnector } from '@/connectors/rootly' import { salesforceConnector } from '@/connectors/salesforce' import { servicenowConnector } from '@/connectors/servicenow' import { sharepointConnector } from '@/connectors/sharepoint' @@ -34,27 +45,38 @@ import { zoomConnector } from '@/connectors/zoom' export const CONNECTOR_REGISTRY: ConnectorRegistry = { airtable: airtableConnector, asana: asanaConnector, + ashby: ashbyConnector, confluence: confluenceConnector, discord: discordConnector, + docusign: docusignConnector, dropbox: dropboxConnector, evernote: evernoteConnector, + fathom: fathomConnector, fireflies: firefliesConnector, github: githubConnector, + gitlab: gitlabConnector, gmail: gmailConnector, + gong: gongConnector, google_calendar: googleCalendarConnector, google_docs: googleDocsConnector, google_drive: googleDriveConnector, google_sheets: googleSheetsConnector, + grain: grainConnector, + granola: granolaConnector, + greenhouse: greenhouseConnector, hubspot: hubspotConnector, + incidentio: incidentioConnector, intercom: intercomConnector, jira: jiraConnector, linear: linearConnector, microsoft_teams: microsoftTeamsConnector, + monday: mondayConnector, notion: notionConnector, obsidian: obsidianConnector, onedrive: onedriveConnector, outlook: outlookConnector, reddit: redditConnector, + rootly: rootlyConnector, salesforce: salesforceConnector, servicenow: servicenowConnector, sharepoint: sharepointConnector, diff --git a/apps/sim/connectors/rootly/index.ts b/apps/sim/connectors/rootly/index.ts new file mode 100644 index 00000000000..ba9502324f2 --- /dev/null +++ b/apps/sim/connectors/rootly/index.ts @@ -0,0 +1 @@ +export { rootlyConnector } from '@/connectors/rootly/rootly' diff --git a/apps/sim/connectors/rootly/rootly.ts b/apps/sim/connectors/rootly/rootly.ts new file mode 100644 index 00000000000..6fdc86989c7 --- /dev/null +++ b/apps/sim/connectors/rootly/rootly.ts @@ -0,0 +1,660 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { RootlyIcon } from '@/components/icons' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' + +const logger = createLogger('RootlyConnector') + +const ROOTLY_API_BASE = 'https://api.rootly.com/v1' +/** JSON:API media type required by Rootly for all requests. */ +const JSON_API_CONTENT_TYPE = 'application/vnd.api+json' +const PAGE_SIZE = 50 +/** Cap on timeline events appended to a document to keep content bounded. */ +const MAX_TIMELINE_EVENTS = 200 +/** + * JSON:API relationships to embed inline within each incident's `attributes`. + * Rootly omits these unless requested via `include`, so both the list (stub) and + * detail requests pass them to ensure tag metadata is identical on either path. + * Scoped to exactly the relationships this connector reads — `environments`, + * `services`, and `groups` (Rootly's API token for teams) — to avoid fetching + * unused relationship payloads on every incident. + */ +const INCIDENT_INCLUDE = 'environments,services,groups' + +/** + * JSON:API named-resource entry as embedded directly inside incident + * `attributes` for relationships (environments, services, etc.). Each entry + * wraps a `data` object whose `attributes.name` is the human-readable label. + */ +interface RootlyNamedResource { + data?: { + id?: string + type?: string + attributes?: { + name?: string + } + } +} + +/** + * Minimal shape of a Rootly incident's `attributes` object. + * Only the fields this connector reads are typed; Rootly returns many more. + * + * Relationship arrays (environments, services, groups) and the freeform + * `labels` map are embedded inline in the `attributes` of both the list and + * detail responses, so the deferred list stub can derive every tag without an + * extra request. + */ +interface RootlyIncidentAttributes { + title?: string + slug?: string + summary?: string + kind?: string + status?: string + url?: string + short_url?: string + mitigation_message?: string + resolution_message?: string + cancellation_message?: string + retrospective_progress_status?: string + started_at?: string + detected_at?: string + mitigated_at?: string + resolved_at?: string + closed_at?: string + created_at?: string + updated_at?: string + severity?: { + data?: { + id?: string + attributes?: { + name?: string + severity?: string + } + } + } + environments?: RootlyNamedResource[] + services?: RootlyNamedResource[] + groups?: RootlyNamedResource[] + labels?: Record +} + +/** A single JSON:API resource object for an incident. */ +interface RootlyIncidentResource { + id?: string + type?: string + attributes?: RootlyIncidentAttributes +} + +/** Attributes of a Rootly incident timeline event. */ +interface RootlyEventAttributes { + event?: string + visibility?: string + occurred_at?: string + created_at?: string + updated_at?: string +} + +interface RootlyEventResource { + id?: string + type?: string + attributes?: RootlyEventAttributes +} + +/** JSON:API list envelope shared by incidents and events list endpoints. */ +interface RootlyListResponse { + data?: T[] + links?: { + next?: string | null + } + meta?: { + total_count?: number + } +} + +interface RootlyResourceResponse { + data?: T +} + +/** + * Metadata persisted on every incident document, identical between the list + * stub and the hydrated document so `contentHash` and tags stay stable. + */ +interface IncidentMetadata { + status?: string + severityName?: string + severityLevel?: string + kind?: string + incidentDate?: string + resolvedDate?: string + environments?: string[] + services?: string[] + teams?: string[] + labels?: string[] + updatedAt?: string +} + +/** + * Builds the standard JSON:API request headers with Bearer auth. + */ +function buildHeaders(accessToken: string): Record { + return { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': JSON_API_CONTENT_TYPE, + Accept: JSON_API_CONTENT_TYPE, + } +} + +/** + * Derives the metadata bag from an incident's attributes. Used by both the list + * stub and getDocument so the two produce an identical `contentHash`. + */ +function buildMetadata(attrs: RootlyIncidentAttributes): IncidentMetadata { + const severityData = attrs.severity?.data + return { + status: attrs.status ?? undefined, + severityName: severityData?.attributes?.name ?? undefined, + severityLevel: severityData?.attributes?.severity ?? undefined, + kind: attrs.kind ?? undefined, + incidentDate: attrs.started_at ?? attrs.created_at ?? undefined, + resolvedDate: attrs.resolved_at ?? undefined, + environments: namedResourceLabels(attrs.environments), + services: namedResourceLabels(attrs.services), + teams: namedResourceLabels(attrs.groups), + labels: labelPairs(attrs.labels), + updatedAt: attrs.updated_at ?? undefined, + } +} + +/** + * Extracts the human-readable `name` from each JSON:API named-resource entry, + * dropping any without a usable label. + */ +function namedResourceLabels(resources: RootlyNamedResource[] | undefined): string[] | undefined { + if (!Array.isArray(resources)) return undefined + const names: string[] = [] + for (const resource of resources) { + const name = resource.data?.attributes?.name?.trim() + if (name) names.push(name) + } + return names.length > 0 ? names : undefined +} + +/** + * Flattens Rootly's freeform `labels` map (e.g. `{platform: "osx"}`) into + * `key:value` strings so they can be joined into a single searchable tag. + */ +function labelPairs(labels: Record | undefined): string[] | undefined { + if (!labels || typeof labels !== 'object') return undefined + const pairs: string[] = [] + for (const [key, value] of Object.entries(labels)) { + const trimmedKey = key.trim() + if (!trimmedKey) continue + const trimmedValue = typeof value === 'string' ? value.trim() : '' + pairs.push(trimmedValue ? `${trimmedKey}:${trimmedValue}` : trimmedKey) + } + return pairs.length > 0 ? pairs : undefined +} + +/** + * Computes a metadata-based content hash. The formula depends only on the + * incident ID and its `updated_at` timestamp, so the deferred list stub and the + * hydrated `getDocument` result hash identically — change detection keys off + * Rootly's own change indicator rather than the rendered text. + */ +function buildContentHash(id: string, updatedAt: string | undefined): string { + return `rootly:${id}:${updatedAt ?? ''}` +} + +function buildSourceUrl(attrs: RootlyIncidentAttributes): string | undefined { + return attrs.url || attrs.short_url || undefined +} + +/** + * Fetches the incident timeline events, following JSON:API pagination until + * exhausted or the event cap is reached. Returns an empty array on any failure + * so timeline enrichment never blocks document creation. + */ +async function fetchTimelineEvents( + accessToken: string, + incidentId: string +): Promise { + const events: RootlyEventAttributes[] = [] + let pageNumber = 1 + + try { + while (events.length < MAX_TIMELINE_EVENTS) { + const url = `${ROOTLY_API_BASE}/incidents/${encodeURIComponent(incidentId)}/events?page[number]=${pageNumber}&page[size]=${PAGE_SIZE}` + const response = await fetchWithRetry(url, { + method: 'GET', + headers: buildHeaders(accessToken), + }) + + if (!response.ok) { + logger.warn('Failed to fetch Rootly incident timeline', { + incidentId, + status: response.status, + }) + break + } + + const body = (await response.json()) as RootlyListResponse + const pageEvents = body.data ?? [] + for (const event of pageEvents) { + if (event.attributes) events.push(event.attributes) + } + + if (!body.links?.next || pageEvents.length === 0) break + pageNumber += 1 + } + } catch (error) { + logger.warn('Error fetching Rootly incident timeline', { + incidentId, + error: toError(error).message, + }) + } + + return events.slice(0, MAX_TIMELINE_EVENTS) +} + +/** + * Renders an incident plus its timeline into plain-text content. Only sections + * with data are emitted, so resolved incidents read cleanly while open ones omit + * empty resolution fields. + */ +function formatIncidentContent( + attrs: RootlyIncidentAttributes, + events: RootlyEventAttributes[] +): string { + const parts: string[] = [] + + if (attrs.title) parts.push(`Incident: ${attrs.title}`) + if (attrs.status) parts.push(`Status: ${attrs.status}`) + if (attrs.kind) parts.push(`Kind: ${attrs.kind}`) + + const severityName = attrs.severity?.data?.attributes?.name + if (severityName) parts.push(`Severity: ${severityName}`) + + const services = namedResourceLabels(attrs.services) + if (services) parts.push(`Services: ${services.join(', ')}`) + + const teams = namedResourceLabels(attrs.groups) + if (teams) parts.push(`Teams: ${teams.join(', ')}`) + + const environments = namedResourceLabels(attrs.environments) + if (environments) parts.push(`Environments: ${environments.join(', ')}`) + + if (attrs.started_at) parts.push(`Started: ${attrs.started_at}`) + if (attrs.resolved_at) parts.push(`Resolved: ${attrs.resolved_at}`) + + if (attrs.summary?.trim()) { + parts.push('') + parts.push('--- Summary ---') + parts.push(attrs.summary.trim()) + } + + if (attrs.mitigation_message?.trim()) { + parts.push('') + parts.push('--- Mitigation ---') + parts.push(attrs.mitigation_message.trim()) + } + + if (attrs.resolution_message?.trim()) { + parts.push('') + parts.push('--- Resolution ---') + parts.push(attrs.resolution_message.trim()) + } + + if (attrs.cancellation_message?.trim()) { + parts.push('') + parts.push('--- Cancellation ---') + parts.push(attrs.cancellation_message.trim()) + } + + if (events.length > 0) { + parts.push('') + parts.push('--- Timeline ---') + for (const event of events) { + if (!event.event?.trim()) continue + const when = event.occurred_at || event.created_at + parts.push(when ? `${when}: ${event.event.trim()}` : event.event.trim()) + } + } + + return parts.join('\n') +} + +/** + * Builds a deferred list stub for an incident — no content, but carrying the + * exact metadata and hash the hydrated document will produce. + */ +function incidentToStub(resource: RootlyIncidentResource): ExternalDocument | null { + const id = resource.id + const attrs = resource.attributes + if (!id || !attrs) return null + + const metadata = buildMetadata(attrs) + return { + externalId: id, + title: attrs.title?.trim() || `Incident ${id}`, + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(attrs), + contentHash: buildContentHash(id, attrs.updated_at), + metadata: { ...metadata }, + } +} + +/** + * Reads the optional `maxIncidents` cap from sourceConfig, returning 0 when + * unset or invalid (treated as unlimited). + */ +function parseMaxIncidents(sourceConfig: Record): number { + const raw = sourceConfig.maxIncidents + if (raw == null || raw === '') return 0 + const value = Number(raw) + return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0 +} + +export const rootlyConnector: ConnectorConfig = { + id: 'rootly', + name: 'Rootly', + description: 'Sync incidents, postmortems, and timelines from Rootly', + version: '1.0.0', + icon: RootlyIcon, + + auth: { + mode: 'apiKey', + label: 'API Key', + placeholder: 'Enter your Rootly API key', + }, + + supportsIncrementalSync: true, + + configFields: [ + { + id: 'status', + title: 'Filter by Status', + type: 'short-input', + required: false, + placeholder: 'e.g. resolved (default: all)', + description: 'Only sync incidents with this status (e.g. resolved, mitigated, started).', + }, + { + id: 'severity', + title: 'Filter by Severity', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. sev0 (default: all)', + description: + 'Only sync incidents with this severity slug (e.g. sev0, sev1). Leave blank to sync all severities.', + }, + { + id: 'services', + title: 'Filter by Services', + type: 'short-input', + required: false, + mode: 'advanced', + multi: true, + placeholder: 'Service slugs (comma-separated, default: all)', + description: 'Only sync incidents affecting these service slugs.', + }, + { + id: 'teams', + title: 'Filter by Teams', + type: 'short-input', + required: false, + mode: 'advanced', + multi: true, + placeholder: 'Team slugs (comma-separated, default: all)', + description: 'Only sync incidents owned by these team slugs.', + }, + { + id: 'environments', + title: 'Filter by Environments', + type: 'short-input', + required: false, + mode: 'advanced', + multi: true, + placeholder: 'Environment slugs (comma-separated, default: all)', + description: 'Only sync incidents in these environment slugs.', + }, + { + id: 'maxIncidents', + title: 'Max Incidents', + type: 'short-input', + required: false, + placeholder: 'e.g. 200 (default: unlimited)', + }, + ], + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const maxIncidents = parseMaxIncidents(sourceConfig) + const status = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : '' + const severity = typeof sourceConfig.severity === 'string' ? sourceConfig.severity.trim() : '' + const services = parseMultiValue(sourceConfig.services) + const teams = parseMultiValue(sourceConfig.teams) + const environments = parseMultiValue(sourceConfig.environments) + const pageNumber = cursor ? Number(cursor) : 1 + const startPage = Number.isFinite(pageNumber) && pageNumber > 0 ? pageNumber : 1 + + const queryParams = new URLSearchParams() + queryParams.set('page[number]', String(startPage)) + queryParams.set('page[size]', String(PAGE_SIZE)) + queryParams.set('include', INCIDENT_INCLUDE) + if (status) queryParams.set('filter[status]', status) + if (severity) queryParams.set('filter[severity]', severity) + if (services.length > 0) queryParams.set('filter[services]', services.join(',')) + if (teams.length > 0) queryParams.set('filter[teams]', teams.join(',')) + if (environments.length > 0) queryParams.set('filter[environments]', environments.join(',')) + + if (lastSyncAt) { + queryParams.set('filter[updated_at][gt]', lastSyncAt.toISOString()) + queryParams.set('sort', '-updated_at') + } + + const url = `${ROOTLY_API_BASE}/incidents?${queryParams.toString()}` + + logger.info('Listing Rootly incidents', { + pageNumber: startPage, + pageSize: PAGE_SIZE, + status: status || undefined, + incremental: Boolean(lastSyncAt), + }) + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: buildHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Rootly incidents', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Rootly incidents: ${response.status}`) + } + + const body = (await response.json()) as RootlyListResponse + const incidents = body.data ?? [] + + const allDocuments: ExternalDocument[] = [] + for (const incident of incidents) { + const stub = incidentToStub(incident) + if (stub) allDocuments.push(stub) + } + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let documents = allDocuments + if (maxIncidents > 0) { + const remaining = Math.max(0, maxIncidents - prevFetched) + if (allDocuments.length > remaining) { + documents = allDocuments.slice(0, remaining) + } + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasNextLink = Boolean(body.links?.next) + const hasMore = !hitLimit && hasNextLink && incidents.length > 0 + + return { + documents, + nextCursor: hasMore ? String(startPage + 1) : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + _sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (!externalId) return null + + const url = `${ROOTLY_API_BASE}/incidents/${encodeURIComponent(externalId)}?include=${encodeURIComponent(INCIDENT_INCLUDE)}` + const response = await fetchWithRetry(url, { + method: 'GET', + headers: buildHeaders(accessToken), + }) + + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch Rootly incident: ${response.status}`) + } + + const body = (await response.json()) as RootlyResourceResponse + const resource = body.data + const attrs = resource?.attributes + const id = resource?.id + if (!id || !attrs) return null + + const events = await fetchTimelineEvents(accessToken, id) + const content = formatIncidentContent(attrs, events) + if (!content.trim()) { + logger.info('Skipping Rootly incident with no indexable content', { externalId: id }) + return null + } + const metadata = buildMetadata(attrs) + + return { + externalId: id, + title: attrs.title?.trim() || `Incident ${id}`, + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(attrs), + contentHash: buildContentHash(id, attrs.updated_at), + metadata: { ...metadata }, + } + } catch (error) { + logger.warn('Failed to get Rootly incident', { + externalId, + error: toError(error).message, + }) + return null + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxIncidents = sourceConfig.maxIncidents as string | undefined + if (maxIncidents && (Number.isNaN(Number(maxIncidents)) || Number(maxIncidents) < 0)) { + return { valid: false, error: 'Max incidents must be a non-negative number' } + } + + try { + const response = await fetchWithRetry( + `${ROOTLY_API_BASE}/incidents?page[size]=1`, + { + method: 'GET', + headers: buildHeaders(accessToken), + }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Rootly access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + const message = getErrorMessage(error, 'Failed to validate configuration') + return { valid: false, error: message } + } + }, + + tagDefinitions: [ + { id: 'status', displayName: 'Status', fieldType: 'text' }, + { id: 'severity', displayName: 'Severity', fieldType: 'text' }, + { id: 'kind', displayName: 'Kind', fieldType: 'text' }, + { id: 'services', displayName: 'Services', fieldType: 'text' }, + { id: 'teams', displayName: 'Teams', fieldType: 'text' }, + { id: 'environments', displayName: 'Environments', fieldType: 'text' }, + { id: 'labels', displayName: 'Labels', fieldType: 'text' }, + { id: 'incidentDate', displayName: 'Incident Date', fieldType: 'date' }, + { id: 'resolvedDate', displayName: 'Resolved Date', fieldType: 'date' }, + ], + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.status === 'string' && metadata.status.trim()) { + result.status = metadata.status + } + + const severity = + (typeof metadata.severityName === 'string' && metadata.severityName.trim() + ? metadata.severityName + : undefined) ?? + (typeof metadata.severityLevel === 'string' && metadata.severityLevel.trim() + ? metadata.severityLevel + : undefined) + if (severity) result.severity = severity + + if (typeof metadata.kind === 'string' && metadata.kind.trim()) { + result.kind = metadata.kind + } + + const services = joinTagArray(metadata.services) + if (services) result.services = services + + const teams = joinTagArray(metadata.teams) + if (teams) result.teams = teams + + const environments = joinTagArray(metadata.environments) + if (environments) result.environments = environments + + const labels = joinTagArray(metadata.labels) + if (labels) result.labels = labels + + const incidentDate = parseTagDate(metadata.incidentDate) + if (incidentDate) result.incidentDate = incidentDate + + const resolvedDate = parseTagDate(metadata.resolvedDate) + if (resolvedDate) result.resolvedDate = resolvedDate + + return result + }, +} diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index 9f3735e9bd7..8ecad136124 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -1207,7 +1207,7 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { clientId, clientSecret, useBasicAuth: true, - supportsRefreshTokenRotation: false, + supportsRefreshTokenRotation: true, } } case 'dropbox': { diff --git a/apps/sim/tools/granola/get_note.ts b/apps/sim/tools/granola/get_note.ts index a4b1556d7eb..34835f65391 100644 --- a/apps/sim/tools/granola/get_note.ts +++ b/apps/sim/tools/granola/get_note.ts @@ -59,6 +59,7 @@ export const getNoteTool: ToolConfig ({ @@ -101,6 +102,7 @@ export const getNoteTool: ToolConfig { return { 'Content-Type': 'application/json', Authorization: accessToken, - 'API-Version': '2024-10', + 'API-Version': '2026-04', } } From 1cb5a1653b65d2f5bcd15da0df6d68b5029c8ca0 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 2 Jun 2026 18:54:53 -0700 Subject: [PATCH 3/5] fix(auth): show "account already exists" on duplicate email signup (#4855) * fix(auth): show "account already exists" on duplicate email signup * fix(auth): use exact path match for duplicate-email signup check --- apps/sim/lib/auth/auth.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index 40037c938c0..76465b45bec 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -844,6 +844,22 @@ export const auth = betterAuth({ } } + if (ctx.path === '/sign-up/email' && ctx.body?.email) { + const signupEmail = ctx.body.email.toLowerCase() + const [existingUser] = await db + .select({ id: schema.user.id }) + .from(schema.user) + .where(eq(schema.user.email, signupEmail)) + .limit(1) + + if (existingUser) { + throw new APIError('UNPROCESSABLE_ENTITY', { + message: 'User already exists', + code: 'USER_ALREADY_EXISTS', + }) + } + } + return }), }, From b58cd1f6b1e5d53e2b80121fb8f9275e9a780c6e Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 2 Jun 2026 19:31:50 -0700 Subject: [PATCH 4/5] fix(slack): request reactions:read in OAuth URL, drop im:history (#4856) * fix(slack): request reactions:read in OAuth URL, drop im:history * chore(slack): update read-messages missing-scope message to drop im:history --- apps/sim/app/api/tools/slack/read-messages/route.ts | 2 +- apps/sim/lib/oauth/oauth.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/sim/app/api/tools/slack/read-messages/route.ts b/apps/sim/app/api/tools/slack/read-messages/route.ts index 5960b6f20db..712ccbc7f0f 100644 --- a/apps/sim/app/api/tools/slack/read-messages/route.ts +++ b/apps/sim/app/api/tools/slack/read-messages/route.ts @@ -105,7 +105,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { success: false, error: - 'Missing required permissions. Please reconnect your Slack account with the necessary scopes (channels:history, groups:history, im:history).', + 'Missing required permissions. Reconnect your Slack account to grant channel history access (channels:history, groups:history). Reading direct message history is not supported with the Sim bot.', }, { status: 400 } ) diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index 8ecad136124..4f22fb05b1d 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -703,7 +703,6 @@ export const OAUTH_PROVIDERS: Record = { 'chat:write', 'chat:write.public', 'im:write', - 'im:history', 'im:read', 'users:read', // TODO: Add 'users:read.email' once Slack app review is approved @@ -712,6 +711,7 @@ export const OAUTH_PROVIDERS: Record = { 'canvases:read', 'canvases:write', 'reactions:write', + 'reactions:read', ], }, }, From ba2e4cca74acbd03a0c9232d5449b75cc24069c9 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 2 Jun 2026 20:11:15 -0700 Subject: [PATCH 5/5] fix(fathom): skip getDocument when header cache is missing instead of emitting a degraded, un-refreshable record (#4859) --- apps/sim/connectors/fathom/fathom.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/sim/connectors/fathom/fathom.ts b/apps/sim/connectors/fathom/fathom.ts index 8c65ef43cd7..d5aa71fab5a 100644 --- a/apps/sim/connectors/fathom/fathom.ts +++ b/apps/sim/connectors/fathom/fathom.ts @@ -498,18 +498,28 @@ export const fathomConnector: ConnectorConfig = { } const header = readCachedHeader(syncContext, externalId) + if (!header) { + logger.warn( + 'No cached header for Fathom meeting; skipping to avoid an un-refreshable record', + { + externalId, + } + ) + return null + } + const content = formatMeetingContent(header, transcript, summary).trim() if (!content) return null return { externalId, - title: header?.title ?? 'Untitled Fathom Meeting', + title: header.title, content, contentDeferred: false, mimeType: 'text/plain', - sourceUrl: header?.sourceUrl, - contentHash: header?.contentHash ?? `fathom:${externalId}`, - metadata: { ...(header?.metadata ?? { recordingId: externalId }) }, + sourceUrl: header.sourceUrl, + contentHash: header.contentHash, + metadata: { ...header.metadata }, } } catch (error) { logger.warn('Failed to get Fathom meeting', {